Merge pull request #6335 from backstage/iameap/techdocs-cache

[TechDocs] Optional static resource caching

.changeset/techdocs-satisfied-you-blinked.md

@@ -0,0 +1,7 @@
+---
+'@backstage/techdocs-common': minor
+'@backstage/plugin-techdocs-backend': minor
+---
+
+Added the ability for the TechDocs Backend to (optionally) leverage a cache
+store to improve performance when reading files from a cloud storage provider.

.changeset/the-renegade-feeling.md

@@ -0,0 +1,42 @@
+---
+'@backstage/create-app': patch
+---
+
+TechDocs Backend may now (optionally) leverage a cache store to improve
+performance when reading content from a cloud storage provider.
+
+To apply this change to an existing app, pass the cache manager from the plugin
+environment to the `createRouter` function in your backend:
+
+```diff
+// packages/backend/src/plugins/techdocs.ts
+
+export default async function createPlugin({
+  logger,
+  config,
+  discovery,
+  reader,
++  cache,
+}: PluginEnvironment): Promise<Router> {
+
+  // ...
+
+  return await createRouter({
+    preparers,
+    generators,
+    publisher,
+    logger,
+    config,
+    discovery,
++    cache,
+  });
+```
+
+If your `PluginEnvironment` does not include a cache manager, be sure you've
+applied [the cache management change][cm-change] to your backend as well.
+
+[Additional configuration][td-rec-arch] is required if you wish to enable
+caching in TechDocs.
+
+[cm-change]: https://github.com/backstage/backstage/blob/master/packages/create-app/CHANGELOG.md#patch-changes-6
+[td-rec-arch]: https://backstage.io/docs/features/techdocs/architecture#recommended-deployment

docs/features/techdocs/architecture.md

@@ -40,7 +40,7 @@ storage system (e.g. AWS S3, GCS or Azure Blob Storage). Read more in
 
 ## Recommended deployment
 
-This is how we recommend deploying TechDocs in production environment.
+This is how we recommend deploying TechDocs in a production environment.
 
 <img data-zoomable src="../../assets/techdocs/architecture-recommended.drawio.svg" alt="TechDocs Architecture diagram" />
 
@@ -58,12 +58,12 @@ Similar to how it is done in the Basic setup, the TechDocs Reader requests
 your configured storage solution for the necessary files and returns them to
 TechDocs Reader.
 
-Note about caching: We have noticed internally that some storage providers can
-be quite slow, which is why we are recommending a cache that sits between the
-TechDocs Reader and the Storage.
-
-_Feel free to suggest better ideas to us in #docs-like-code channel in Discord
-or via a GitHub issue._
+Depending on your chosen cloud storage provider and its real-world proximity to
+your backend server, there may be a comparably high amount of latency when
+loading TechDocs sites using this deployment approach. If you encounter this,
+you can optionally configure the `techdocs-backend` to cache responses in a
+cache store
+[supported by Backstage](../../overview/architecture-overview.md#cache).
 
 ### Security consideration
 

docs/features/techdocs/configuration.md

@@ -135,6 +135,22 @@ techdocs:
   # the old, case-sensitive entity triplet behavior.
   legacyUseCaseSensitiveTripletPaths: false
 
+  # techdocs.cache is optional, and is only recommended when you've configured
+  # an external techdocs.publisher.type above. Also requires backend.cache to
+  # be configured with a valid cache store.
+  cache:
+    # Represents the number of milliseconds a statically built asset should
+    # stay cached. Cache invalidation is handled automatically by the frontend,
+    # which compares the build times in cached metadata vs. canonical storage,
+    # allowing long TTLs (e.g. 1 month/year)
+    ttl: 3600000
+
+    # (Optional) The time (in milliseconds) that the TechDocs backend will wait
+    # for a cache service to respond before continuing on as though the cached
+    # object was not found (e.g. when the cache sercice is unavailable). The
+    # default value is 1000
+    readTimeout: 500
+
   # (Optional and Legacy) TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc.
   # You don't have to specify this anymore.
 

packages/backend/src/plugins/techdocs.ts

@@ -29,6 +29,7 @@ export default async function createPlugin({
   config,
   discovery,
   reader,
+  cache,
 }: PluginEnvironment): Promise<Router> {
   // Preparers are responsible for fetching source files for documentation.
   const preparers = await Preparers.fromConfig(config, {
@@ -64,5 +65,6 @@ export default async function createPlugin({
     logger,
     config,
     discovery,
+    cache,
   });
 }

packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts

@@ -14,6 +14,7 @@ export default async function createPlugin({
   config,
   discovery,
   reader,
+  cache,
 }: PluginEnvironment): Promise<Router> {
   // Preparers are responsible for fetching source files for documentation.
   const preparers = await Preparers.fromConfig(config, {
@@ -49,5 +50,6 @@ export default async function createPlugin({
     logger,
     config,
     discovery,
+    cache,
   });
 }

packages/techdocs-common/api-report.md

@@ -287,6 +287,8 @@ export type TechDocsMetadata = {
   site_name: string;
   site_description: string;
   etag: string;
+  build_timestamp: number;
+  files?: string[];
 };
 
 // Warning: (ae-missing-release-tag) "transformDirLocation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)

packages/techdocs-common/src/stages/generate/helpers.test.ts

@@ -23,7 +23,7 @@ import os from 'os';
 import path, { resolve as resolvePath } from 'path';
 import { ParsedLocationAnnotation } from '../../helpers';
 import {
-  addBuildTimestampMetadata,
+  createOrUpdateMetadata,
   getGeneratorKey,
   getMkdocsYml,
   getRepoUrlFromLocationAnnotation,
@@ -369,13 +369,15 @@ describe('helpers', () => {
   });
 
   describe('addBuildTimestampMetadata', () => {
+    const mockFiles = {
+      'invalid_techdocs_metadata.json': 'dsds',
+      'techdocs_metadata.json': '{"site_name": "Tech Docs"}',
+    };
+
     beforeEach(() => {
       mockFs.restore();
       mockFs({
-        [rootDir]: {
-          'invalid_techdocs_metadata.json': 'dsds',
-          'techdocs_metadata.json': '{"site_name": "Tech Docs"}',
-        },
+        [rootDir]: mockFiles,
       });
     });
 
@@ -385,7 +387,7 @@ describe('helpers', () => {
 
     it('should create the file if it does not exist', async () => {
       const filePath = path.join(rootDir, 'wrong_techdocs_metadata.json');
-      await addBuildTimestampMetadata(filePath, mockLogger);
+      await createOrUpdateMetadata(filePath, mockLogger);
 
       // Check if the file exists
       await expect(
@@ -397,18 +399,28 @@ describe('helpers', () => {
       const filePath = path.join(rootDir, 'invalid_techdocs_metadata.json');
 
       await expect(
-        addBuildTimestampMetadata(filePath, mockLogger),
+        createOrUpdateMetadata(filePath, mockLogger),
       ).rejects.toThrowError('Unexpected token d in JSON at position 0');
     });
 
     it('should add build timestamp to the metadata json', async () => {
       const filePath = path.join(rootDir, 'techdocs_metadata.json');
 
-      await addBuildTimestampMetadata(filePath, mockLogger);
+      await createOrUpdateMetadata(filePath, mockLogger);
 
       const json = await fs.readJson(filePath);
       expect(json.build_timestamp).toBeLessThanOrEqual(Date.now());
     });
+
+    it('should add list of files to the metadata json', async () => {
+      const filePath = path.join(rootDir, 'techdocs_metadata.json');
+
+      await createOrUpdateMetadata(filePath, mockLogger);
+
+      const json = await fs.readJson(filePath);
+      expect(json.files[0]).toEqual(Object.keys(mockFiles)[0]);
+      expect(json.files[1]).toEqual(Object.keys(mockFiles)[1]);
+    });
   });
 
   describe('storeEtagMetadata', () => {

packages/techdocs-common/src/stages/generate/helpers.ts

@@ -27,6 +27,7 @@ import { PassThrough, Writable } from 'stream';
 import { Logger } from 'winston';
 import { ParsedLocationAnnotation } from '../../helpers';
 import { SupportedGeneratorKey } from './types';
+import { getFileTreeRecursively } from '../publish/helpers';
 
 // TODO: Implement proper support for more generators.
 export function getGeneratorKey(entity: Entity): SupportedGeneratorKey {
@@ -345,14 +346,20 @@ export const patchIndexPreBuild = async ({
 };
 
 /**
- * Update the techdocs_metadata.json to add a new build timestamp metadata. Create the .json file if it doesn't exist.
+ * Create or update the techdocs_metadata.json. Values initialized/updated are:
+ * - The build_timestamp (now)
+ * - The list of files generated
  *
  * @param {string} techdocsMetadataPath File path to techdocs_metadata.json
  */
-export const addBuildTimestampMetadata = async (
+export const createOrUpdateMetadata = async (
   techdocsMetadataPath: string,
   logger: Logger,
 ): Promise<void> => {
+  const techdocsMetadataDir = techdocsMetadataPath
+    .split(path.sep)
+    .slice(0, -1)
+    .join(path.sep);
   // check if file exists, create if it does not.
   try {
     await fs.access(techdocsMetadataPath, fs.constants.F_OK);
@@ -372,6 +379,19 @@ export const addBuildTimestampMetadata = async (
   }
 
   json.build_timestamp = Date.now();
+
+  // Get and write generated files to the metadata JSON. Each file string is in
+  // a form appropriate for invalidating the associated object from cache.
+  try {
+    json.files = (await getFileTreeRecursively(techdocsMetadataDir)).map(file =>
+      file.replace(`${techdocsMetadataDir}/`, ''),
+    );
+  } catch (err) {
+    assertError(err);
+    json.files = [];
+    logger.warn(`Unable to add files list to metadata: ${err.message}`);
+  }
+
   await fs.writeJson(techdocsMetadataPath, json);
   return;
 };

packages/techdocs-common/src/stages/generate/techdocs.ts

@@ -23,7 +23,7 @@ import {
   ScmIntegrations,
 } from '@backstage/integration';
 import {
-  addBuildTimestampMetadata,
+  createOrUpdateMetadata,
   getMkdocsYml,
   patchIndexPreBuild,
   patchMkdocsYmlPreBuild,
@@ -164,9 +164,9 @@ export class TechdocsGenerator implements GeneratorBase {
      * Post Generate steps
      */
 
-    // Add build timestamp to techdocs_metadata.json
+    // Add build timestamp and files to techdocs_metadata.json
     // Creates techdocs_metadata.json if file does not exist.
-    await addBuildTimestampMetadata(
+    await createOrUpdateMetadata(
       path.join(outputDir, 'techdocs_metadata.json'),
       childLogger,
     );

packages/techdocs-common/src/stages/publish/awsS3.test.ts

@@ -95,6 +95,7 @@ describe('AwsS3Publish', () => {
     site_name: 'backstage',
     site_description: 'site_content',
     etag: 'etag',
+    build_timestamp: 612741599,
   };
 
   const directory = getEntityRootDir(entity);
@@ -149,21 +150,39 @@ describe('AwsS3Publish', () => {
   describe('publish', () => {
     it('should publish a directory', async () => {
       const publisher = createPublisherFromConfig();
-      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/component/backstage/404.html',
+          `default/component/backstage/index.html`,
+          `default/component/backstage/assets/main.css`,
+        ]),
+      });
     });
 
     it('should publish a directory as well when legacy casing is used', async () => {
       const publisher = createPublisherFromConfig({
         legacyUseCaseSensitiveTripletPaths: true,
       });
-      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/Component/backstage/404.html',
+          `default/Component/backstage/index.html`,
+          `default/Component/backstage/assets/main.css`,
+        ]),
+      });
     });
 
     it('should publish a directory when root path is specified', async () => {
       const publisher = createPublisherFromConfig({
         bucketRootPath: 'backstage-data/techdocs',
       });
-      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'backstage-data/techdocs/default/component/backstage/404.html',
+          `backstage-data/techdocs/default/component/backstage/index.html`,
+          `backstage-data/techdocs/default/component/backstage/assets/main.css`,
+        ]),
+      });
     });
 
     it('should publish a directory when root path is specified and legacy casing is used', async () => {
@@ -171,14 +190,26 @@ describe('AwsS3Publish', () => {
         bucketRootPath: 'backstage-data/techdocs',
         legacyUseCaseSensitiveTripletPaths: true,
       });
-      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'backstage-data/techdocs/default/Component/backstage/404.html',
+          `backstage-data/techdocs/default/Component/backstage/index.html`,
+          `backstage-data/techdocs/default/Component/backstage/assets/main.css`,
+        ]),
+      });
     });
 
     it('should publish a directory when sse is specified', async () => {
       const publisher = createPublisherFromConfig({
         sse: 'aws:kms',
       });
-      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/component/backstage/404.html',
+          'default/component/backstage/index.html',
+          'default/component/backstage/assets/main.css',
+        ]),
+      });
     });
 
     it('should fail to publish a directory', async () => {

packages/techdocs-common/src/stages/publish/awsS3.ts

@@ -39,6 +39,7 @@ import {
 import {
   PublisherBase,
   PublishRequest,
+  PublishResponse,
   ReadinessResponse,
   TechDocsMetadata,
 } from './types';
@@ -214,7 +215,11 @@ export class AwsS3Publish implements PublisherBase {
    * Upload all the files from the generated `directory` to the S3 bucket.
    * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
    */
-  async publish({ entity, directory }: PublishRequest): Promise<void> {
+  async publish({
+    entity,
+    directory,
+  }: PublishRequest): Promise<PublishResponse> {
+    const objects: string[] = [];
     const useLegacyPathCasing = this.legacyPathCasing;
     const bucketRootPath = this.bucketRootPath;
     const sse = this.sse;
@@ -263,6 +268,7 @@ export class AwsS3Publish implements PublisherBase {
             ...(sse && { ServerSideEncryption: sse }),
           } as aws.S3.PutObjectRequest;
 
+          objects.push(params.Key);
           return this.storageClient.upload(params).promise();
         },
         absoluteFilesToUpload,
@@ -311,6 +317,7 @@ export class AwsS3Publish implements PublisherBase {
       const errorMessage = `Unable to delete file(s) from AWS S3. ${error}`;
       this.logger.error(errorMessage);
     }
+    return { objects };
   }
 
   async fetchTechDocsMetadata(

packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts

@@ -89,6 +89,7 @@ describe('AzureBlobStoragePublish', () => {
     site_name: 'backstage',
     site_description: 'site_content',
     etag: 'etag',
+    build_timestamp: 612741599,
   };
 
   const directory = getEntityRootDir(entity);
@@ -154,14 +155,26 @@ describe('AzureBlobStoragePublish', () => {
   describe('publish', () => {
     it('should publish a directory', async () => {
       const publisher = createPublisherFromConfig();
-      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/component/backstage/404.html',
+          `default/component/backstage/index.html`,
+          `default/component/backstage/assets/main.css`,
+        ]),
+      });
     });
 
     it('should publish a directory as well when legacy casing is used', async () => {
       const publisher = createPublisherFromConfig({
         legacyUseCaseSensitiveTripletPaths: true,
       });
-      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/Component/backstage/404.html',
+          `default/Component/backstage/index.html`,
+          `default/Component/backstage/assets/main.css`,
+        ]),
+      });
     });
 
     it('should fail to publish a directory', async () => {

packages/techdocs-common/src/stages/publish/azureBlobStorage.ts

@@ -39,6 +39,7 @@ import {
 import {
   PublisherBase,
   PublishRequest,
+  PublishResponse,
   ReadinessResponse,
   TechDocsMetadata,
 } from './types';
@@ -156,7 +157,11 @@ export class AzureBlobStoragePublish implements PublisherBase {
    * Upload all the files from the generated `directory` to the Azure Blob Storage container.
    * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
    */
-  async publish({ entity, directory }: PublishRequest): Promise<void> {
+  async publish({
+    entity,
+    directory,
+  }: PublishRequest): Promise<PublishResponse> {
+    const objects: string[] = [];
     const useLegacyPathCasing = this.legacyPathCasing;
 
     // First, try to retrieve a list of all individual files currently existing
@@ -194,14 +199,14 @@ export class AzureBlobStoragePublish implements PublisherBase {
           const relativeFilePath = path.normalize(
             path.relative(directory, absoluteFilePath),
           );
+          const remotePath = getCloudPathForLocalPath(
+            entity,
+            relativeFilePath,
+            useLegacyPathCasing,
+          );
+          objects.push(remotePath);
           const response = await container
-            .getBlockBlobClient(
-              getCloudPathForLocalPath(
-                entity,
-                relativeFilePath,
-                useLegacyPathCasing,
-              ),
-            )
+            .getBlockBlobClient(remotePath)
             .uploadFile(absoluteFilePath);
 
           if (response._response.status >= 400) {
@@ -264,6 +269,8 @@ export class AzureBlobStoragePublish implements PublisherBase {
       const errorMessage = `Unable to delete file(s) from Azure. ${error}`;
       this.logger.error(errorMessage);
     }
+
+    return { objects };
   }
 
   private download(containerName: string, blobPath: string): Promise<Buffer> {

packages/techdocs-common/src/stages/publish/googleStorage.test.ts

@@ -88,6 +88,7 @@ describe('GoogleGCSPublish', () => {
     site_name: 'backstage',
     site_description: 'site_content',
     etag: 'etag',
+    build_timestamp: 612741599,
   };
 
   const directory = getEntityRootDir(entity);
@@ -142,21 +143,39 @@ describe('GoogleGCSPublish', () => {
   describe('publish', () => {
     it('should publish a directory', async () => {
       const publisher = createPublisherFromConfig();
-      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/component/backstage/404.html',
+          `default/component/backstage/index.html`,
+          `default/component/backstage/assets/main.css`,
+        ]),
+      });
     });
 
     it('should publish a directory as well when legacy casing is used', async () => {
       const publisher = createPublisherFromConfig({
         legacyUseCaseSensitiveTripletPaths: true,
       });
-      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/Component/backstage/404.html',
+          `default/Component/backstage/index.html`,
+          `default/Component/backstage/assets/main.css`,
+        ]),
+      });
     });
 
     it('should publish a directory when root path is specified', async () => {
       const publisher = createPublisherFromConfig({
         bucketRootPath: 'backstage-data/techdocs',
       });
-      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'backstage-data/techdocs/default/component/backstage/404.html',
+          `backstage-data/techdocs/default/component/backstage/index.html`,
+          `backstage-data/techdocs/default/component/backstage/assets/main.css`,
+        ]),
+      });
     });
 
     it('should publish a directory when root path is specified and legacy casing is used', async () => {
@@ -164,7 +183,13 @@ describe('GoogleGCSPublish', () => {
         bucketRootPath: 'backstage-data/techdocs',
         legacyUseCaseSensitiveTripletPaths: true,
       });
-      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'backstage-data/techdocs/default/Component/backstage/404.html',
+          `backstage-data/techdocs/default/Component/backstage/index.html`,
+          `backstage-data/techdocs/default/Component/backstage/assets/main.css`,
+        ]),
+      });
     });
 
     it('should fail to publish a directory', async () => {

packages/techdocs-common/src/stages/publish/googleStorage.ts

@@ -36,6 +36,7 @@ import { MigrateWriteStream } from './migrations';
 import {
   PublisherBase,
   PublishRequest,
+  PublishResponse,
   ReadinessResponse,
   TechDocsMetadata,
 } from './types';
@@ -145,7 +146,11 @@ export class GoogleGCSPublish implements PublisherBase {
    * Upload all the files from the generated `directory` to the GCS bucket.
    * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
    */
-  async publish({ entity, directory }: PublishRequest): Promise<void> {
+  async publish({
+    entity,
+    directory,
+  }: PublishRequest): Promise<PublishResponse> {
+    const objects: string[] = [];
     const useLegacyPathCasing = this.legacyPathCasing;
     const bucket = this.storageClient.bucket(this.bucketName);
     const bucketRootPath = this.bucketRootPath;
@@ -178,14 +183,14 @@ export class GoogleGCSPublish implements PublisherBase {
       await bulkStorageOperation(
         async absoluteFilePath => {
           const relativeFilePath = path.relative(directory, absoluteFilePath);
-          return await bucket.upload(absoluteFilePath, {
-            destination: getCloudPathForLocalPath(
-              entity,
-              relativeFilePath,
-              useLegacyPathCasing,
-              bucketRootPath,
-            ),
-          });
+          const destination = getCloudPathForLocalPath(
+            entity,
+            relativeFilePath,
+            useLegacyPathCasing,
+            bucketRootPath,
+          );
+          objects.push(destination);
+          return await bucket.upload(absoluteFilePath, { destination });
         },
         absoluteFilesToUpload,
         { concurrencyLimit: 10 },
@@ -228,6 +233,8 @@ export class GoogleGCSPublish implements PublisherBase {
       const errorMessage = `Unable to delete file(s) from Google Cloud Storage. ${error}`;
       this.logger.error(errorMessage);
     }
+
+    return { objects };
   }
 
   fetchTechDocsMetadata(entityName: EntityName): Promise<TechDocsMetadata> {

packages/techdocs-common/src/stages/publish/local.ts

@@ -98,7 +98,10 @@ export class LocalPublish implements PublisherBase {
     };
   }
 
-  publish({ entity, directory }: PublishRequest): Promise<PublishResponse> {
+  async publish({
+    entity,
+    directory,
+  }: PublishRequest): Promise<PublishResponse> {
     const entityNamespace = entity.metadata.namespace ?? 'default';
 
     const publishDir = this.staticEntityPathJoin(
@@ -112,27 +115,30 @@ export class LocalPublish implements PublisherBase {
       fs.mkdirSync(publishDir, { recursive: true });
     }
 
-    return new Promise((resolve, reject) => {
-      fs.copy(directory, publishDir, err => {
-        if (err) {
-          this.logger.debug(
-            `Failed to copy docs from ${directory} to ${publishDir}`,
-          );
-          reject(err);
-        }
-        this.logger.info(`Published site stored at ${publishDir}`);
-        this.discovery
-          .getBaseUrl('techdocs')
-          .then(techdocsApiUrl => {
-            resolve({
-              remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`,
-            });
-          })
-          .catch(reason => {
-            reject(reason);
-          });
-      });
-    });
+    try {
+      await fs.copy(directory, publishDir);
+      this.logger.info(`Published site stored at ${publishDir}`);
+    } catch (error) {
+      this.logger.debug(
+        `Failed to copy docs from ${directory} to ${publishDir}`,
+      );
+      throw error;
+    }
+
+    // Generate publish response.
+    const techdocsApiUrl = await this.discovery.getBaseUrl('techdocs');
+    const publishedFilePaths = (await getFileTreeRecursively(publishDir)).map(
+      abs => {
+        return abs.split(`${staticDocsDir}/`)[1];
+      },
+    );
+
+    return {
+      remoteUrl: `${techdocsApiUrl}/static/docs/${encodeURIComponent(
+        entity.metadata.name,
+      )}`,
+      objects: publishedFilePaths,
+    };
   }
 
   async fetchTechDocsMetadata(

packages/techdocs-common/src/stages/publish/openStackSwift.test.ts

@@ -170,7 +170,13 @@ describe('OpenStackSwiftPublish', () => {
           entity,
           directory: entityRootDir,
         }),
-      ).toBeUndefined();
+      ).toMatchObject({
+        objects: expect.arrayContaining([
+          'test-namespace/TestKind/test-component-name/404.html',
+          `test-namespace/TestKind/test-component-name/index.html`,
+          `test-namespace/TestKind/test-component-name/assets/main.css`,
+        ]),
+      });
     });
 
     it('should fail to publish a directory', async () => {
@@ -241,7 +247,7 @@ describe('OpenStackSwiftPublish', () => {
       mockFs({
         [entityRootDir]: {
           'techdocs_metadata.json':
-            '{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}',
+            '{"site_name": "backstage", "site_description": "site_content", "etag": "etag", "build_timestamp": 612741599}',
         },
       });
 
@@ -249,6 +255,7 @@ describe('OpenStackSwiftPublish', () => {
         site_name: 'backstage',
         site_description: 'site_content',
         etag: 'etag',
+        build_timestamp: 612741599,
       };
       expect(
         await publisher.fetchTechDocsMetadata(entityNameMock),
@@ -263,7 +270,7 @@ describe('OpenStackSwiftPublish', () => {
 
       mockFs({
         [entityRootDir]: {
-          'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`,
+          'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag', 'build_timestamp': 612741599}`,
         },
       });
 
@@ -271,6 +278,7 @@ describe('OpenStackSwiftPublish', () => {
         site_name: 'backstage',
         site_description: 'site_content',
         etag: 'etag',
+        build_timestamp: 612741599,
       };
       expect(
         await publisher.fetchTechDocsMetadata(entityNameMock),

packages/techdocs-common/src/stages/publish/openStackSwift.ts

@@ -32,6 +32,7 @@ import {
 import {
   PublisherBase,
   PublishRequest,
+  PublishResponse,
   ReadinessResponse,
   TechDocsMetadata,
 } from './types';
@@ -139,8 +140,13 @@ export class OpenStackSwiftPublish implements PublisherBase {
    * Upload all the files from the generated `directory` to the OpenStack Swift container.
    * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
    */
-  async publish({ entity, directory }: PublishRequest): Promise<void> {
+  async publish({
+    entity,
+    directory,
+  }: PublishRequest): Promise<PublishResponse> {
     try {
+      const objects: string[] = [];
+
       // Note: OpenStack Swift manages creation of parent directories if they do not exist.
       // So collecting path of only the files is good enough.
       const allFilesToUpload = await getFileTreeRecursively(directory);
@@ -161,6 +167,7 @@ export class OpenStackSwiftPublish implements PublisherBase {
         // The / delimiter is intentional since it represents the cloud storage and not the local file system.
         const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
         const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Swift container file relative path
+        objects.push(destination);
 
         // Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
         const uploadFile = limiter(async () => {
@@ -178,7 +185,7 @@ export class OpenStackSwiftPublish implements PublisherBase {
       this.logger.info(
         `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
       );
-      return;
+      return { objects };
     } catch (e) {
       const errorMessage = `Unable to upload file(s) to OpenStack Swift. ${e}`;
       this.logger.error(errorMessage);

packages/techdocs-common/src/stages/publish/types.ts

@@ -32,9 +32,21 @@ export type PublishRequest = {
   directory: string;
 };
 
-/* `remoteUrl` is the URL which serves files from the local publisher's static directory. */
+/**
+ * Response containing metadata about where files were published and what may
+ * have been published or updated.
+ */
 export type PublishResponse = {
+  /**
+   * The URL which serves files from the local publisher's static directory.
+   */
   remoteUrl?: string;
+  /**
+   * The list of objects (specifically their paths) that were published.
+   * Objects do not have a preceding slash, and match how one would load the
+   * object over the `/static/docs/*` TechDocs Backend Plugin endpoint.
+   */
+  objects?: string[];
 } | void;
 
 /**
@@ -53,6 +65,8 @@ export type TechDocsMetadata = {
   site_name: string;
   site_description: string;
   etag: string;
+  build_timestamp: number;
+  files?: string[];
 };
 
 export type MigrateRequest = {

plugins/techdocs-backend/api-report.md

@@ -10,6 +10,7 @@ import express from 'express';
 import { GeneratorBuilder } from '@backstage/techdocs-common';
 import { Knex } from 'knex';
 import { Logger as Logger_2 } from 'winston';
+import { PluginCacheManager } from '@backstage/backend-common';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
 import { PreparerBuilder } from '@backstage/techdocs-common';
 import { PublisherBase } from '@backstage/techdocs-common';

plugins/techdocs-backend/config.d.ts

@@ -226,6 +226,32 @@ export interface Config {
           };
         };
 
+    /**
+     * @example http://localhost:7007/api/techdocs
+     * Techdocs cache information
+     */
+    cache?: {
+      /**
+       * The cache time-to-live for TechDocs sites (in milliseconds). Set this
+       * to a non-zero value to cache TechDocs sites and assets as they are
+       * read from storage.
+       *
+       * Note: you must also configure `backend.cache` appropriately as well,
+       * and to pass a PluginCacheManager instance to TechDocs Backend's
+       * createRouter method in your backend.
+       */
+      ttl: number;
+
+      /**
+       * The time (in milliseconds) that the TechDocs backend will wait for
+       * a cache service to respond before continuing on as though the cached
+       * object was not found (e.g. when the cache sercice is unavailable).
+       *
+       * Defaults to 1000 milliseconds.
+       */
+      readTimeout?: number;
+    };
+
     /**
      * @example http://localhost:7007/api/techdocs
      * @visibility frontend

plugins/techdocs-backend/package.json

@@ -40,6 +40,7 @@
     "@backstage/search-common": "^0.2.1",
     "@backstage/techdocs-common": "^0.10.8",
     "@types/express": "^4.17.6",
+    "cross-fetch": "^3.0.6",
     "dockerode": "^3.3.1",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",

plugins/techdocs-backend/src/DocsBuilder/builder.ts

@@ -36,6 +36,7 @@ import path from 'path';
 import { Writable } from 'stream';
 import { Logger } from 'winston';
 import { BuildMetadataStorage } from './BuildMetadataStorage';
+import { TechDocsCache } from '../cache';
 
 type DocsBuilderArguments = {
   preparers: PreparerBuilder;
@@ -46,6 +47,7 @@ type DocsBuilderArguments = {
   config: Config;
   scmIntegrations: ScmIntegrationRegistry;
   logStream?: Writable;
+  cache?: TechDocsCache;
 };
 
 export class DocsBuilder {
@@ -57,6 +59,7 @@ export class DocsBuilder {
   private config: Config;
   private scmIntegrations: ScmIntegrationRegistry;
   private logStream: Writable | undefined;
+  private cache?: TechDocsCache;
 
   constructor({
     preparers,
@@ -67,6 +70,7 @@ export class DocsBuilder {
     config,
     scmIntegrations,
     logStream,
+    cache,
   }: DocsBuilderArguments) {
     this.preparer = preparers.get(entity);
     this.generator = generators.get(entity);
@@ -76,6 +80,7 @@ export class DocsBuilder {
     this.config = config;
     this.scmIntegrations = scmIntegrations;
     this.logStream = logStream;
+    this.cache = cache;
   }
 
   /**
@@ -210,11 +215,19 @@ export class DocsBuilder {
       )}`,
     );
 
-    await this.publisher.publish({
+    const published = await this.publisher.publish({
       entity: this.entity,
       directory: outputDir,
     });
 
+    // Invalidate the cache for any published objects.
+    if (this.cache && published && published?.objects?.length) {
+      this.logger.debug(
+        `Invalidating ${published.objects.length} cache objects`,
+      );
+      await this.cache.invalidateMultiple(published.objects);
+    }
+
     try {
       // Not a blocker hence no need to await this.
       fs.remove(outputDir);

plugins/techdocs-backend/src/cache/TechDocsCache.test.ts

@@ -0,0 +1,168 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { CacheClient, getVoidLogger } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+import { CacheInvalidationError, TechDocsCache } from './TechDocsCache';
+
+const cached = (str: string): string => {
+  return Buffer.from(str).toString('base64');
+};
+
+describe('TechDocsCache', () => {
+  let CacheUnderTest: TechDocsCache;
+  let MockClient: jest.Mocked<CacheClient>;
+
+  beforeEach(() => {
+    MockClient = {
+      get: jest.fn(),
+      set: jest.fn(),
+      delete: jest.fn(),
+    };
+    CacheUnderTest = TechDocsCache.fromConfig(new ConfigReader({}), {
+      cache: MockClient,
+      logger: getVoidLogger(),
+    });
+  });
+
+  describe('get', () => {
+    it('returns undefined if no response', async () => {
+      const expectedPath = 'some/index.html';
+      MockClient.get.mockResolvedValueOnce(undefined);
+
+      const actual = await CacheUnderTest.get(expectedPath);
+      expect(MockClient.get).toHaveBeenCalledWith(expectedPath);
+      expect(actual).toBe(undefined);
+    });
+
+    it('returns undefined if cache get throws', async () => {
+      const expectedPath = 'some/index.html';
+      MockClient.get.mockRejectedValueOnce(new Error());
+
+      const actual = await CacheUnderTest.get(expectedPath);
+      expect(actual).toBe(undefined);
+    });
+
+    it('returns undefined if no response after 1s by default', async () => {
+      const expectedPath = 'some/index.html';
+      MockClient.get.mockImplementationOnce(() => {
+        return new Promise(resolve => {
+          setTimeout(() => resolve(cached('value')), 1500);
+        });
+      });
+      const actual = await CacheUnderTest.get(expectedPath);
+      expect(actual).toBe(undefined);
+    });
+
+    it('returns undefined if no response after configured readTimeout', async () => {
+      const expectedPath = 'some/index.html';
+      MockClient.get.mockImplementationOnce(() => {
+        return new Promise(resolve => {
+          setTimeout(() => resolve(cached('value')), 20);
+        });
+      });
+
+      CacheUnderTest = TechDocsCache.fromConfig(
+        new ConfigReader({
+          techdocs: { cache: { readTimeout: 10 } },
+        }),
+        {
+          cache: MockClient,
+          logger: getVoidLogger(),
+        },
+      );
+
+      const actual = await CacheUnderTest.get(expectedPath);
+      expect(actual).toBe(undefined);
+    });
+
+    it('returns data if cache get returns it', async () => {
+      const expectedPath = 'some/index.html';
+      MockClient.get.mockResolvedValueOnce(cached('expected value'));
+
+      const actual = await CacheUnderTest.get(expectedPath);
+      expect(actual?.toString()).toBe('expected value');
+    });
+  });
+
+  describe('set', () => {
+    it('sets a base64-encoded string', async () => {
+      const expectedPath = 'some/index.html';
+      MockClient.set.mockResolvedValueOnce(undefined);
+
+      await CacheUnderTest.set(expectedPath, Buffer.from('some data'));
+      expect(MockClient.set).toHaveBeenCalledWith(
+        expectedPath,
+        cached('some data'),
+      );
+    });
+
+    it('does not throw if client throws', () => {
+      MockClient.set.mockRejectedValueOnce(new Error());
+      expect(() => CacheUnderTest.set('i.html', Buffer.from(''))).not.toThrow();
+    });
+  });
+
+  describe('invalidate', () => {
+    it('calls delete on client', async () => {
+      const expectedPath = 'some/index.html';
+      MockClient.delete.mockResolvedValueOnce(undefined);
+
+      await CacheUnderTest.invalidate(expectedPath);
+      expect(MockClient.delete).toHaveBeenCalledWith(expectedPath);
+    });
+  });
+
+  describe('invalidateMultiple', () => {
+    it('calls delete once per given path', async () => {
+      const expectedPaths = ['one/index.html', 'two/index.html'];
+      MockClient.delete.mockResolvedValue(undefined);
+
+      await CacheUnderTest.invalidateMultiple(expectedPaths);
+      expect(MockClient.delete).toHaveBeenNthCalledWith(1, expectedPaths[0]);
+      expect(MockClient.delete).toHaveBeenNthCalledWith(2, expectedPaths[1]);
+    });
+
+    it('returns an array of as many paths provided', async () => {
+      const expectedPaths = ['one/index.html', 'two/index.html'];
+      MockClient.delete.mockResolvedValue(undefined);
+
+      const actual = await CacheUnderTest.invalidateMultiple(expectedPaths);
+      expect(actual.length).toBe(2);
+    });
+
+    it('calls delete on all paths even if the first rejects', async () => {
+      const expectedPaths = ['one/index.html', 'two/index.html'];
+      MockClient.delete.mockRejectedValueOnce(new Error());
+      MockClient.delete.mockResolvedValueOnce(undefined);
+
+      await expect(
+        CacheUnderTest.invalidateMultiple(expectedPaths),
+      ).rejects.toThrowError(CacheInvalidationError);
+      expect(MockClient.delete).toHaveBeenCalledTimes(2);
+    });
+
+    it('rejects with invalidations error response', async () => {
+      const expectedPaths = ['one/index.html', 'two/index.html'];
+      MockClient.delete.mockResolvedValueOnce(undefined);
+      MockClient.delete.mockRejectedValueOnce(new Error());
+
+      await expect(
+        CacheUnderTest.invalidateMultiple.bind(CacheUnderTest, expectedPaths),
+      ).rejects.toThrow(CacheInvalidationError);
+    });
+  });
+});

plugins/techdocs-backend/src/cache/TechDocsCache.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { CacheClient } from '@backstage/backend-common';
+import { assertError, CustomErrorBase } from '@backstage/errors';
+import { Config } from '@backstage/config';
+import { Logger } from 'winston';
+
+export class CacheInvalidationError extends CustomErrorBase {}
+
+export class TechDocsCache {
+  protected readonly cache: CacheClient;
+  protected readonly logger: Logger;
+  protected readonly readTimeout: number;
+
+  private constructor({
+    cache,
+    logger,
+    readTimeout,
+  }: {
+    cache: CacheClient;
+    logger: Logger;
+    readTimeout: number;
+  }) {
+    this.cache = cache;
+    this.logger = logger;
+    this.readTimeout = readTimeout;
+  }
+
+  static fromConfig(
+    config: Config,
+    { cache, logger }: { cache: CacheClient; logger: Logger },
+  ) {
+    const timeout = config.getOptionalNumber('techdocs.cache.readTimeout');
+    const readTimeout = timeout === undefined ? 1000 : timeout;
+    return new TechDocsCache({ cache, logger, readTimeout });
+  }
+
+  async get(path: string): Promise<Buffer | undefined> {
+    try {
+      // Promise.race ensures we don't hang the client for long if the cache is
+      // temporarily unreachable.
+      const response = (await Promise.race([
+        this.cache.get(path),
+        new Promise(cancelAfter => setTimeout(cancelAfter, this.readTimeout)),
+      ])) as string | undefined;
+
+      if (response !== undefined) {
+        this.logger.debug(`Cache hit: ${path}`);
+        return Buffer.from(response, 'base64');
+      }
+
+      this.logger.debug(`Cache miss: ${path}`);
+      return response;
+    } catch (e) {
+      assertError(e);
+      this.logger.warn(`Error getting cache entry ${path}: ${e.message}`);
+      this.logger.debug(e.stack);
+      return undefined;
+    }
+  }
+
+  async set(path: string, data: Buffer): Promise<void> {
+    this.logger.debug(`Writing cache entry for ${path}`);
+    this.cache
+      .set(path, data.toString('base64'))
+      .catch(e => this.logger.error('write error', e));
+  }
+
+  async invalidate(path: string): Promise<void> {
+    return this.cache.delete(path);
+  }
+
+  async invalidateMultiple(
+    paths: string[],
+  ): Promise<PromiseSettledResult<void>[]> {
+    const settled = await Promise.allSettled(
+      paths.map(path => this.cache.delete(path)),
+    );
+    const rejected = settled.filter(
+      s => s.status === 'rejected',
+    ) as PromiseRejectedResult[];
+
+    if (rejected.length) {
+      throw new CacheInvalidationError(
+        'TechDocs cache invalidation error',
+        rejected,
+      );
+    }
+
+    return settled;
+  }
+}

plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import express from 'express';
+import request from 'supertest';
+import { createCacheMiddleware, TechDocsCache } from '.';
+
+/**
+ * Mocks cached HTTP response.
+ */
+const getMockHttpResponseFor = (content: string): Buffer => {
+  return Buffer.concat([
+    Buffer.from(`HTTP/1.1 200 OK
+Content-Type: text/plain; charset=utf-8
+Accept-Ranges: bytes
+Cache-Control: public, max-age=0
+Last-Modified: Sat, 1 Jul 2021 12:00:00 GMT
+Date: Sat, 1 Jul 2021 12:00:00 GMT
+Connection: close
+Content-Length: ${content.length}\n\n`),
+    Buffer.from(content),
+  ]);
+};
+
+/**
+ * Wait for the socket to close. Works because, above, we set connection: close
+ */
+const waitForSocketClose = () => new Promise(resolve => setTimeout(resolve, 0));
+
+describe('createCacheMiddleware', () => {
+  let cache: jest.Mocked<TechDocsCache>;
+  let app: express.Express;
+
+  beforeEach(async () => {
+    cache = {
+      get: jest.fn().mockResolvedValue(undefined),
+      set: jest.fn().mockResolvedValue(undefined),
+      invalidate: jest.fn().mockResolvedValue(undefined),
+      invalidateMultiple: jest.fn().mockResolvedValue(undefined),
+    } as unknown as jest.Mocked<TechDocsCache>;
+    const router = await createCacheMiddleware({
+      logger: getVoidLogger(),
+      cache,
+    });
+    app = express().use(router);
+    app.use((req, res, next) => {
+      // By default, send cacheable content.
+      if (req.path !== '/static/docs/error.png') {
+        res.send('default-response');
+      } else {
+        next(new Error());
+      }
+    });
+  });
+
+  describe('middleware', () => {
+    it('does not apply to non-static/docs paths', async () => {
+      await request(app)
+        .get('/static/not-docs')
+        .expect(200, 'default-response');
+
+      expect(cache.set).not.toHaveBeenCalled();
+    });
+
+    it('responds with cached response', async () => {
+      cache.get.mockResolvedValueOnce(getMockHttpResponseFor('xyz'));
+
+      await request(app).get('/static/docs/foo.html').expect(200, 'xyz');
+
+      await waitForSocketClose();
+      expect(cache.set).not.toHaveBeenCalled();
+    });
+
+    it('sets cache when content is cacheable', async () => {
+      const expectedPath = 'default/api/xyz/index.html';
+      await request(app)
+        .get(`/static/docs/${expectedPath}`)
+        .expect(200, 'default-response');
+
+      await waitForSocketClose();
+      expect(cache.set).toHaveBeenCalled();
+
+      const [actualPath, actualBuffer] = (cache.set as jest.Mock).mock.calls[0];
+      expect(actualPath).toBe(expectedPath);
+      expect(actualBuffer.toString()).toContain('default-response');
+    });
+
+    it('does not set cache on error', async () => {
+      await request(app).get('/static/docs/error.png').expect(500);
+
+      await waitForSocketClose();
+      expect(cache.set).not.toHaveBeenCalled();
+    });
+  });
+});

plugins/techdocs-backend/src/cache/cacheMiddleware.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Router } from 'express';
+import router from 'express-promise-router';
+import { Logger } from 'winston';
+import { TechDocsCache } from './TechDocsCache';
+
+type CacheMiddlewareOptions = {
+  cache: TechDocsCache;
+  logger: Logger;
+};
+
+type ErrorCallback = (err?: Error) => void;
+
+export const createCacheMiddleware = ({
+  cache,
+}: CacheMiddlewareOptions): Router => {
+  const cacheMiddleware = router();
+
+  // Middleware that, through socket monkey patching, captures responses as
+  // they're sent over /static/docs/* and caches them. Subsequent requests are
+  // loaded from cache. Cache key is the object's path (after `/static/docs/`).
+  cacheMiddleware.use(async (req, res, next) => {
+    const socket = res.socket;
+    const isCacheable = req.path.startsWith('/static/docs/');
+
+    // Continue early if this is non-cacheable, or there's no socket.
+    if (!isCacheable || !socket) {
+      next();
+      return;
+    }
+
+    // Make concrete references to these things.
+    const reqPath = decodeURI(req.path.match(/\/static\/docs\/(.*)$/)![1]);
+    const realEnd = socket.end.bind(socket);
+    const realWrite = socket.write.bind(socket);
+    let writeToCache = true;
+    const chunks: Buffer[] = [];
+
+    // Monkey-patch the response's socket to keep track of chunks as they are
+    // written over the wire.
+    socket.write = (
+      data: string | Uint8Array,
+      encoding?: BufferEncoding | ErrorCallback,
+      callback?: ErrorCallback,
+    ) => {
+      chunks.push(Buffer.from(data));
+      if (typeof encoding === 'function') {
+        return realWrite(data, encoding);
+      }
+      return realWrite(data, encoding, callback);
+    };
+
+    // When a socket is closed, if there were no errors and the data written
+    // over the socket should be cached, cache it!
+    socket.on('close', async hadError => {
+      const content = Buffer.concat(chunks);
+      const head = content.toString('utf8', 0, 12);
+      if (writeToCache && !hadError && head.match(/HTTP\/\d\.\d 200/)) {
+        await cache.set(reqPath, content);
+      }
+    });
+
+    // Attempt to retrieve data from the cache.
+    const cached = await cache.get(reqPath);
+
+    // If there is a cache hit, write it out on the socket, ensure we don't re-
+    // cache the data, and prevent going back to canonical storage by never
+    // calling next().
+    if (cached) {
+      writeToCache = false;
+      realEnd(cached);
+      return;
+    }
+
+    // No data retrieved from cache: allow retrieval from canonical storage.
+    next();
+  });
+
+  return cacheMiddleware;
+};

plugins/techdocs-backend/src/cache/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { createCacheMiddleware } from './cacheMiddleware';
+export { TechDocsCache } from './TechDocsCache';

plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts

@@ -25,11 +25,25 @@ import {
   PreparerBuilder,
   PublisherBase,
 } from '@backstage/techdocs-common';
+import { TechDocsCache } from '../cache';
 import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder';
 import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer';
 
 jest.mock('../DocsBuilder');
 
+jest.mock('cross-fetch', () => ({
+  __esModule: true,
+  default: async () => {
+    return {
+      json: async () => {
+        return {
+          build_timestamp: 123,
+        };
+      },
+    };
+  },
+}));
+
 const MockedDocsBuilder = DocsBuilder as jest.MockedClass<typeof DocsBuilder>;
 
 describe('DocsSynchronizer', () => {
@@ -52,6 +66,12 @@ describe('DocsSynchronizer', () => {
     getBaseUrl: jest.fn(),
     getExternalBaseUrl: jest.fn(),
   };
+  const cache: jest.Mocked<TechDocsCache> = {
+    get: jest.fn(),
+    set: jest.fn(),
+    invalidate: jest.fn(),
+    invalidateMultiple: jest.fn(),
+  } as unknown as jest.Mocked<TechDocsCache>;
 
   let docsSynchronizer: DocsSynchronizer;
   const mockResponseHandler: jest.Mocked<DocsSynchronizerSyncOpts> = {
@@ -71,6 +91,7 @@ describe('DocsSynchronizer', () => {
       config: new ConfigReader({}),
       logger: getVoidLogger(),
       scmIntegrations: ScmIntegrations.fromConfig(new ConfigReader({})),
+      cache,
     });
   });
 
@@ -193,4 +214,112 @@ describe('DocsSynchronizer', () => {
       expect(mockResponseHandler.error).toBeCalledWith(error);
     });
   });
+
+  describe('doCacheSync', () => {
+    const entity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      metadata: {
+        uid: '0',
+        name: 'test',
+        namespace: 'default',
+      },
+    };
+
+    it('should not check metadata too often', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(false);
+
+      await docsSynchronizer.doCacheSync({
+        responseHandler: mockResponseHandler,
+        discovery,
+        token: undefined,
+        entity,
+      });
+
+      expect(mockResponseHandler.finish).toBeCalledWith({ updated: false });
+      expect(shouldCheckForUpdate).toBeCalledTimes(1);
+    });
+
+    it('should do nothing if source/cached metadata matches', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
+      (publisher.fetchTechDocsMetadata as jest.Mock).mockResolvedValue({
+        build_timestamp: 123,
+      });
+
+      await docsSynchronizer.doCacheSync({
+        responseHandler: mockResponseHandler,
+        discovery,
+        token: undefined,
+        entity,
+      });
+
+      expect(mockResponseHandler.finish).toBeCalledWith({ updated: false });
+    });
+
+    it('should invalidate expected files when source/cached metadata differ', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
+      (publisher.fetchTechDocsMetadata as jest.Mock).mockResolvedValue({
+        build_timestamp: 456,
+        files: ['index.html'],
+      });
+
+      await docsSynchronizer.doCacheSync({
+        responseHandler: mockResponseHandler,
+        discovery,
+        token: undefined,
+        entity,
+      });
+
+      expect(mockResponseHandler.finish).toBeCalledWith({ updated: true });
+      expect(cache.invalidateMultiple).toHaveBeenCalledWith([
+        'default/component/test/index.html',
+      ]);
+    });
+
+    it('should invalidate expected files when source/cached metadata differ with legacy casing', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
+      (publisher.fetchTechDocsMetadata as jest.Mock).mockResolvedValue({
+        build_timestamp: 456,
+        files: ['index.html'],
+      });
+
+      const docsSynchronizerWithLegacy = new DocsSynchronizer({
+        publisher,
+        config: new ConfigReader({
+          techdocs: { legacyUseCaseSensitiveTripletPaths: true },
+        }),
+        logger: getVoidLogger(),
+        scmIntegrations: ScmIntegrations.fromConfig(new ConfigReader({})),
+        cache,
+      });
+
+      await docsSynchronizerWithLegacy.doCacheSync({
+        responseHandler: mockResponseHandler,
+        discovery,
+        token: undefined,
+        entity,
+      });
+
+      expect(mockResponseHandler.finish).toBeCalledWith({ updated: true });
+      expect(cache.invalidateMultiple).toHaveBeenCalledWith([
+        'default/Component/test/index.html',
+      ]);
+    });
+
+    it('should gracefully handle errors', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
+      (publisher.fetchTechDocsMetadata as jest.Mock).mockRejectedValue(
+        new Error(),
+      );
+
+      await docsSynchronizer.doCacheSync({
+        responseHandler: mockResponseHandler,
+        discovery,
+        token: undefined,
+        entity,
+      });
+
+      expect(mockResponseHandler.finish).toBeCalledWith({ updated: false });
+    });
+  });
 });

plugins/techdocs-backend/src/service/DocsSynchronizer.ts

@@ -14,7 +14,8 @@
  * limitations under the License.
  */
 
-import { Entity } from '@backstage/catalog-model';
+import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { assertError, NotFoundError } from '@backstage/errors';
 import { ScmIntegrationRegistry } from '@backstage/integration';
@@ -23,9 +24,15 @@ import {
   PreparerBuilder,
   PublisherBase,
 } from '@backstage/techdocs-common';
+import fetch from 'cross-fetch';
 import { PassThrough } from 'stream';
 import * as winston from 'winston';
-import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder';
+import { TechDocsCache } from '../cache';
+import {
+  BuildMetadataStorage,
+  DocsBuilder,
+  shouldCheckForUpdate,
+} from '../DocsBuilder';
 
 export type DocsSynchronizerSyncOpts = {
   log: (message: string) => void;
@@ -38,22 +45,26 @@ export class DocsSynchronizer {
   private readonly logger: winston.Logger;
   private readonly config: Config;
   private readonly scmIntegrations: ScmIntegrationRegistry;
+  private readonly cache: TechDocsCache | undefined;
 
   constructor({
     publisher,
     logger,
     config,
     scmIntegrations,
+    cache,
   }: {
     publisher: PublisherBase;
     logger: winston.Logger;
     config: Config;
     scmIntegrations: ScmIntegrationRegistry;
+    cache: TechDocsCache | undefined;
   }) {
     this.config = config;
     this.logger = logger;
     this.publisher = publisher;
     this.scmIntegrations = scmIntegrations;
+    this.cache = cache;
   }
 
   async doSync({
@@ -104,6 +115,7 @@ export class DocsSynchronizer {
         config: this.config,
         scmIntegrations: this.scmIntegrations,
         logStream,
+        cache: this.cache,
       });
 
       const updated = await docsBuilder.build();
@@ -145,4 +157,76 @@ export class DocsSynchronizer {
 
     finish({ updated: true });
   }
+
+  async doCacheSync({
+    responseHandler: { finish },
+    discovery,
+    token,
+    entity,
+  }: {
+    responseHandler: DocsSynchronizerSyncOpts;
+    discovery: PluginEndpointDiscovery;
+    token: string | undefined;
+    entity: Entity;
+  }) {
+    // Check if the last update check was too recent.
+    if (!shouldCheckForUpdate(entity.metadata.uid!) || !this.cache) {
+      finish({ updated: false });
+      return;
+    }
+
+    // Fetch techdocs_metadata.json from the publisher and from cache.
+    const baseUrl = await discovery.getBaseUrl('techdocs');
+    const namespace = entity.metadata?.namespace || ENTITY_DEFAULT_NAMESPACE;
+    const kind = entity.kind;
+    const name = entity.metadata.name;
+    const legacyPathCasing =
+      this.config.getOptionalBoolean(
+        'techdocs.legacyUseCaseSensitiveTripletPaths',
+      ) || false;
+    const tripletPath = `${namespace}/${kind}/${name}`;
+    const entityTripletPath = `${
+      legacyPathCasing ? tripletPath : tripletPath.toLocaleLowerCase('en-US')
+    }`;
+    try {
+      const [sourceMetadata, cachedMetadata] = await Promise.all([
+        this.publisher.fetchTechDocsMetadata({ namespace, kind, name }),
+        fetch(
+          `${baseUrl}/static/docs/${entityTripletPath}/techdocs_metadata.json`,
+          {
+            headers: token ? { Authorization: `Bearer ${token}` } : {},
+          },
+        ).then(
+          f =>
+            f.json().catch(() => undefined) as ReturnType<
+              PublisherBase['fetchTechDocsMetadata']
+            >,
+        ),
+      ]);
+
+      // If build timestamps differ, merge their files[] lists and invalidate all objects.
+      if (sourceMetadata.build_timestamp !== cachedMetadata.build_timestamp) {
+        const files = [
+          ...new Set([
+            ...(sourceMetadata.files || []),
+            ...(cachedMetadata.files || []),
+          ]),
+        ].map(f => `${entityTripletPath}/${f}`);
+        await this.cache.invalidateMultiple(files);
+        finish({ updated: true });
+      } else {
+        finish({ updated: false });
+      }
+    } catch (e) {
+      assertError(e);
+      // In case of error, log and allow the user to go about their business.
+      this.logger.error(
+        `Error syncing cache for ${entityTripletPath}: ${e.message}`,
+      );
+      finish({ updated: false });
+    } finally {
+      // Update the last check time for the entity
+      new BuildMetadataStorage(entity.metadata.uid!).setLastUpdated();
+    }
+  }
 }

plugins/techdocs-backend/src/service/router.ts

@@ -13,7 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import {
+  PluginEndpointDiscovery,
+  PluginCacheManager,
+} from '@backstage/backend-common';
 import { CatalogClient } from '@backstage/catalog-client';
 import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
@@ -31,6 +34,7 @@ import { Knex } from 'knex';
 import { Logger } from 'winston';
 import { ScmIntegrations } from '@backstage/integration';
 import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer';
+import { createCacheMiddleware, TechDocsCache } from '../cache';
 
 /**
  * All of the required dependencies for running TechDocs in the "out-of-the-box"
@@ -44,6 +48,7 @@ type OutOfTheBoxDeploymentOptions = {
   discovery: PluginEndpointDiscovery;
   database?: Knex; // TODO: Make database required when we're implementing database stuff.
   config: Config;
+  cache?: PluginCacheManager;
 };
 
 /**
@@ -80,12 +85,22 @@ export async function createRouter(
   const router = Router();
   const { publisher, config, logger, discovery } = options;
   const catalogClient = new CatalogClient({ discoveryApi: discovery });
+
+  // Set up a cache client if configured.
+  let cache: TechDocsCache | undefined;
+  const defaultTtl = config.getOptionalNumber('techdocs.cache.ttl');
+  if (isOutOfTheBoxOption(options) && options.cache && defaultTtl) {
+    const cacheClient = options.cache.getClient({ defaultTtl });
+    cache = TechDocsCache.fromConfig(config, { cache: cacheClient, logger });
+  }
+
   const scmIntegrations = ScmIntegrations.fromConfig(config);
   const docsSynchronizer = new DocsSynchronizer({
     publisher,
     logger,
     config,
     scmIntegrations,
+    cache,
   });
 
   router.get('/metadata/techdocs/:namespace/:kind/:name', async (req, res) => {
@@ -175,6 +190,17 @@ export async function createRouter(
     // If set to 'external', it will assume that an external process (e.g. CI/CD pipeline
     // of the repository) is responsible for building and publishing documentation to the storage provider
     if (config.getString('techdocs.builder') !== 'local') {
+      // However, if caching is enabled, take the opportunity to check and
+      // invalidate stale cache entries.
+      if (cache) {
+        await docsSynchronizer.doCacheSync({
+          responseHandler,
+          discovery,
+          token,
+          entity,
+        });
+        return;
+      }
       responseHandler.finish({ updated: false });
       return;
     }
@@ -199,6 +225,11 @@ export async function createRouter(
     );
   });
 
+  // If a cache manager was provided, attach the cache middleware.
+  if (cache) {
+    router.use(createCacheMiddleware({ logger, cache }));
+  }
+
   // Route middleware which serves files from the storage set in the publisher.
   router.use('/static/docs', publisher.docsRouter());