TechDocs: Preparers will take and return an etag for cache invalidation

plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.ts

@@ -18,6 +18,11 @@ type buildInfo = {
   [key: string]: number;
 };
 
+// TODO: Build info should be part of TechDocs storage, inside `techdocs_metadata.json`
+// instead of in-memory storage of the Backstage instance.
+// In case of multi-region Backstage deployments, or even using multiple Kubernetes pods,
+// if each instance creates its separate build info in-memory, it will result in duplicate
+// builds per instance. Also if the pod restarts, all the sites will have to be re-built.
 const builds = {} as buildInfo;
 
 /**

plugins/techdocs-backend/src/DocsBuilder/builder.ts

@@ -13,22 +13,22 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import fs from 'fs-extra';
-import os from 'os';
-import path from 'path';
-import Docker from 'dockerode';
-import { Logger } from 'winston';
 import { Entity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import {
+  GeneratorBase,
+  GeneratorBuilder,
+  getLastCommitTimestamp,
+  getLocationForEntity,
+  PreparerBase,
   PreparerBuilder,
   PublisherBase,
-  GeneratorBuilder,
-  PreparerBase,
-  GeneratorBase,
-  getLocationForEntity,
-  getLastCommitTimestamp,
 } from '@backstage/techdocs-common';
+import Docker from 'dockerode';
+import fs from 'fs-extra';
+import os from 'os';
+import path from 'path';
+import { Logger } from 'winston';
 import { BuildMetadataStorage } from '.';
 
 const getEntityId = (entity: Entity) => {
@@ -76,7 +76,9 @@ export class DocsBuilder {
 
   public async build() {
     this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`);
-    const preparedDir = await this.preparer.prepare(this.entity);
+    // TODO: This will throw an error if no further build is required, site is cached.
+    // TODO: Use etag from here and store in memory.
+    const { preparedDir } = await this.preparer.prepare(this.entity);
 
     const parsedLocationAnnotation = getLocationForEntity(this.entity);
 

plugins/techdocs-backend/src/service/router.ts

@@ -13,23 +13,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Logger } from 'winston';
-import Router from 'express-promise-router';
-import express from 'express';
-import Knex from 'knex';
-import fetch from 'cross-fetch';
-import { Config } from '@backstage/config';
-import Docker from 'dockerode';
-import {
-  GeneratorBuilder,
-  PreparerBuilder,
-  PublisherBase,
-  getLocationForEntity,
-} from '@backstage/techdocs-common';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
 import { Entity } from '@backstage/catalog-model';
-import { getEntityNameFromUrlPath } from './helpers';
+import { Config } from '@backstage/config';
+import {
+  GeneratorBuilder,
+  getLocationForEntity,
+  PreparerBuilder,
+  PublisherBase,
+} from '@backstage/techdocs-common';
+import fetch from 'cross-fetch';
+import Docker from 'dockerode';
+import express from 'express';
+import Router from 'express-promise-router';
+import Knex from 'knex';
+import { Logger } from 'winston';
 import { DocsBuilder } from '../DocsBuilder';
+import { getEntityNameFromUrlPath } from './helpers';
 
 type RouterOptions = {
   preparers: PreparerBuilder;
@@ -145,9 +145,7 @@ export async function createRouter({
       });
       switch (publisherType) {
         case 'local':
-          if (!(await docsBuilder.docsUpToDate())) {
-            await docsBuilder.build();
-          }
+          await docsBuilder.build();
           break;
         case 'awsS3':
         case 'azureBlobStorage':
@@ -186,9 +184,11 @@ export async function createRouter({
               'Found pre-generated docs for this entity. Serving them.',
             );
             // TODO: re-trigger build for cache invalidation.
-            // Compare the date modified of the requested file on storage and compare it against
-            // the last modified or last commit timestamp in the repository.
+            // Add build info in techdocs_metadata.json and compare it against
+            // the eTag/commit in the repository.
             // Without this, docs will not be re-built once they have been generated.
+            // Although it is unconventional that anyone will face this issue - because
+            // if you have an external storage, you should be using CI/CD to build and publish docs.
           }
           break;
         default: