Merge pull request #4540 from backstage/orkohunter/techdocs-store-etag-in-filesystem-based-cache

.changeset/techdocs-spotty-cooks-wait.md

@@ -0,0 +1,6 @@
+---
+'@backstage/techdocs-common': patch
+'@backstage/plugin-techdocs-backend': patch
+---
+
+Add etag of the prepared file tree to techdocs_metadata.json in the storage

packages/techdocs-common/src/stages/generate/helpers.test.ts

@@ -29,6 +29,7 @@ import {
   isValidRepoUrlForMkdocs,
   patchMkdocsYmlPreBuild,
   runDockerContainer,
+  storeEtagMetadata,
 } from './helpers';
 
 const mockEntity = {
@@ -362,7 +363,7 @@ describe('helpers', () => {
 
       await expect(
         addBuildTimestampMetadata(filePath, mockLogger),
-      ).rejects.toThrowError();
+      ).rejects.toThrowError('Unexpected token d in JSON at position 0');
     });
 
     it('should add build timestamp to the metadata json', async () => {
@@ -374,4 +375,37 @@ describe('helpers', () => {
       expect(json.build_timestamp).toBeLessThanOrEqual(Date.now());
     });
   });
+
+  describe('storeEtagMetadata', () => {
+    beforeEach(() => {
+      mockFs.restore();
+      mockFs({
+        [rootDir]: {
+          'invalid_techdocs_metadata.json': 'dsds',
+          'techdocs_metadata.json': '{"site_name": "Tech Docs"}',
+        },
+      });
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    it('should throw error when the JSON is invalid', async () => {
+      const filePath = path.join(rootDir, 'invalid_techdocs_metadata.json');
+
+      await expect(
+        storeEtagMetadata(filePath, 'etag123abc'),
+      ).rejects.toThrowError('Unexpected token d in JSON at position 0');
+    });
+
+    it('should add etag to the metadata json', async () => {
+      const filePath = path.join(rootDir, 'techdocs_metadata.json');
+
+      await storeEtagMetadata(filePath, 'etag123abc');
+
+      const json = await fs.readJson(filePath);
+      expect(json.etag).toBe('etag123abc');
+    });
+  });
 });

packages/techdocs-common/src/stages/generate/helpers.ts

@@ -306,3 +306,20 @@ export const addBuildTimestampMetadata = async (
   await fs.writeJson(techdocsMetadataPath, json);
   return;
 };
+
+/**
+ * Update the techdocs_metadata.json to add etag of the prepared tree (e.g. commit SHA or actual Etag of the resource).
+ * This is helpful to check if a TechDocs site in storage has gone outdated, without maintaining an in-memory build info
+ * per Backstage instance.
+ *
+ * @param {string} techdocsMetadataPath File path to techdocs_metadata.json
+ * @param {string} etag
+ */
+export const storeEtagMetadata = async (
+  techdocsMetadataPath: string,
+  etag: string,
+): Promise<void> => {
+  const json = await fs.readJson(techdocsMetadataPath);
+  json.etag = etag;
+  await fs.writeJson(techdocsMetadataPath, json);
+};

packages/techdocs-common/src/stages/generate/techdocs.ts

@@ -23,6 +23,7 @@ import {
   patchMkdocsYmlPreBuild,
   runCommand,
   runDockerContainer,
+  storeEtagMetadata,
 } from './helpers';
 import { GeneratorBase, GeneratorRunOptions } from './types';
 
@@ -62,6 +63,7 @@ export class TechdocsGenerator implements GeneratorBase {
     outputDir,
     dockerClient,
     parsedLocationAnnotation,
+    etag,
   }: GeneratorRunOptions): Promise<void> {
     const [log, logStream] = createStream();
 
@@ -119,12 +121,24 @@ export class TechdocsGenerator implements GeneratorBase {
       );
     }
 
-    // Post Generate steps
+    /**
+     * Post Generate steps
+     */
 
     // Add build timestamp to techdocs_metadata.json
-    addBuildTimestampMetadata(
+    // Creates techdocs_metadata.json if file does not exist.
+    await addBuildTimestampMetadata(
       path.join(outputDir, 'techdocs_metadata.json'),
       this.logger,
     );
+
+    // Add etag of the prepared tree to techdocs_metadata.json
+    // Assumes that the file already exists.
+    if (etag) {
+      await storeEtagMetadata(
+        path.join(outputDir, 'techdocs_metadata.json'),
+        etag,
+      );
+    }
   }
 }

packages/techdocs-common/src/stages/generate/types.ts

@@ -13,9 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Writable } from 'stream';
-import Docker from 'dockerode';
 import { Entity } from '@backstage/catalog-model';
+import Docker from 'dockerode';
+import { Writable } from 'stream';
 import { ParsedLocationAnnotation } from '../../helpers';
 
 /**
@@ -25,6 +25,7 @@ import { ParsedLocationAnnotation } from '../../helpers';
  * @param {string} outputDir Directory to store generated docs in. Usually - a newly created temporary directory.
  * @param {Docker} dockerClient A docker client to run any generator on top of your directory
  * @param {ParsedLocationAnnotation} parsedLocationAnnotation backstage.io/techdocs-ref annotation of an entity
+ * @param {string} etag A unique identifier for the prepared tree e.g. commit SHA. If provided it will be stored in techdocs_metadata.json.
  * @param {Writable} [logStream] A dedicated log stream
  */
 export type GeneratorRunOptions = {
@@ -32,6 +33,7 @@ export type GeneratorRunOptions = {
   outputDir: string;
   dockerClient: Docker;
   parsedLocationAnnotation?: ParsedLocationAnnotation;
+  etag?: string;
   logStream?: Writable;
 };
 

plugins/techdocs-backend/src/DocsBuilder/builder.ts

@@ -130,6 +130,7 @@ export class DocsBuilder {
       outputDir,
       dockerClient: this.dockerClient,
       parsedLocationAnnotation,
+      etag,
     });
 
     this.logger.debug(`Generated files temporarily stored at ${outputDir}`);