TechDocs: Preparers will take and return an etag for cache invalidation

docs/features/techdocs/getting-started.md

@@ -157,15 +157,17 @@ techdocs:
   builder: 'local'
 ```
 
-Set `techdocs.builder` to `'local'` if you want your TechDocs Backend to be
-responsible for generating documentation sites. If set to `'external'`,
-Backstage will assume that the sites are being generated on each entity's CI/CD
-pipeline, and are being stored in a storage somewhere.
+Note that we recommend generating docs on CI/CD instead. Read more in the
+"Basic" and "Recommended" sections of the
+[TechDocs Architecture](architecture.md). But if you want to get started quickly
+set `techdocs.builder` to `'local'` so that TechDocs Backend is responsible for
+generating documentation sites. If set to `'external'`, Backstage will assume
+that the sites are being generated on each entity's CI/CD pipeline, and are
+being stored in a storage somewhere.
 
 When `techdocs.builder` is set to `'external'`, TechDocs becomes more or less a
 read-only experience where it serves static files from a storage containing all
-the generated documentation. Read more in the "Basic" and "Recommended" sections
-of the [TechDocs Architecture](architecture.md).
+the generated documentation.
 
 ### Choosing storage (publisher)
 

packages/techdocs-common/src/stages/prepare/commonGit.test.ts

@@ -16,8 +16,8 @@
 
 import { getVoidLogger } from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
-import { CommonGitPreparer } from './commonGit';
 import { checkoutGitRepository } from '../../helpers';
+import { CommonGitPreparer } from './commonGit';
 
 function normalizePath(path: string) {
   return path
@@ -57,9 +57,9 @@ describe('commonGit preparer', () => {
         'github:https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component',
     });
 
-    const tempDocsPath = await preparer.prepare(mockEntity);
+    const { preparedDir } = await preparer.prepare(mockEntity);
     expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
-    expect(normalizePath(tempDocsPath)).toEqual(
+    expect(normalizePath(preparedDir)).toEqual(
       '/tmp/backstage-repo/org/name/branch/plugins/techdocs-backend/examples/documented-component',
     );
   });
@@ -72,9 +72,9 @@ describe('commonGit preparer', () => {
         'gitlab:https://gitlab.com/xesjkeee/go-logger/blob/master/catalog-info.yaml',
     });
 
-    const tempDocsPath = await preparer.prepare(mockEntity);
+    const { preparedDir } = await preparer.prepare(mockEntity);
     expect(checkoutGitRepository).toHaveBeenCalledTimes(2);
-    expect(normalizePath(tempDocsPath)).toEqual(
+    expect(normalizePath(preparedDir)).toEqual(
       '/tmp/backstage-repo/org/name/branch/catalog-info.yaml',
     );
   });
@@ -87,9 +87,9 @@ describe('commonGit preparer', () => {
         'azure/api:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml',
     });
 
-    const tempDocsPath = await preparer.prepare(mockEntity);
+    const { preparedDir } = await preparer.prepare(mockEntity);
     expect(checkoutGitRepository).toHaveBeenCalledTimes(3);
-    expect(normalizePath(tempDocsPath)).toEqual(
+    expect(normalizePath(preparedDir)).toEqual(
       '/tmp/backstage-repo/org/name/branch/template.yaml',
     );
   });

packages/techdocs-common/src/stages/prepare/commonGit.ts

@@ -13,14 +13,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import path from 'path';
-import parseGitUrl from 'git-url-parse';
 import { Entity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
-import { PreparerBase } from './types';
-import { parseReferenceAnnotation, checkoutGitRepository } from '../../helpers';
-
+import parseGitUrl from 'git-url-parse';
+import path from 'path';
 import { Logger } from 'winston';
+import { checkoutGitRepository, parseReferenceAnnotation } from '../../helpers';
+import { PreparerBase, PreparerResponse } from './types';
 
 export class CommonGitPreparer implements PreparerBase {
   private readonly config: Config;
@@ -31,7 +30,7 @@ export class CommonGitPreparer implements PreparerBase {
     this.logger = logger;
   }
 
-  async prepare(entity: Entity): Promise<string> {
+  async prepare(entity: Entity): Promise<PreparerResponse> {
     const { target } = parseReferenceAnnotation(
       'backstage.io/techdocs-ref',
       entity,
@@ -45,7 +44,13 @@ export class CommonGitPreparer implements PreparerBase {
       );
       const parsedGitLocation = parseGitUrl(target);
 
-      return path.join(repoPath, parsedGitLocation.filepath);
+      // TODO: Return git commit sha
+      const etag = '';
+
+      return {
+        preparedDir: path.join(repoPath, parsedGitLocation.filepath),
+        etag,
+      };
     } catch (error) {
       this.logger.debug(`Repo checkout failed with error ${error.message}`);
       throw error;

packages/techdocs-common/src/stages/prepare/dir.test.ts

@@ -15,8 +15,8 @@
  */
 import { getVoidLogger, UrlReader } from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
-import { DirectoryPreparer } from './dir';
 import { checkoutGitRepository } from '../../helpers';
+import { DirectoryPreparer } from './dir';
 
 function normalizePath(path: string) {
   return path
@@ -66,9 +66,8 @@ describe('directory preparer', () => {
       'backstage.io/techdocs-ref': 'dir:./our-documentation',
     });
 
-    expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual(
-      '/directory/our-documentation',
-    );
+    const { preparedDir } = await directoryPreparer.prepare(mockEntity);
+    expect(normalizePath(preparedDir)).toEqual('/directory/our-documentation');
   });
 
   it('should merge managed-by-location and techdocs-ref when techdocs-ref is absolute', async () => {
@@ -84,9 +83,8 @@ describe('directory preparer', () => {
       'backstage.io/techdocs-ref': 'dir:/our-documentation/techdocs',
     });
 
-    expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual(
-      '/our-documentation/techdocs',
-    );
+    const { preparedDir } = await directoryPreparer.prepare(mockEntity);
+    expect(normalizePath(preparedDir)).toEqual('/our-documentation/techdocs');
   });
 
   it('should merge managed-by-location and techdocs-ref when managed-by-location is a git repository', async () => {
@@ -102,7 +100,8 @@ describe('directory preparer', () => {
       'backstage.io/techdocs-ref': 'dir:./docs',
     });
 
-    expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual(
+    const { preparedDir } = await directoryPreparer.prepare(mockEntity);
+    expect(normalizePath(preparedDir)).toEqual(
       '/tmp/backstage-repo/org/name/branch/docs',
     );
     expect(checkoutGitRepository).toHaveBeenCalledTimes(1);

packages/techdocs-common/src/stages/prepare/dir.ts

@@ -13,14 +13,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { PreparerBase } from './types';
+import { InputError, UrlReader } from '@backstage/backend-common';
 import { Entity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
-import path from 'path';
-import { parseReferenceAnnotation, checkoutGitRepository } from '../../helpers';
-import { UrlReader, InputError } from '@backstage/backend-common';
 import parseGitUrl from 'git-url-parse';
+import path from 'path';
 import { Logger } from 'winston';
+import { checkoutGitRepository, parseReferenceAnnotation } from '../../helpers';
+import { PreparerBase, PreparerResponse } from './types';
 
 export class DirectoryPreparer implements PreparerBase {
   constructor(
@@ -68,7 +68,7 @@ export class DirectoryPreparer implements PreparerBase {
     }
   }
 
-  async prepare(entity: Entity): Promise<string> {
+  async prepare(entity: Entity): Promise<PreparerResponse> {
     const { target } = parseReferenceAnnotation(
       'backstage.io/techdocs-ref',
       entity,
@@ -78,8 +78,12 @@ export class DirectoryPreparer implements PreparerBase {
       entity,
     );
 
-    return new Promise(resolve => {
-      resolve(path.resolve(managedByLocationDirectory, target));
-    });
+    // TODO: etag will be returned as a commit sha from resolveManagedByLocationToDir.
+    const etag = '';
+
+    return {
+      preparedDir: path.resolve(managedByLocationDirectory, target),
+      etag,
+    };
   }
 }

packages/techdocs-common/src/stages/prepare/types.ts

@@ -16,13 +16,31 @@
 import type { Entity } from '@backstage/catalog-model';
 import { Logger } from 'winston';
 
+export type PreparerResponse = {
+  /**
+   * The path to directory where the tree is downloaded.
+   */
+  preparedDir: string;
+  /**
+   * A unique identifer of the tree blob, usually the commit SHA or etag from the target.
+   */
+  etag: string;
+};
+
 export type PreparerBase = {
   /**
    * Given an Entity definition from the Service Catalog, go and prepare a directory
-   * with contents from the location in temporary storage and return the path
+   * with contents from the location in temporary storage and return the path.
+   *
    * @param entity The entity from the Service Catalog
+   * @param opts.etag (Optional) If etag is provider, it will be used to check if the target has
+   * updated since the last build.
+   * @throws {NotModifiedError} when the prepared directory has not been changed since the last build.
    */
-  prepare(entity: Entity, opts?: { logger: Logger }): Promise<string>;
+  prepare(
+    entity: Entity,
+    opts?: { logger: Logger; etag?: string },
+  ): Promise<PreparerResponse>;
 };
 
 export type PreparerBuilder = {
@@ -30,10 +48,14 @@ export type PreparerBuilder = {
   get(entity: Entity): PreparerBase;
 };
 
+/**
+ * Everything except `url` will be deprecated.
+ * Read more https://github.com/backstage/backstage/issues/4409
+ */
 export type RemoteProtocol =
+  | 'url'
   | 'dir'
   | 'github'
   | 'gitlab'
   | 'file'
-  | 'azure/api'
-  | 'url';
+  | 'azure/api';

packages/techdocs-common/src/stages/prepare/url.ts

@@ -13,11 +13,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Logger } from 'winston';
-import { Entity } from '@backstage/catalog-model';
 import { UrlReader } from '@backstage/backend-common';
-import { PreparerBase } from './types';
+import { Entity } from '@backstage/catalog-model';
+import { Logger } from 'winston';
 import { getDocFilesFromRepository } from '../../helpers';
+import { PreparerBase, PreparerResponse } from './types';
 
 export class UrlPreparer implements PreparerBase {
   private readonly logger: Logger;
@@ -28,9 +28,15 @@ export class UrlPreparer implements PreparerBase {
     this.reader = reader;
   }
 
-  async prepare(entity: Entity): Promise<string> {
+  async prepare(entity: Entity): Promise<PreparerResponse> {
     try {
-      return getDocFilesFromRepository(this.reader, entity);
+      const preparedDir = await getDocFilesFromRepository(this.reader, entity);
+      // TODO: This will be actual etag from URL Reader.
+      const etag = '';
+      return {
+        preparedDir,
+        etag,
+      };
     } catch (error) {
       this.logger.debug(
         `Unable to fetch files for building docs ${error.message}`,

plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.ts

@@ -18,6 +18,11 @@ type buildInfo = {
   [key: string]: number;
 };
 
+// TODO: Build info should be part of TechDocs storage, inside `techdocs_metadata.json`
+// instead of in-memory storage of the Backstage instance.
+// In case of multi-region Backstage deployments, or even using multiple Kubernetes pods,
+// if each instance creates its separate build info in-memory, it will result in duplicate
+// builds per instance. Also if the pod restarts, all the sites will have to be re-built.
 const builds = {} as buildInfo;
 
 /**

plugins/techdocs-backend/src/DocsBuilder/builder.ts

@@ -13,22 +13,22 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import fs from 'fs-extra';
-import os from 'os';
-import path from 'path';
-import Docker from 'dockerode';
-import { Logger } from 'winston';
 import { Entity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import {
+  GeneratorBase,
+  GeneratorBuilder,
+  getLastCommitTimestamp,
+  getLocationForEntity,
+  PreparerBase,
   PreparerBuilder,
   PublisherBase,
-  GeneratorBuilder,
-  PreparerBase,
-  GeneratorBase,
-  getLocationForEntity,
-  getLastCommitTimestamp,
 } from '@backstage/techdocs-common';
+import Docker from 'dockerode';
+import fs from 'fs-extra';
+import os from 'os';
+import path from 'path';
+import { Logger } from 'winston';
 import { BuildMetadataStorage } from '.';
 
 const getEntityId = (entity: Entity) => {
@@ -76,7 +76,9 @@ export class DocsBuilder {
 
   public async build() {
     this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`);
-    const preparedDir = await this.preparer.prepare(this.entity);
+    // TODO: This will throw an error if no further build is required, site is cached.
+    // TODO: Use etag from here and store in memory.
+    const { preparedDir } = await this.preparer.prepare(this.entity);
 
     const parsedLocationAnnotation = getLocationForEntity(this.entity);
 

plugins/techdocs-backend/src/service/router.ts

@@ -13,23 +13,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Logger } from 'winston';
-import Router from 'express-promise-router';
-import express from 'express';
-import Knex from 'knex';
-import fetch from 'cross-fetch';
-import { Config } from '@backstage/config';
-import Docker from 'dockerode';
-import {
-  GeneratorBuilder,
-  PreparerBuilder,
-  PublisherBase,
-  getLocationForEntity,
-} from '@backstage/techdocs-common';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
 import { Entity } from '@backstage/catalog-model';
-import { getEntityNameFromUrlPath } from './helpers';
+import { Config } from '@backstage/config';
+import {
+  GeneratorBuilder,
+  getLocationForEntity,
+  PreparerBuilder,
+  PublisherBase,
+} from '@backstage/techdocs-common';
+import fetch from 'cross-fetch';
+import Docker from 'dockerode';
+import express from 'express';
+import Router from 'express-promise-router';
+import Knex from 'knex';
+import { Logger } from 'winston';
 import { DocsBuilder } from '../DocsBuilder';
+import { getEntityNameFromUrlPath } from './helpers';
 
 type RouterOptions = {
   preparers: PreparerBuilder;
@@ -145,9 +145,7 @@ export async function createRouter({
       });
       switch (publisherType) {
         case 'local':
-          if (!(await docsBuilder.docsUpToDate())) {
-            await docsBuilder.build();
-          }
+          await docsBuilder.build();
           break;
         case 'awsS3':
         case 'azureBlobStorage':
@@ -186,9 +184,11 @@ export async function createRouter({
               'Found pre-generated docs for this entity. Serving them.',
             );
             // TODO: re-trigger build for cache invalidation.
-            // Compare the date modified of the requested file on storage and compare it against
-            // the last modified or last commit timestamp in the repository.
+            // Add build info in techdocs_metadata.json and compare it against
+            // the eTag/commit in the repository.
             // Without this, docs will not be re-built once they have been generated.
+            // Although it is unconventional that anyone will face this issue - because
+            // if you have an external storage, you should be using CI/CD to build and publish docs.
           }
           break;
         default: