Invalidate stale cache entries on read for external builder config when cache is enabled.

Signed-off-by: Eric Peterson <ericpeterson@spotify.com>

plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts

@@ -31,6 +31,19 @@ import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer';
 
 jest.mock('../DocsBuilder');
 
+jest.mock('cross-fetch', () => ({
+  __esModule: true,
+  default: async () => {
+    return {
+      json: async () => {
+        return {
+          build_timestamp: 123,
+        };
+      },
+    };
+  },
+}));
+
 const MockedDocsBuilder = DocsBuilder as jest.MockedClass<typeof DocsBuilder>;
 
 describe('DocsSynchronizer', () => {
@@ -201,4 +214,82 @@ describe('DocsSynchronizer', () => {
       expect(mockResponseHandler.error).toBeCalledWith(error);
     });
   });
+
+  describe('doCacheSync', () => {
+    const entity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      metadata: {
+        uid: '0',
+        name: 'test',
+        namespace: 'default',
+      },
+    };
+
+    it('should not check metadata too often', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(false);
+
+      await docsSynchronizer.doCacheSync({
+        responseHandler: mockResponseHandler,
+        discovery,
+        token: undefined,
+        entity,
+      });
+
+      expect(mockResponseHandler.finish).toBeCalledWith({ updated: false });
+      expect(shouldCheckForUpdate).toBeCalledTimes(1);
+    });
+
+    it('should do nothing if source/cached metadata matches', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
+      (publisher.fetchTechDocsMetadata as jest.Mock).mockResolvedValue({
+        build_timestamp: 123,
+      });
+
+      await docsSynchronizer.doCacheSync({
+        responseHandler: mockResponseHandler,
+        discovery,
+        token: undefined,
+        entity,
+      });
+
+      expect(mockResponseHandler.finish).toBeCalledWith({ updated: false });
+    });
+
+    it('should invalidate expected files when source/cached metadata differ', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
+      (publisher.fetchTechDocsMetadata as jest.Mock).mockResolvedValue({
+        build_timestamp: 456,
+        files: ['index.html'],
+      });
+
+      await docsSynchronizer.doCacheSync({
+        responseHandler: mockResponseHandler,
+        discovery,
+        token: undefined,
+        entity,
+      });
+
+      expect(mockResponseHandler.finish).toBeCalledWith({ updated: true });
+      expect(cache.invalidateMultiple).toHaveBeenCalledWith([
+        'default/Component/test/index.html',
+      ]);
+    });
+
+    it('should gracefully handle errors', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
+      (publisher.fetchTechDocsMetadata as jest.Mock).mockRejectedValue(
+        new Error(),
+      );
+
+      await docsSynchronizer.doCacheSync({
+        responseHandler: mockResponseHandler,
+        discovery,
+        token: undefined,
+        entity,
+      });
+
+      expect(mockResponseHandler.finish).toBeCalledWith({ updated: false });
+    });
+  });
 });

plugins/techdocs-backend/src/service/DocsSynchronizer.ts

@@ -14,7 +14,8 @@
  * limitations under the License.
  */
 
-import { Entity } from '@backstage/catalog-model';
+import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { assertError, NotFoundError } from '@backstage/errors';
 import { ScmIntegrationRegistry } from '@backstage/integration';
@@ -23,10 +24,15 @@ import {
   PreparerBuilder,
   PublisherBase,
 } from '@backstage/techdocs-common';
+import fetch from 'cross-fetch';
 import { PassThrough } from 'stream';
 import * as winston from 'winston';
 import { TechDocsCache } from '../cache';
-import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder';
+import {
+  BuildMetadataStorage,
+  DocsBuilder,
+  shouldCheckForUpdate,
+} from '../DocsBuilder';
 
 export type DocsSynchronizerSyncOpts = {
   log: (message: string) => void;
@@ -151,4 +157,68 @@ export class DocsSynchronizer {
 
     finish({ updated: true });
   }
+
+  async doCacheSync({
+    responseHandler: { finish },
+    discovery,
+    token,
+    entity,
+  }: {
+    responseHandler: DocsSynchronizerSyncOpts;
+    discovery: PluginEndpointDiscovery;
+    token: string | undefined;
+    entity: Entity;
+  }) {
+    // Check if the last update check was too recent.
+    if (!shouldCheckForUpdate(entity.metadata.uid!) || !this.cache) {
+      finish({ updated: false });
+      return;
+    }
+
+    // Fetch techdocs_metadata.json from the publisher and from cache.
+    const baseUrl = await discovery.getBaseUrl('techdocs');
+    const namespace = entity.metadata?.namespace || ENTITY_DEFAULT_NAMESPACE;
+    const kind = entity.kind;
+    const name = entity.metadata.name;
+    const entityTripletPath = `${namespace}/${kind}/${name}`;
+    try {
+      const [sourceMetadata, cachedMetadata] = await Promise.all([
+        this.publisher.fetchTechDocsMetadata({ namespace, kind, name }),
+        fetch(
+          `${baseUrl}/static/docs/${entityTripletPath}/techdocs_metadata.json`,
+          {
+            headers: token ? { Authorization: `Bearer ${token}` } : {},
+          },
+        ).then(
+          f =>
+            f.json().catch(() => undefined) as ReturnType<
+              PublisherBase['fetchTechDocsMetadata']
+            >,
+        ),
+      ]);
+
+      // If build timestamps differ, merge their files[] lists and invalidate all objects.
+      if (sourceMetadata.build_timestamp !== cachedMetadata.build_timestamp) {
+        const files = [
+          ...new Set([
+            ...(sourceMetadata.files || []),
+            ...(cachedMetadata.files || []),
+          ]),
+        ].map(f => `${entityTripletPath}/${f}`);
+        await this.cache.invalidateMultiple(files);
+        finish({ updated: true });
+      } else {
+        finish({ updated: false });
+      }
+    } catch (e) {
+      // In case of error, log and allow the user to go about their business.
+      this.logger.error(
+        `Error syncing cache for ${entityTripletPath}: ${e.message}`,
+      );
+      finish({ updated: false });
+    } finally {
+      // Update the last check time for the entity
+      new BuildMetadataStorage(entity.metadata.uid!).setLastUpdated();
+    }
+  }
 }

plugins/techdocs-backend/src/service/router.ts

@@ -190,6 +190,17 @@ export async function createRouter(
     // If set to 'external', it will assume that an external process (e.g. CI/CD pipeline
     // of the repository) is responsible for building and publishing documentation to the storage provider
     if (config.getString('techdocs.builder') !== 'local') {
+      // However, if caching is enabled, take the opportunity to check and
+      // invalidate stale cache entries.
+      if (cache) {
+        await docsSynchronizer.doCacheSync({
+          responseHandler,
+          discovery,
+          token,
+          entity,
+        });
+        return;
+      }
       responseHandler.finish({ updated: false });
       return;
     }