From 47dece5e3b783691d37b2723fe6c4f2c84613338 Mon Sep 17 00:00:00 2001
From: Johan Haals <johan.haals@gmail.com>
Date: Fri, 24 Sep 2021 12:04:27 +0200
Subject: [PATCH] Add docs for implementing cachable processing

Signed-off-by: Johan Haals <johan.haals@gmail.com>
---
 .../software-catalog/external-integrations.md | 94 +++++++++++++++++++
 1 file changed, 94 insertions(+)

diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md
index 247324a46c..b4e382fb37 100644
--- a/docs/features/software-catalog/external-integrations.md
+++ b/docs/features/software-catalog/external-integrations.md
@@ -171,3 +171,97 @@ export default async function createPlugin(
 
 Start up the backend - it should now start reading from the previously
 registered location and you'll see your entities start to appear in Backstage.
+
+## Caching processing results
+
+The catalog periodically refresh entities in the catalog and by doing so it call
+out to external systems to fetch changes. This can be taxing for upstream
+services and large deployments may get rate limited if too many requests are
+sent. Luckily many external systems provide ETag support to check for changes
+which usually don't count towards the quota and saves resources both internally
+and externally.
+
+The catalog has built in support for caching when refreshing external locations
+in GitHub. This example aims to demonstrate the same behavior for `system-x`
+that we implemented earlier.
+
+```ts
+import { UrlReader } from '@backstage/backend-common';
+import { Entity, LocationSpec } from '@backstage/catalog-model';
+import {
+  results,
+  CatalogProcessor,
+  CatalogProcessorEmit,
+  CatalogProcessorCache,
+} from '@backstage/plugin-catalog-backend';
+import { CatalogProcessorParser } from '.';
+
+// Change this key to if the processor has undergone major changes
+// and the existing cache data is no longer compatible.
+const CACHE_KEY = 'v1';
+
+// CacheItem is our cache containing ETag used in the upstream request
+// as well as the processing result used when the Etag matches.
+type CacheItem = {
+  etag: string;
+  entity: Entity;
+};
+
+export class SystemXReaderProcessor implements CatalogProcessor {
+  constructor(private readonly reader: UrlReader) {}
+
+  // It's recommended to give the processor a unique name.
+  getProcessorName() {
+    return 'system-x-processor';
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+    _parser: CatalogProcessorParser,
+    cache: CatalogProcessorCache,
+  ): Promise<boolean> {
+    // Pick a custom location type string. A location will be
+    // registered later with this type.
+    if (location.type !== 'system-x') {
+      return false;
+    }
+    const cacheItem = await cache.get<CacheItem>(CACHE_KEY);
+    try {
+      // This assumes an URL reader that returns the response together with the ETag.
+      // We send the ETag from the previous run if it exists.
+      // The previous ETag will be set in the headers for the outgoing request and system-x
+      // is going to throw NOT_MODIFIED (HTTP 304) if the ETag matches.
+      const response = await this.reader.readUrl?.(location.target, {
+        etag: cacheItem?.etag,
+      });
+      if (!response) {
+        // readUrl is currently optional to implement so we have to check if we get a response back.
+        throw new Error(
+          'No URL reader that can parse system-x targets installed',
+        );
+      }
+
+      // For this example the JSON payload is a single entity.
+      const entity: Entity = JSON.parse(response.buffer.toString());
+      emit(results.entity(location, entity));
+
+      // Update the cache with the new ETag and entity used for the next run.
+      await cache.set<CacheItem>(CACHE_KEY, {
+        etag: response.etag ? response.etag : '',
+        entity,
+      });
+    } catch (error) {
+      if (error.name === 'NotModifiedError' && cacheItem) {
+        // The ETag matches and we have a cached value from the previous run.
+        emit(results.entity(location, cacheItem.entity));
+      }
+      const message = `Unable to read ${location.type}, ${error}`;
+      emit(results.generalError(location, message));
+    }
+
+    return true;
+  }
+}
+```