Add docs for implementing cachable processing

Signed-off-by: Johan Haals <johan.haals@gmail.com>
2021-09-24 12:04:27 +02:00
parent c6c308ab83
commit 47dece5e3b
1 changed files with 94 additions and 0 deletions
@@ -171,3 +171,97 @@ export default async function createPlugin(

 Start up the backend - it should now start reading from the previously
 registered location and you'll see your entities start to appear in Backstage.
+
+## Caching processing results
+
+The catalog periodically refresh entities in the catalog and by doing so it call
+out to external systems to fetch changes. This can be taxing for upstream
+services and large deployments may get rate limited if too many requests are
+sent. Luckily many external systems provide ETag support to check for changes
+which usually don't count towards the quota and saves resources both internally
+and externally.
+
+The catalog has built in support for caching when refreshing external locations
+in GitHub. This example aims to demonstrate the same behavior for `system-x`
+that we implemented earlier.
+
+```ts
+import { UrlReader } from '@backstage/backend-common';
+import { Entity, LocationSpec } from '@backstage/catalog-model';
+import {
+  results,
+  CatalogProcessor,
+  CatalogProcessorEmit,
+  CatalogProcessorCache,
+} from '@backstage/plugin-catalog-backend';
+import { CatalogProcessorParser } from '.';
+
+// Change this key to if the processor has undergone major changes
+// and the existing cache data is no longer compatible.
+const CACHE_KEY = 'v1';
+
+// CacheItem is our cache containing ETag used in the upstream request
+// as well as the processing result used when the Etag matches.
+type CacheItem = {
+  etag: string;
+  entity: Entity;
+};
+
+export class SystemXReaderProcessor implements CatalogProcessor {
+  constructor(private readonly reader: UrlReader) {}
+
+  // It's recommended to give the processor a unique name.
+  getProcessorName() {
+    return 'system-x-processor';
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+    _parser: CatalogProcessorParser,
+    cache: CatalogProcessorCache,
+  ): Promise<boolean> {
+    // Pick a custom location type string. A location will be
+    // registered later with this type.
+    if (location.type !== 'system-x') {
+      return false;
+    }
+    const cacheItem = await cache.get<CacheItem>(CACHE_KEY);
+    try {
+      // This assumes an URL reader that returns the response together with the ETag.
+      // We send the ETag from the previous run if it exists.
+      // The previous ETag will be set in the headers for the outgoing request and system-x
+      // is going to throw NOT_MODIFIED (HTTP 304) if the ETag matches.
+      const response = await this.reader.readUrl?.(location.target, {
+        etag: cacheItem?.etag,
+      });
+      if (!response) {
+        // readUrl is currently optional to implement so we have to check if we get a response back.
+        throw new Error(
+          'No URL reader that can parse system-x targets installed',
+        );
+      }
+
+      // For this example the JSON payload is a single entity.
+      const entity: Entity = JSON.parse(response.buffer.toString());
+      emit(results.entity(location, entity));
+
+      // Update the cache with the new ETag and entity used for the next run.
+      await cache.set<CacheItem>(CACHE_KEY, {
+        etag: response.etag ? response.etag : '',
+        entity,
+      });
+    } catch (error) {
+      if (error.name === 'NotModifiedError' && cacheItem) {
+        // The ETag matches and we have a cached value from the previous run.
+        emit(results.entity(location, cacheItem.entity));
+      }
+      const message = `Unable to read ${location.type}, ${error}`;
+      emit(results.generalError(location, message));
+    }
+
+    return true;
+  }
+}
+```