From 47dece5e3b783691d37b2723fe6c4f2c84613338 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 24 Sep 2021 12:04:27 +0200 Subject: [PATCH 1/9] Add docs for implementing cachable processing Signed-off-by: Johan Haals --- .../software-catalog/external-integrations.md | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index 247324a46c..b4e382fb37 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -171,3 +171,97 @@ export default async function createPlugin( Start up the backend - it should now start reading from the previously registered location and you'll see your entities start to appear in Backstage. + +## Caching processing results + +The catalog periodically refresh entities in the catalog and by doing so it call +out to external systems to fetch changes. This can be taxing for upstream +services and large deployments may get rate limited if too many requests are +sent. Luckily many external systems provide ETag support to check for changes +which usually don't count towards the quota and saves resources both internally +and externally. + +The catalog has built in support for caching when refreshing external locations +in GitHub. This example aims to demonstrate the same behavior for `system-x` +that we implemented earlier. + +```ts +import { UrlReader } from '@backstage/backend-common'; +import { Entity, LocationSpec } from '@backstage/catalog-model'; +import { + results, + CatalogProcessor, + CatalogProcessorEmit, + CatalogProcessorCache, +} from '@backstage/plugin-catalog-backend'; +import { CatalogProcessorParser } from '.'; + +// Change this key to if the processor has undergone major changes +// and the existing cache data is no longer compatible. +const CACHE_KEY = 'v1'; + +// CacheItem is our cache containing ETag used in the upstream request +// as well as the processing result used when the Etag matches. +type CacheItem = { + etag: string; + entity: Entity; +}; + +export class SystemXReaderProcessor implements CatalogProcessor { + constructor(private readonly reader: UrlReader) {} + + // It's recommended to give the processor a unique name. + getProcessorName() { + return 'system-x-processor'; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + _parser: CatalogProcessorParser, + cache: CatalogProcessorCache, + ): Promise { + // Pick a custom location type string. A location will be + // registered later with this type. + if (location.type !== 'system-x') { + return false; + } + const cacheItem = await cache.get(CACHE_KEY); + try { + // This assumes an URL reader that returns the response together with the ETag. + // We send the ETag from the previous run if it exists. + // The previous ETag will be set in the headers for the outgoing request and system-x + // is going to throw NOT_MODIFIED (HTTP 304) if the ETag matches. + const response = await this.reader.readUrl?.(location.target, { + etag: cacheItem?.etag, + }); + if (!response) { + // readUrl is currently optional to implement so we have to check if we get a response back. + throw new Error( + 'No URL reader that can parse system-x targets installed', + ); + } + + // For this example the JSON payload is a single entity. + const entity: Entity = JSON.parse(response.buffer.toString()); + emit(results.entity(location, entity)); + + // Update the cache with the new ETag and entity used for the next run. + await cache.set(CACHE_KEY, { + etag: response.etag ? response.etag : '', + entity, + }); + } catch (error) { + if (error.name === 'NotModifiedError' && cacheItem) { + // The ETag matches and we have a cached value from the previous run. + emit(results.entity(location, cacheItem.entity)); + } + const message = `Unable to read ${location.type}, ${error}`; + emit(results.generalError(location, message)); + } + + return true; + } +} +``` From f775ce3264639a675b843ef27e60b7c38be02e64 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 24 Sep 2021 13:54:40 +0200 Subject: [PATCH 2/9] clarify when cache key should be bumped Signed-off-by: Johan Haals --- docs/features/software-catalog/external-integrations.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index b4e382fb37..8a4eb01140 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -196,12 +196,13 @@ import { } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorParser } from '.'; -// Change this key to if the processor has undergone major changes -// and the existing cache data is no longer compatible. +// It's recommended to always bump the CACHE_KEY version if you make +// changes to the processor implementation or CacheItem. const CACHE_KEY = 'v1'; // CacheItem is our cache containing ETag used in the upstream request // as well as the processing result used when the Etag matches. +// Bump the CACHE_KEY version if you make any changes to this type. type CacheItem = { etag: string; entity: Entity; From 2588a65865eb85424b846769cea544f8ad7524c6 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 24 Sep 2021 14:43:59 +0200 Subject: [PATCH 3/9] Make etag optional Signed-off-by: Johan Haals --- docs/features/software-catalog/external-integrations.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index 8a4eb01140..3187efc551 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -204,7 +204,7 @@ const CACHE_KEY = 'v1'; // as well as the processing result used when the Etag matches. // Bump the CACHE_KEY version if you make any changes to this type. type CacheItem = { - etag: string; + etag?: string; entity: Entity; }; @@ -250,7 +250,7 @@ export class SystemXReaderProcessor implements CatalogProcessor { // Update the cache with the new ETag and entity used for the next run. await cache.set(CACHE_KEY, { - etag: response.etag ? response.etag : '', + etag: response.etag, entity, }); } catch (error) { From 9bd7ecd4fa15079e70f4c46dc5caa65a113d4690 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 29 Sep 2021 10:26:44 +0200 Subject: [PATCH 4/9] Update docs/features/software-catalog/external-integrations.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw --- docs/features/software-catalog/external-integrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index 3187efc551..da480be0a5 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -174,7 +174,7 @@ registered location and you'll see your entities start to appear in Backstage. ## Caching processing results -The catalog periodically refresh entities in the catalog and by doing so it call +The catalog periodically refreshes entities in the catalog, and in doing so it calls out to external systems to fetch changes. This can be taxing for upstream services and large deployments may get rate limited if too many requests are sent. Luckily many external systems provide ETag support to check for changes From 8621a63eb6adc1d67fbd55d9d118ad9c855234bd Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 29 Sep 2021 10:26:56 +0200 Subject: [PATCH 5/9] Update docs/features/software-catalog/external-integrations.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw --- docs/features/software-catalog/external-integrations.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index da480be0a5..2740d7ad96 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -181,8 +181,8 @@ sent. Luckily many external systems provide ETag support to check for changes which usually don't count towards the quota and saves resources both internally and externally. -The catalog has built in support for caching when refreshing external locations -in GitHub. This example aims to demonstrate the same behavior for `system-x` +The catalog has built in support for leveraging ETags when refreshing external locations +in GitHub. This example aims to demonstrate how to add the same behavior for `system-x` that we implemented earlier. ```ts From 95f324cb7fff18ce4d0f0b512107aaba50e603f9 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 29 Sep 2021 10:27:04 +0200 Subject: [PATCH 6/9] Update docs/features/software-catalog/external-integrations.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw --- docs/features/software-catalog/external-integrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index 2740d7ad96..c28d7c4f9c 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -200,7 +200,7 @@ import { CatalogProcessorParser } from '.'; // changes to the processor implementation or CacheItem. const CACHE_KEY = 'v1'; -// CacheItem is our cache containing ETag used in the upstream request +// Our cache item contains the ETag used in the upstream request // as well as the processing result used when the Etag matches. // Bump the CACHE_KEY version if you make any changes to this type. type CacheItem = { From 58dba06461ab04d789b1cebcdf2c9b35e233a1be Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 29 Sep 2021 10:27:32 +0200 Subject: [PATCH 7/9] Update docs/features/software-catalog/external-integrations.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw --- docs/features/software-catalog/external-integrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index c28d7c4f9c..1015af2630 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -194,7 +194,7 @@ import { CatalogProcessorEmit, CatalogProcessorCache, } from '@backstage/plugin-catalog-backend'; -import { CatalogProcessorParser } from '.'; +import { CatalogProcessorParser } from './CatalogProcessorParser'; // It's recommended to always bump the CACHE_KEY version if you make // changes to the processor implementation or CacheItem. From db920df0533b303635009405c481df2ca7a1e151 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 29 Sep 2021 10:27:50 +0200 Subject: [PATCH 8/9] Update docs/features/software-catalog/external-integrations.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw --- docs/features/software-catalog/external-integrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index 1015af2630..508b04d77a 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -178,7 +178,7 @@ The catalog periodically refreshes entities in the catalog, and in doing so it c out to external systems to fetch changes. This can be taxing for upstream services and large deployments may get rate limited if too many requests are sent. Luckily many external systems provide ETag support to check for changes -which usually don't count towards the quota and saves resources both internally +which usually doesn't count towards the quota and saves resources both internally and externally. The catalog has built in support for leveraging ETags when refreshing external locations From 9c8142cdacf38c8f4e41378897cf8bf7171fed7c Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 29 Sep 2021 10:56:26 +0200 Subject: [PATCH 9/9] Format, require ETag, fix imports Signed-off-by: Johan Haals --- .../software-catalog/external-integrations.md | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index 508b04d77a..4d15371a6b 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -174,16 +174,16 @@ registered location and you'll see your entities start to appear in Backstage. ## Caching processing results -The catalog periodically refreshes entities in the catalog, and in doing so it calls -out to external systems to fetch changes. This can be taxing for upstream +The catalog periodically refreshes entities in the catalog, and in doing so it +calls out to external systems to fetch changes. This can be taxing for upstream services and large deployments may get rate limited if too many requests are sent. Luckily many external systems provide ETag support to check for changes -which usually doesn't count towards the quota and saves resources both internally -and externally. +which usually doesn't count towards the quota and saves resources both +internally and externally. -The catalog has built in support for leveraging ETags when refreshing external locations -in GitHub. This example aims to demonstrate how to add the same behavior for `system-x` -that we implemented earlier. +The catalog has built in support for leveraging ETags when refreshing external +locations in GitHub. This example aims to demonstrate how to add the same +behavior for `system-x` that we implemented earlier. ```ts import { UrlReader } from '@backstage/backend-common'; @@ -193,8 +193,8 @@ import { CatalogProcessor, CatalogProcessorEmit, CatalogProcessorCache, + CatalogProcessorParser, } from '@backstage/plugin-catalog-backend'; -import { CatalogProcessorParser } from './CatalogProcessorParser'; // It's recommended to always bump the CACHE_KEY version if you make // changes to the processor implementation or CacheItem. @@ -204,7 +204,7 @@ const CACHE_KEY = 'v1'; // as well as the processing result used when the Etag matches. // Bump the CACHE_KEY version if you make any changes to this type. type CacheItem = { - etag?: string; + etag: string; entity: Entity; }; @@ -244,6 +244,13 @@ export class SystemXReaderProcessor implements CatalogProcessor { ); } + // ETag is optional in the response but we need it to cache the result. + if (!response.etag) { + throw new Error( + 'No ETag returned from system-x, cannot use response for caching', + ); + } + // For this example the JSON payload is a single entity. const entity: Entity = JSON.parse(response.buffer.toString()); emit(results.entity(location, entity));