From 7d12e4cf32e828c01b1fae098de75ebc25b0244b Mon Sep 17 00:00:00 2001 From: Magnus Persson Date: Mon, 21 Feb 2022 21:37:12 +0100 Subject: [PATCH] backend-common: add Redis backed cache store Signed-off-by: Magnus Persson --- .changeset/thick-guests-greet.md | 5 ++ docs/overview/architecture-overview.md | 19 ++++-- packages/backend-common/config.d.ts | 8 +++ packages/backend-common/package.json | 1 + .../src/cache/CacheManager.test.ts | 31 +++++++++ .../backend-common/src/cache/CacheManager.ts | 14 +++++ yarn.lock | 63 ++++++++++++++++++- 7 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 .changeset/thick-guests-greet.md diff --git a/.changeset/thick-guests-greet.md b/.changeset/thick-guests-greet.md new file mode 100644 index 0000000000..6efff5fc87 --- /dev/null +++ b/.changeset/thick-guests-greet.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': minor +--- + +feat(backend-common): add Redis backed cache store diff --git a/docs/overview/architecture-overview.md b/docs/overview/architecture-overview.md index 5d27cc4184..358dc53852 100644 --- a/docs/overview/architecture-overview.md +++ b/docs/overview/architecture-overview.md @@ -281,11 +281,11 @@ stores as a means of improving performance or reliability. Similar to how databases are supported, plugins receive logically separated cache connections, which are powered by [Keyv](https://github.com/lukechilds/keyv) under the hood. -At this time of writing, Backstage can be configured to use one of two cache -stores: memory, which is mainly used for local testing, and memcache, which is a -cache store better suited for production deployment. The right cache store for -your Backstage instance will depend on your own run-time constraints and those -required of the plugins you're running. +At this time of writing, Backstage can be configured to use one of three cache +stores: memory, which is mainly used for local testing, memcache or Redis, +which are cache stores better suited for production deployment. The right cache +store for your Backstage instance will depend on your own run-time constraints +and those required of the plugins you're running. ### Use memory for cache @@ -304,6 +304,15 @@ backend: connection: user:pass@cache.example.com:11211 ``` +### Use Redis for cache + +```yaml +backend: + cache: + store: redis + connection: user:pass@cache.example.com:6379 +``` + Contributions supporting other cache stores are welcome! ## Containerization diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 16ca1698f6..d658aaeb00 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -134,6 +134,14 @@ export interface Config { | { store: 'memory'; } + | { + store: 'redis'; + /** + * A redis connection string in the form `user:pass@host:port`. + * @secret + */ + connection: string; + } | { store: 'memcache'; /** diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 229134dffd..57016228f8 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -62,6 +62,7 @@ "jose": "^1.27.1", "keyv": "^4.0.3", "keyv-memcache": "^1.2.5", + "@keyv/redis": "^2.2.3", "knex": "^1.0.2", "lodash": "^4.17.21", "logform": "^2.3.2", diff --git a/packages/backend-common/src/cache/CacheManager.test.ts b/packages/backend-common/src/cache/CacheManager.test.ts index ce5b6f5999..660aa82bb0 100644 --- a/packages/backend-common/src/cache/CacheManager.test.ts +++ b/packages/backend-common/src/cache/CacheManager.test.ts @@ -18,6 +18,8 @@ import { ConfigReader } from '@backstage/config'; import Keyv from 'keyv'; /* @ts-expect-error */ import KeyvMemcache from 'keyv-memcache'; +/* @ts-expect-error */ +import KeyvRedis from '@keyv/redis'; import { DefaultCacheClient } from './CacheClient'; import { CacheManager } from './CacheManager'; import { NoStore } from './NoStore'; @@ -26,6 +28,8 @@ jest.createMockFromModule('keyv'); jest.mock('keyv'); jest.createMockFromModule('keyv-memcache'); jest.mock('keyv-memcache'); +jest.createMockFromModule('@keyv/redis'); +jest.mock('@keyv/redis'); jest.mock('./CacheClient', () => { return { DefaultCacheClient: jest.fn(), @@ -190,6 +194,33 @@ describe('CacheManager', () => { }); }); + it('returns a Redis client when configured', () => { + const redisHostAndPort = '127.0.0.1:6379'; + const expectedHost = `redis://${redisHostAndPort}`; + const manager = CacheManager.fromConfig( + new ConfigReader({ + backend: { + cache: { + store: 'redis', + connection: redisHostAndPort, + }, + }, + }), + ); + const expectedTtl = 3600; + manager.forPlugin('test').getClient({ defaultTtl: expectedTtl }); + + const cache = Keyv as unknown as jest.Mock; + const mockCacheCalls = cache.mock.calls.splice(-1); + expect(mockCacheCalls[0][0]).toMatchObject({ + ttl: expectedTtl, + }); + expect(mockCacheCalls[0][0].store).toBeInstanceOf(KeyvRedis); + const redis = KeyvRedis as jest.Mock; + const mockRedisCalls = redis.mock.calls.splice(-1); + expect(mockRedisCalls[0][0]).toEqual(expectedHost); + }); + describe('connection errors', () => { it('uses provided logger', () => { // Set up and inject mock logger. diff --git a/packages/backend-common/src/cache/CacheManager.ts b/packages/backend-common/src/cache/CacheManager.ts index 66e9849e77..d63ac6d940 100644 --- a/packages/backend-common/src/cache/CacheManager.ts +++ b/packages/backend-common/src/cache/CacheManager.ts @@ -18,6 +18,8 @@ import { Config } from '@backstage/config'; import Keyv from 'keyv'; // @ts-expect-error import KeyvMemcache from 'keyv-memcache'; +// @ts-expect-error +import KeyvRedis from '@keyv/redis'; import { Logger } from 'winston'; import { getRootLogger } from '../logging'; import { DefaultCacheClient, CacheClient } from './CacheClient'; @@ -37,6 +39,7 @@ export class CacheManager { * that return Keyv instances appropriate to the store. */ private readonly storeFactories = { + redis: this.getRedisClient, memcache: this.getMemcacheClient, memory: this.getMemoryClient, none: this.getNoneClient, @@ -123,6 +126,17 @@ export class CacheManager { return this.storeFactories[this.store].call(this, pluginId, ttl); } + private getRedisClient( + pluginId: string, + defaultTtl: number | undefined, + ): Keyv { + return new Keyv({ + namespace: pluginId, + ttl: defaultTtl, + store: new KeyvRedis(`redis://${this.connection}`), + }); + } + private getMemcacheClient( pluginId: string, defaultTtl: number | undefined, diff --git a/yarn.lock b/yarn.lock index bbd75035ce..0354fd8f11 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3075,6 +3075,13 @@ resolved "https://registry.npmjs.org/@juggle/resize-observer/-/resize-observer-3.3.1.tgz#b50a781709c81e10701004214340f25475a171a0" integrity sha512-zMM9Ds+SawiUkakS7y94Ymqx+S0ORzpG3frZirN3l+UlXUmSUR7hF4wxCVqW+ei94JzV5kt0uXBcoOEAuiydrw== +"@keyv/redis@^2.2.3": + version "2.2.3" + resolved "https://registry.npmjs.org/@keyv/redis/-/redis-2.2.3.tgz#af5b1ea32d847a63ce24012844af7323b3c421a7" + integrity sha512-d9Maf1LzT6Ti5hWsVzaWFriFmXrscK1eUl/etNquQgAJxH7Drecbn+uNZXMc6xb78Ju9szy0fD9RAp/G9RzAdg== + dependencies: + ioredis "^4.28.5" + "@kubernetes/client-node@^0.16.0": version "0.16.1" resolved "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.16.1.tgz#c78ef667579777c1a532983922807e228dbc9b90" @@ -8986,6 +8993,11 @@ clsx@^1.0.2, clsx@^1.0.4: resolved "https://registry.npmjs.org/clsx/-/clsx-1.1.1.tgz#98b3134f9abbdf23b2663491ace13c5c03a73188" integrity sha512-6/bPho624p3S2pMyvP5kKBPXnI3ufHLObBFCfgx+LkeR5lg2XYy2hqZqUf45ypD8COn2bhgGJSUE+l5dhNBieA== +cluster-key-slot@^1.1.0: + version "1.1.0" + resolved "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" + integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== + cmd-shim@^4.0.1, cmd-shim@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd" @@ -10617,6 +10629,11 @@ delegates@^1.0.0: resolved "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= +denque@^1.1.0: + version "1.5.1" + resolved "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf" + integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== + denque@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/denque/-/denque-2.0.1.tgz#bcef4c1b80dc32efe97515744f21a4229ab8934a" @@ -14294,6 +14311,23 @@ invert-kv@^1.0.0: resolved "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" integrity sha1-EEqOSqym09jNFXqO+L+rLXo//bY= +ioredis@^4.28.5: + version "4.28.5" + resolved "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f" + integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A== + dependencies: + cluster-key-slot "^1.1.0" + debug "^4.3.1" + denque "^1.1.0" + lodash.defaults "^4.2.0" + lodash.flatten "^4.4.0" + lodash.isarguments "^3.1.0" + p-map "^2.1.0" + redis-commands "1.7.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ip-regex@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" @@ -16605,6 +16639,11 @@ lodash.includes@^4.3.0: resolved "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f" integrity sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8= +lodash.isarguments@^3.1.0: + version "3.1.0" + resolved "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" + integrity sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo= + lodash.isboolean@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6" @@ -19010,7 +19049,7 @@ p-map-series@^2.1.0: resolved "https://registry.npmjs.org/p-map-series/-/p-map-series-2.1.0.tgz#7560d4c452d9da0c07e692fdbfe6e2c81a2a91f2" integrity sha512-RpYIIK1zXSNEOdwxcfe7FdvGcs7+y5n8rifMhMNWvaxRNMPINJHF5GDeuVxWqnfrcHPSCnp7Oo5yNXHId9Av2Q== -p-map@^2.0.0: +p-map@^2.0.0, p-map@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== @@ -21286,6 +21325,23 @@ redent@^3.0.0: indent-string "^4.0.0" strip-indent "^3.0.0" +redis-commands@1.7.0: + version "1.7.0" + resolved "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89" + integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ== + +redis-errors@^1.0.0, redis-errors@^1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad" + integrity sha1-62LSrbFeTq9GEMBK/hUpOEJQq60= + +redis-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4" + integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ= + dependencies: + redis-errors "^1.0.0" + reduce-css-calc@^1.3.0: version "1.3.0" resolved "https://registry.npmjs.org/reduce-css-calc/-/reduce-css-calc-1.3.0.tgz#747c914e049614a4c9cfbba629871ad1d2927716" @@ -22924,6 +22980,11 @@ stacktrace-js@^2.0.2: stack-generator "^2.0.5" stacktrace-gps "^3.0.4" +standard-as-callback@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45" + integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== + start-server-and-test@^1.10.11: version "1.14.0" resolved "https://registry.npmjs.org/start-server-and-test/-/start-server-and-test-1.14.0.tgz#c57f04f73eac15dd51733b551d775b40837fdde3"