paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 40 Packages Projects Releases Wiki Activity

Merge pull request #29162 from marknach/cache-options

Browse Source 
implement more comprehensive redis option passthrough
This commit is contained in:
Fredrik Adelöw
2025-03-19 14:20:13 +01:00
committed by GitHub
parent 304a422ef4 893e8730b7
commit da4b41bd6a
5 changed files with 306 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/clever-tomatoes-jump.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/backend-defaults': minor
---
Allow pass through of redis client and cluster options to Cache core service
packages/backend-defaults/config.d.ts
Vendored
+61
View File
@@ -559,6 +559,67 @@ export interface Config {
connection: string;
/** An optional default TTL (in milliseconds, if given as a number). */
defaultTtl?: number | HumanDuration | string;
redis?: {
/**
* An optional Redis client configuration. These options are passed to the `@keyv/redis` client.
*/
client?: {
/**
* Namespace for the current instance.
*/
namespace?: string;
/**
* Separator to use between namespace and key.
*/
keyPrefixSeparator?: string;
/**
* Number of keys to delete in a single batch.
*/
clearBatchSize?: number;
/**
* Enable Unlink instead of using Del for clearing keys. This is more performant but may not be supported by all Redis versions.
*/
useUnlink?: boolean;
/**
* Whether to allow clearing all keys when no namespace is set.
* If set to true and no namespace is set, iterate() will return all keys.
* Defaults to `false`.
*/
noNamespaceAffectsAll?: boolean;
};
/**
* An optional Redis cluster configuration.
*/
cluster?: {
/**
* Cluster configuration options to be passed to the `@keyv/redis` client (and node-redis under the hood)
* https://github.com/redis/node-redis/blob/master/docs/clustering.md
*
* @visibility secret
*/
rootNodes: Array<object>;
/**
* Cluster node default configuration options to be passed to the `@keyv/redis` client (and node-redis under the hood)
* https://github.com/redis/node-redis/blob/master/docs/clustering.md
*
* @visibility secret
*/
defaults?: Partial<object>;
/**
* When `true`, `.connect()` will only discover the cluster topology, without actually connecting to all the nodes.
* Useful for short-term or PubSub-only connections.
*/
minimizeConnections?: boolean;
/**
* When `true`, distribute load by executing readonly commands (such as `GET`, `GEOSEARCH`, etc.) across all cluster nodes. When `false`, only use master nodes.
*/
useReplicas?: boolean;
/**
* The maximum number of times a command will be redirected due to `MOVED` or `ASK` errors.
*/
maxCommandRedirections?: number;
};
};
}
| {
store: 'memcache';
packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts
Vendored
+127 -1
View File
@@ -15,7 +15,7 @@
*/
import { mockServices, TestCaches } from '@backstage/backend-test-utils';
import KeyvRedis from '@keyv/redis';
import KeyvRedis, { createCluster } from '@keyv/redis';
import KeyvMemcache from '@keyv/memcache';
import { CacheManager } from './CacheManager';
@@ -30,6 +30,7 @@ jest.mock('@keyv/redis', () => {
...Actual,
__esModule: true,
default: jest.fn((...args: any[]) => new DefaultConstructor(...args)),
createCluster: jest.fn(),
};
});
jest.mock('@keyv/memcache', () => {
@@ -194,3 +195,128 @@ describe('CacheManager integration', () => {
).toThrow(/Invalid duration 'hello' in config/);
});
});
describe('CacheManager store options', () => {
it('uses default options when no store-specific config exists', () => {
const manager = CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'redis',
connection: 'redis://localhost:6379',
},
},
},
}),
);
manager.forPlugin('p1');
expect(KeyvRedis).toHaveBeenCalledWith('redis://localhost:6379', {
keyPrefixSeparator: ':',
});
});
it('defaults to non-clustered mode when cluster config is missing root nodes', () => {
const manager = CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'redis',
connection: 'redis://localhost:6379',
redis: {
cluster: {},
},
},
},
},
}),
);
manager.forPlugin('p1');
expect(KeyvRedis).toHaveBeenCalledWith('redis://localhost:6379', {
keyPrefixSeparator: ':',
});
});
it('uses cluster config when present', () => {
const manager = CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'redis',
connection: 'redis://localhost:6379',
redis: {
cluster: {
rootNodes: [{ url: 'redis://localhost:6379' }],
},
},
},
},
},
}),
);
manager.forPlugin('p1');
expect(createCluster).toHaveBeenCalledWith({
rootNodes: [{ url: 'redis://localhost:6379' }],
defaults: undefined,
});
});
it('respects client config for non-clustered mode', () => {
const manager = CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'redis',
connection: 'redis://localhost:6379',
redis: {
client: {
keyPrefixSeparator: '!',
},
},
},
},
},
}),
);
manager.forPlugin('p1');
expect(KeyvRedis).toHaveBeenCalledWith('redis://localhost:6379', {
keyPrefixSeparator: '!',
});
});
it('accepts client config for clustered mode', () => {
const manager = CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'redis',
connection: 'redis://localhost:6379',
redis: {
client: {
keyPrefixSeparator: '!',
},
cluster: {
rootNodes: [{ url: 'redis://localhost:6379' }],
},
},
},
},
},
}),
);
manager.forPlugin('p1');
expect(KeyvRedis).toHaveBeenCalledWith(expect.anything(), {
keyPrefixSeparator: '!',
});
});
});
packages/backend-defaults/src/entrypoints/cache/CacheManager.ts
Vendored
+95 -3
View File
@@ -22,7 +22,12 @@ import {
} from '@backstage/backend-plugin-api';
import Keyv from 'keyv';
import { DefaultCacheClient } from './CacheClient';
import { CacheManagerOptions, ttlToMilliseconds } from './types';
import {
CacheManagerOptions,
ttlToMilliseconds,
CacheStoreOptions,
RedisCacheStoreOptions,
} from './types';
import { durationToMilliseconds } from '@backstage/types';
import { readDurationFromConfig } from '@backstage/config';
@@ -51,6 +56,7 @@ export class CacheManager {
private readonly connection: string;
private readonly errorHandler: CacheManagerOptions['onError'];
private readonly defaultTtl?: number;
private readonly storeOptions?: CacheStoreOptions;
/**
* Creates a new {@link CacheManager} instance by reading from the `backend`
@@ -89,15 +95,89 @@ export class CacheManager {
}
}
// Read store-specific options from config
const storeOptions = CacheManager.parseStoreOptions(store, config, logger);
return new CacheManager(
store,
connectionString,
options.onError,
logger,
defaultTtl,
storeOptions,
);
}
/**
* Parse store-specific options from configuration.
*
* @param store - The cache store type ('redis', 'memcache', or 'memory')
* @param config - The configuration service
* @param logger - Optional logger for warnings
* @returns The parsed store options
*/
private static parseStoreOptions(
store: string,
config: RootConfigService,
logger?: LoggerService,
): CacheStoreOptions | undefined {
const storeConfigPath = `backend.cache.${store}`;
if (store === 'redis' && config.has(storeConfigPath)) {
return CacheManager.parseRedisOptions(storeConfigPath, config, logger);
}
return undefined;
}
/**
* Parse Redis-specific options from configuration.
*/
private static parseRedisOptions(
storeConfigPath: string,
config: RootConfigService,
logger?: LoggerService,
): RedisCacheStoreOptions {
const redisOptions: RedisCacheStoreOptions = {};
const redisConfig = config.getConfig(storeConfigPath);
redisOptions.client = {
namespace: redisConfig.getOptionalString('client.namespace'),
keyPrefixSeparator:
redisConfig.getOptionalString('client.keyPrefixSeparator') || ':',
clearBatchSize: redisConfig.getOptionalNumber('client.clearBatchSize'),
useUnlink: redisConfig.getOptionalBoolean('client.useUnlink'),
noNamespaceAffectsAll: redisConfig.getOptionalBoolean(
'client.noNamespaceAffectsAll',
),
};
if (redisConfig.has('cluster')) {
const clusterConfig = redisConfig.getConfig('cluster');
if (!clusterConfig.has('rootNodes')) {
logger?.warn(
`Redis cluster config has no 'rootNodes' key, defaulting to non-clustered mode`,
);
return redisOptions;
}
redisOptions.cluster = {
rootNodes: clusterConfig.get('rootNodes'),
defaults: clusterConfig.getOptional('defaults'),
minimizeConnections: clusterConfig.getOptionalBoolean(
'minimizeConnections',
),
useReplicas: clusterConfig.getOptionalBoolean('useReplicas'),
maxCommandRedirections: clusterConfig.getOptionalNumber(
'maxCommandRedirections',
),
};
}
return redisOptions;
}
/** @internal */
constructor(
store: string,
@@ -105,6 +185,7 @@ export class CacheManager {
errorHandler: CacheManagerOptions['onError'],
logger?: LoggerService,
defaultTtl?: number,
storeOptions?: CacheStoreOptions,
) {
if (!this.storeFactories.hasOwnProperty(store)) {
throw new Error(`Unknown cache store: ${store}`);
@@ -114,6 +195,7 @@ export class CacheManager {
this.connection = connectionString;
this.errorHandler = errorHandler;
this.defaultTtl = defaultTtl;
this.storeOptions = storeOptions;
}
/**
@@ -140,13 +222,23 @@ export class CacheManager {
private createRedisStoreFactory(): StoreFactory {
const KeyvRedis = require('@keyv/redis').default;
const { createCluster } = require('@keyv/redis');
const stores: Record<string, typeof KeyvRedis> = {};
return (pluginId, defaultTtl) => {
if (!stores[pluginId]) {
stores[pluginId] = new KeyvRedis(this.connection, {
const redisOptions = this.storeOptions?.client || {
keyPrefixSeparator: ':',
});
};
if (this.storeOptions?.cluster) {
// Create a Redis cluster
const cluster = createCluster(this.storeOptions?.cluster);
stores[pluginId] = new KeyvRedis(cluster, redisOptions);
} else {
// Create a regular Redis connection
stores[pluginId] = new KeyvRedis(this.connection, redisOptions);
}
// Always provide an error handler to avoid stopping the process
stores[pluginId].on('error', (err: Error) => {
this.logger?.error('Failed to create redis cache client', err);
packages/backend-defaults/src/entrypoints/cache/types.ts
Vendored
+18
View File
@@ -16,6 +16,24 @@
import { LoggerService } from '@backstage/backend-plugin-api';
import { HumanDuration, durationToMilliseconds } from '@backstage/types';
import { RedisClusterOptions, KeyvRedisOptions } from '@keyv/redis';
/**
* Options for Redis cache store.
*
* @public
*/
export type RedisCacheStoreOptions = {
client?: KeyvRedisOptions;
cluster?: RedisClusterOptions;
};
/**
* Union type of all cache store options.
*
* @public
*/
export type CacheStoreOptions = RedisCacheStoreOptions;
/**
* Options given when constructing a {@link CacheManager}.