feat(search-elasticsearch): add authentication extension point for dynamic auth in ElasticSearch

Signed-off-by: nojaf <florian.verdonck@outlook.com>
This commit is contained in:
nojaf
2026-01-19 17:22:44 +01:00
parent cfac8a460d
commit d0c59a218b
7 changed files with 641 additions and 12 deletions
@@ -0,0 +1,117 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { createExtensionPoint } from '@backstage/backend-plugin-api';
/**
* A provider that supplies authentication headers for Elasticsearch/OpenSearch requests.
*
* @remarks
*
* This interface allows for dynamic authentication mechanisms such as bearer tokens
* that need to be refreshed or rotated. The `getAuthHeaders` method is called before
* each request to the Elasticsearch/OpenSearch cluster, allowing for just-in-time
* token retrieval and automatic rotation.
*
* @example
*
* ```ts
* const authProvider: ElasticSearchAuthProvider = {
* async getAuthHeaders() {
* const token = await myTokenService.getToken();
* return { Authorization: `Bearer ${token}` };
* },
* };
* ```
*
* @public
*/
export interface ElasticSearchAuthProvider {
/**
* Returns authentication headers to be included in requests to Elasticsearch/OpenSearch.
*
* @remarks
*
* This method is called before each request, allowing for dynamic token refresh
* and rotation. Implementations should handle caching internally if needed to
* avoid excessive token generation.
*
* @returns A promise that resolves to a record of header names and values
*/
getAuthHeaders(): Promise<Record<string, string>>;
}
/**
* Extension point for providing custom authentication to the Elasticsearch search engine.
*
* @remarks
*
* Use this extension point to provide dynamic authentication mechanisms such as
* bearer tokens with automatic rotation. When an auth provider is set, it takes
* precedence over any static authentication configured in app-config.yaml.
*
* @example
*
* ```ts
* import { createBackendModule } from '@backstage/backend-plugin-api';
* import { elasticsearchAuthExtensionPoint } from '@backstage/plugin-search-backend-module-elasticsearch';
*
* export default createBackendModule({
* pluginId: 'search',
* moduleId: 'elasticsearch-custom-auth',
* register(env) {
* env.registerInit({
* deps: {
* elasticsearchAuth: elasticsearchAuthExtensionPoint,
* },
* async init({ elasticsearchAuth }) {
* elasticsearchAuth.setAuthProvider({
* async getAuthHeaders() {
* const token = await fetchTokenFromIdentityService();
* return { Authorization: `Bearer ${token}` };
* },
* });
* },
* });
* },
* });
* ```
*
* @public
*/
export interface ElasticSearchAuthExtensionPoint {
/**
* Sets the authentication provider for the Elasticsearch search engine.
*
* @remarks
*
* This method can only be called once. Subsequent calls will throw an error.
* The auth provider takes precedence over static authentication configuration.
*
* @param provider - The authentication provider to use
*/
setAuthProvider(provider: ElasticSearchAuthProvider): void;
}
/**
* Extension point used to customize Elasticsearch/OpenSearch authentication.
*
* @public
*/
export const elasticsearchAuthExtensionPoint =
createExtensionPoint<ElasticSearchAuthExtensionPoint>({
id: 'search.elasticsearchEngine.auth',
});
@@ -0,0 +1,194 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
createOpenSearchAuthTransport,
createElasticSearchAuthTransport,
} from './ElasticSearchAuthTransport';
import { ElasticSearchAuthProvider } from '../auth';
describe('ElasticSearchAuthTransport', () => {
describe('createOpenSearchAuthTransport', () => {
it('should return a Transport class', () => {
const authProvider: ElasticSearchAuthProvider = {
getAuthHeaders: jest
.fn()
.mockResolvedValue({ Authorization: 'Bearer test' }),
};
const Transport = createOpenSearchAuthTransport(authProvider);
expect(Transport).toBeDefined();
expect(typeof Transport).toBe('function');
});
it('should call getAuthHeaders when request is made', async () => {
const authProvider: ElasticSearchAuthProvider = {
getAuthHeaders: jest
.fn()
.mockResolvedValue({ Authorization: 'Bearer test-token' }),
};
const Transport = createOpenSearchAuthTransport(authProvider);
// Create a mock parent class request method
const mockParentRequest = jest.fn().mockResolvedValue({ body: {} });
// Create an instance with mocked internals
const transportInstance = Object.create(Transport.prototype);
Object.getPrototypeOf(Object.getPrototypeOf(transportInstance)).request =
mockParentRequest;
// Call request method (promise style)
const requestPromise = transportInstance.request(
{ method: 'GET', path: '/' },
{},
);
// The auth provider should be called
await requestPromise.catch(() => {}); // Ignore errors from incomplete mock
expect(authProvider.getAuthHeaders).toHaveBeenCalled();
});
it('should be usable as Transport option', () => {
const authProvider: ElasticSearchAuthProvider = {
getAuthHeaders: jest
.fn()
.mockResolvedValue({ Authorization: 'Bearer test' }),
};
const Transport = createOpenSearchAuthTransport(authProvider);
// Verify the Transport has the expected static properties
expect(Transport).toHaveProperty('sniffReasons');
});
});
describe('createElasticSearchAuthTransport', () => {
it('should return a Transport class', () => {
const authProvider: ElasticSearchAuthProvider = {
getAuthHeaders: jest
.fn()
.mockResolvedValue({ Authorization: 'Bearer test' }),
};
const Transport = createElasticSearchAuthTransport(authProvider);
expect(Transport).toBeDefined();
expect(typeof Transport).toBe('function');
});
it('should call getAuthHeaders when request is made', async () => {
const authProvider: ElasticSearchAuthProvider = {
getAuthHeaders: jest
.fn()
.mockResolvedValue({ Authorization: 'Bearer test-token' }),
};
const Transport = createElasticSearchAuthTransport(authProvider);
// Create a mock parent class request method
const mockParentRequest = jest.fn().mockResolvedValue({ body: {} });
// Create an instance with mocked internals
const transportInstance = Object.create(Transport.prototype);
Object.getPrototypeOf(Object.getPrototypeOf(transportInstance)).request =
mockParentRequest;
// Call request method (promise style)
const requestPromise = transportInstance.request(
{ method: 'GET', path: '/' },
{},
);
// The auth provider should be called
await requestPromise.catch(() => {}); // Ignore errors from incomplete mock
expect(authProvider.getAuthHeaders).toHaveBeenCalled();
});
it('should be usable as Transport option', () => {
const authProvider: ElasticSearchAuthProvider = {
getAuthHeaders: jest
.fn()
.mockResolvedValue({ Authorization: 'Bearer test' }),
};
const Transport = createElasticSearchAuthTransport(authProvider);
// Verify the Transport has the expected static properties
expect(Transport).toHaveProperty('sniffReasons');
});
});
describe('auth provider integration', () => {
it('should support async token retrieval', async () => {
let tokenRetrievalCount = 0;
const authProvider: ElasticSearchAuthProvider = {
getAuthHeaders: jest.fn().mockImplementation(async () => {
tokenRetrievalCount++;
// Simulate async token fetch
await new Promise(resolve => setTimeout(resolve, 10));
return { Authorization: `Bearer token-${tokenRetrievalCount}` };
}),
};
// Just verify the provider works correctly
const headers1 = await authProvider.getAuthHeaders();
const headers2 = await authProvider.getAuthHeaders();
expect(headers1.Authorization).toBe('Bearer token-1');
expect(headers2.Authorization).toBe('Bearer token-2');
expect(authProvider.getAuthHeaders).toHaveBeenCalledTimes(2);
});
it('should support token rotation pattern', async () => {
const tokens = ['initial-token', 'rotated-token', 'final-token'];
let currentTokenIndex = 0;
const authProvider: ElasticSearchAuthProvider = {
getAuthHeaders: jest.fn().mockImplementation(async () => {
const token = tokens[currentTokenIndex];
currentTokenIndex = Math.min(
currentTokenIndex + 1,
tokens.length - 1,
);
return { Authorization: `Bearer ${token}` };
}),
};
// Simulate multiple requests that would trigger token rotation
const headers1 = await authProvider.getAuthHeaders();
const headers2 = await authProvider.getAuthHeaders();
const headers3 = await authProvider.getAuthHeaders();
expect(headers1.Authorization).toBe('Bearer initial-token');
expect(headers2.Authorization).toBe('Bearer rotated-token');
expect(headers3.Authorization).toBe('Bearer final-token');
});
it('should handle errors in getAuthHeaders gracefully', async () => {
const authProvider: ElasticSearchAuthProvider = {
getAuthHeaders: jest
.fn()
.mockRejectedValue(new Error('Token fetch failed')),
};
await expect(authProvider.getAuthHeaders()).rejects.toThrow(
'Token fetch failed',
);
});
});
});
@@ -0,0 +1,153 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Transport as OpenSearchTransport } from '@opensearch-project/opensearch';
import { Transport as ElasticSearchTransport } from '@elastic/elasticsearch';
import { ElasticSearchAuthProvider } from '../auth';
import { ElasticSearchTransportConstructor } from './ElasticSearchClientOptions';
/**
* Creates a custom OpenSearch Transport class that injects authentication headers
* from the provided auth provider before each request.
*
* @param authProvider - The authentication provider to use for getting headers
* @returns A Transport class constructor that can be used with the OpenSearch client
*
* @internal
*/
export function createOpenSearchAuthTransport(
authProvider: ElasticSearchAuthProvider,
): ElasticSearchTransportConstructor {
class AuthTransport extends OpenSearchTransport {
request(
params: any,
optionsOrCallback: any = {},
maybeCallback?: any,
): any {
// options is optional, so if it's a function, it's the callback
const isOptionsCallback = typeof optionsOrCallback === 'function';
const options = isOptionsCallback ? {} : optionsOrCallback;
const callback = isOptionsCallback ? optionsOrCallback : maybeCallback;
// If callback style, wrap it to inject headers
if (callback !== undefined) {
authProvider
.getAuthHeaders()
.then(authHeaders => {
const mergedOptions = {
...options,
headers: {
...options.headers,
...authHeaders,
},
};
return super.request(params, mergedOptions, callback);
})
.catch(callback);
// Return an abort-able object similar to the AWS transport
return {
abort: () => {},
};
}
// Promise style
return authProvider.getAuthHeaders().then(authHeaders => {
const mergedOptions = {
...options,
headers: {
...options.headers,
...authHeaders,
},
};
return super.request(params, mergedOptions);
});
}
}
return AuthTransport as unknown as ElasticSearchTransportConstructor;
}
/**
* Creates a custom Elasticsearch Transport class that injects authentication headers
* from the provided auth provider before each request.
*
* @param authProvider - The authentication provider to use for getting headers
* @returns A Transport class constructor that can be used with the Elasticsearch client
*
* @internal
*/
export function createElasticSearchAuthTransport(
authProvider: ElasticSearchAuthProvider,
): ElasticSearchTransportConstructor {
class AuthTransport extends ElasticSearchTransport {
request(params: any, options?: any, callback?: any): any {
// Handle overloaded signatures
if (typeof options === 'function') {
// Callback style without options
const cb = options;
authProvider
.getAuthHeaders()
.then(authHeaders => {
const mergedOptions = {
headers: authHeaders,
};
return super.request(params, mergedOptions, cb);
})
.catch(cb);
return { abort: () => {} };
}
if (callback !== undefined) {
// Callback style with options
authProvider
.getAuthHeaders()
.then(authHeaders => {
const mergedOptions = {
...options,
headers: {
...options?.headers,
...authHeaders,
},
};
return super.request(params, mergedOptions, callback);
})
.catch(callback);
return { abort: () => {} };
}
// Promise style
const result = authProvider.getAuthHeaders().then(authHeaders => {
const mergedOptions = {
...options,
headers: {
...options?.headers,
...authHeaders,
},
};
return super.request(params, mergedOptions);
});
// Add abort method to match TransportRequestPromise interface
(result as any).abort = () => {};
return result;
}
}
return AuthTransport as unknown as ElasticSearchTransportConstructor;
}
@@ -996,6 +996,84 @@ describe('ElasticSearchSearchEngine', () => {
}),
).not.toThrow();
});
it('should accept an authProvider and not require auth config', async () => {
const config = new ConfigReader({
search: {
elasticsearch: {
node: 'http://test-node',
// No auth config - using authProvider instead
},
},
});
const authProvider = {
getAuthHeaders: jest
.fn()
.mockResolvedValue({ Authorization: 'Bearer test-token' }),
};
const engine = await ElasticSearchSearchEngine.fromConfig({
logger: mockServices.logger.mock(),
config,
authProvider,
});
expect(engine).toBeDefined();
});
it('should accept an authProvider with opensearch provider', async () => {
const config = new ConfigReader({
search: {
elasticsearch: {
provider: 'opensearch',
node: 'http://test-node',
// No auth config - using authProvider instead
},
},
});
const authProvider = {
getAuthHeaders: jest
.fn()
.mockResolvedValue({ Authorization: 'Bearer test-token' }),
};
const engine = await ElasticSearchSearchEngine.fromConfig({
logger: mockServices.logger.mock(),
config,
authProvider,
});
expect(engine).toBeDefined();
});
it('should throw error when using authProvider with aws provider', async () => {
const config = new ConfigReader({
search: {
elasticsearch: {
provider: 'aws',
node: 'http://test-node.us-east-1.es.amazonaws.com',
},
},
});
const authProvider = {
getAuthHeaders: jest
.fn()
.mockResolvedValue({ Authorization: 'Bearer test-token' }),
};
await expect(
ElasticSearchSearchEngine.fromConfig({
logger: mockServices.logger.mock(),
config,
authProvider,
}),
).rejects.toThrow(
'Custom auth provider is not supported with AWS provider',
);
});
});
});
@@ -41,6 +41,11 @@ import {
DefaultAwsCredentialsManager,
} from '@backstage/integration-aws-node';
import { LoggerService } from '@backstage/backend-plugin-api';
import { ElasticSearchAuthProvider } from '../auth';
import {
createOpenSearchAuthTransport,
createElasticSearchAuthTransport,
} from './ElasticSearchAuthTransport';
export type { ElasticSearchClientOptions };
@@ -82,6 +87,19 @@ export type ElasticSearchOptions = {
aliasPostfix?: string;
indexPrefix?: string;
translator?: ElasticSearchQueryTranslator;
/**
* An optional authentication provider for dynamic authentication.
*
* @remarks
*
* When provided, this auth provider will be used to inject authentication
* headers into each request to Elasticsearch/OpenSearch. This is useful
* for scenarios requiring dynamic tokens (e.g., bearer tokens with rotation).
*
* The auth provider takes precedence over static authentication configured
* in app-config.yaml.
*/
authProvider?: ElasticSearchAuthProvider;
};
/**
@@ -180,11 +198,13 @@ export class ElasticSearchSearchEngine implements SearchEngine {
aliasPostfix = `search`,
indexPrefix = ``,
translator,
authProvider,
} = options;
const credentialProvider = DefaultAwsCredentialsManager.fromConfig(config);
const clientOptions = await this.createElasticSearchClientOptions(
await credentialProvider?.getCredentialProvider(),
config.getConfig('search.elasticsearch'),
authProvider,
);
if (clientOptions.provider === 'elastic') {
logger.info('Initializing Elastic.co ElasticSearch search engine.');
@@ -504,6 +524,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
private static async createElasticSearchClientOptions(
credentialProvider: AwsCredentialProvider,
config?: Config,
authProvider?: ElasticSearchAuthProvider,
): Promise<ElasticSearchClientOptions> {
if (!config) {
throw new Error('No elastic search config found');
@@ -512,16 +533,26 @@ export class ElasticSearchSearchEngine implements SearchEngine {
const sslConfig = clientOptionsConfig?.getOptionalConfig('ssl');
if (config.getOptionalString('provider') === 'elastic') {
const authConfig = config.getConfig('auth');
const authConfig = authProvider ? undefined : config.getConfig('auth');
return {
provider: 'elastic',
cloud: {
id: config.getString('cloudId'),
},
auth: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
},
// When using authProvider, we inject auth via custom Transport, not static auth
...(authConfig
? {
auth: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
},
}
: {}),
...(authProvider
? {
Transport: createElasticSearchAuthTransport(authProvider),
}
: {}),
...(sslConfig
? {
ssl: {
@@ -533,6 +564,12 @@ export class ElasticSearchSearchEngine implements SearchEngine {
};
}
if (config.getOptionalString('provider') === 'aws') {
// AWS provider uses SigV4 signing; authProvider is not applicable here
if (authProvider) {
throw new Error(
'Custom auth provider is not supported with AWS provider. AWS uses SigV4 signing.',
);
}
const requestSigner = new RequestSigner(config.getString('node'));
const service =
config.getOptionalString('service') ?? requestSigner.service;
@@ -560,14 +597,26 @@ export class ElasticSearchSearchEngine implements SearchEngine {
};
}
if (config.getOptionalString('provider') === 'opensearch') {
const authConfig = config.getConfig('auth');
const authConfig = authProvider
? undefined
: config.getOptionalConfig('auth');
return {
provider: 'opensearch',
node: config.getString('node'),
auth: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
},
// When using authProvider, we inject auth via custom Transport, not static auth
...(authConfig
? {
auth: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
},
}
: {}),
...(authProvider
? {
Transport: createOpenSearchAuthTransport(authProvider),
}
: {}),
...(sslConfig
? {
ssl: {
@@ -578,7 +627,10 @@ export class ElasticSearchSearchEngine implements SearchEngine {
: {}),
};
}
const authConfig = config.getOptionalConfig('auth');
// Default provider (standard Elasticsearch)
const authConfig = authProvider
? undefined
: config.getOptionalConfig('auth');
const auth =
authConfig &&
(authConfig.has('apiKey')
@@ -591,7 +643,12 @@ export class ElasticSearchSearchEngine implements SearchEngine {
});
return {
node: config.getString('node'),
auth,
...(auth ? { auth } : {}),
...(authProvider
? {
Transport: createElasticSearchAuthTransport(authProvider),
}
: {}),
...(sslConfig
? {
ssl: {
@@ -22,6 +22,11 @@
export { default } from './module';
export * from './module';
export type {
ElasticSearchAuthProvider,
ElasticSearchAuthExtensionPoint,
} from './auth';
export { elasticsearchAuthExtensionPoint } from './auth';
export {
decodeElasticSearchPageCursor,
ElasticSearchSearchEngine,
@@ -23,6 +23,15 @@ import {
ElasticSearchQueryTranslator,
ElasticSearchSearchEngine,
} from './engines/ElasticSearchSearchEngine';
import {
type ElasticSearchAuthProvider,
type ElasticSearchAuthExtensionPoint,
elasticsearchAuthExtensionPoint,
} from './auth';
// Re-export auth types and extension point
export type { ElasticSearchAuthProvider, ElasticSearchAuthExtensionPoint };
export { elasticsearchAuthExtensionPoint };
/** @public */
export interface ElasticSearchQueryTranslatorExtensionPoint {
@@ -49,6 +58,7 @@ export default createBackendModule({
moduleId: 'elasticsearch-engine',
register(env) {
let translator: ElasticSearchQueryTranslator | undefined;
let authProvider: ElasticSearchAuthProvider | undefined;
env.registerExtensionPoint(elasticsearchTranslatorExtensionPoint, {
setTranslator(newTranslator) {
@@ -61,6 +71,15 @@ export default createBackendModule({
},
});
env.registerExtensionPoint(elasticsearchAuthExtensionPoint, {
setAuthProvider(provider) {
if (authProvider) {
throw new Error('ElasticSearch auth provider may only be set once');
}
authProvider = provider;
},
});
env.registerInit({
deps: {
searchEngineRegistry: searchEngineRegistryExtensionPoint,
@@ -82,11 +101,17 @@ export default createBackendModule({
if (indexPrefix) {
logger.info(`Index prefix will be used for indices: ${indexPrefix}`);
}
if (authProvider) {
logger.info(
'Using custom auth provider for ElasticSearch authentication.',
);
}
searchEngineRegistry.setSearchEngine(
await ElasticSearchSearchEngine.fromConfig({
logger,
config,
translator,
authProvider,
indexPrefix,
}),
);