diff --git a/.changeset/khaki-camels-rush.md b/.changeset/khaki-camels-rush.md new file mode 100644 index 0000000000..a38600abf3 --- /dev/null +++ b/.changeset/khaki-camels-rush.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-newrelic': patch +--- + +The newrelic plugin now supports pagination when retrieving results from newrelic. It will no longer truncate results. To see all applications, the link header will need to be allowed through the proxy (see the newrelic plugin readme). diff --git a/app-config.yaml b/app-config.yaml index 2b562ee689..da505d390b 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -70,6 +70,8 @@ proxy: target: https://api.newrelic.com/v2 headers: X-Api-Key: ${NEW_RELIC_REST_API_KEY} + allowedHeaders: + - link '/newrelic/api': target: https://api.newrelic.com diff --git a/plugins/newrelic/README.md b/plugins/newrelic/README.md index 8ac7ed2c72..14049ba39c 100644 --- a/plugins/newrelic/README.md +++ b/plugins/newrelic/README.md @@ -18,6 +18,8 @@ APIs. target: https://api.newrelic.com/v2 headers: X-Api-Key: ${NEW_RELIC_REST_API_KEY} + allowedHeaders: + - link ``` There is some types of api key on new relic, to this use must be `User` type of key, In your production deployment of Backstage, you would also need to ensure that @@ -33,6 +35,8 @@ APIs. '/newrelic/apm/api': headers: X-Api-Key: NRRA-YourActualApiKey + allowedHeaders: + - link ``` Read more about how to find or generate this key in diff --git a/plugins/newrelic/src/api/index.test.ts b/plugins/newrelic/src/api/index.test.ts new file mode 100644 index 0000000000..13d9c5304d --- /dev/null +++ b/plugins/newrelic/src/api/index.test.ts @@ -0,0 +1,312 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NewRelicClient } from '.'; +import { FetchApi } from '@backstage/core-plugin-api'; +import { DiscoveryApi } from '@backstage/core-plugin-api'; + +beforeEach(() => { + jest.resetAllMocks(); +}); + +describe('NewRelicClient', () => { + test.each([ + ['https://test.test/BASEPATH/apm/api/applications.json', '/BASEPATH'], + ['https://test.test/BASEPATH2/apm/api/applications.json', '/BASEPATH2'], + ['https://test.testBASEPATH3/apm/api/applications.json', 'BASEPATH3'], + ['https://test.test/newrelic/apm/api/applications.json', undefined], + ])( + 'It correctly forms the request url (%p) when proxyPathBase is %p', + async (expectedUrl, basePathOverride) => { + const mockedDiscoveryApi: DiscoveryApi = { + getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'), + }; + + const mockedFetchApi: FetchApi = { + fetch: jest.fn().mockResolvedValueOnce({ + ok: true, + json: async () => [], + headers: new Map(), + }), + }; + + const client = new NewRelicClient({ + discoveryApi: mockedDiscoveryApi, + fetchApi: mockedFetchApi, + proxyPathBase: basePathOverride, + }); + await client.getApplications(); + + expect(mockedFetchApi.fetch).toHaveBeenCalledWith(expectedUrl); + }, + ); + + it('Correctly reads all pages of results and returns the expected results', async () => { + const mockedDiscoveryApi: DiscoveryApi = { + getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'), + }; + + const mockedApplicationOne = { + id: 1, + application_summary: { + apdex_score: 0, + error_rate: 100, + host_count: 500, + instance_count: 5000, + response_time: 20, + throughput: 500000, + }, + name: 'Testing Application #1', + language: 'en-us', + health_status: 'Failing', + reporting: true, + settings: { + app_apdex_threshold: 0, + end_user_apdex_threshold: 0, + enable_real_user_monitoring: true, + use_server_side_config: true, + }, + }; + + const mockedApplicationTwo = { + id: 2, + name: 'Testing Application #2', + language: 'en-us', + health_status: 'Working', + reporting: true, + settings: { + app_apdex_threshold: 0, + end_user_apdex_threshold: 0, + enable_real_user_monitoring: true, + use_server_side_config: true, + }, + }; + + const mockedApplicationThree = { + id: 3, + application_summary: { + apdex_score: -900, + error_rate: 0, + host_count: 0, + instance_count: 0, + response_time: 0, + throughput: 0, + }, + name: 'Testing Application #3', + language: 'en-us', + health_status: 'Waiting', + reporting: false, + settings: { + app_apdex_threshold: 1000, + end_user_apdex_threshold: 500, + enable_real_user_monitoring: false, + use_server_side_config: false, + }, + }; + + const mockedFetchApi: FetchApi = { + fetch: jest + .fn() + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ applications: [mockedApplicationOne] }), + headers: new Map([ + [ + 'link', + '; rel="next", ; rel="next"', + ], + ['otherheader', 'otherValue'], + ]), + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ applications: [] }), + headers: new Map([ + ['Link', '; rel="next",'], + ]), + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + applications: [mockedApplicationTwo, mockedApplicationThree], + }), + headers: new Map([ + [ + 'link', + '; rel="first", ; rel="next"', + ], + ]), + }), + }; + + const client = new NewRelicClient({ + discoveryApi: mockedDiscoveryApi, + fetchApi: mockedFetchApi, + }); + const actual = await client.getApplications(); + const expected = { + applications: [ + mockedApplicationOne, + mockedApplicationTwo, + mockedApplicationThree, + ], + }; + + expect(mockedFetchApi.fetch).toHaveBeenCalledTimes(3); + expect(mockedFetchApi.fetch).toHaveBeenCalledWith( + 'https://test.test/newrelic/apm/api/applications.json', + ); + expect(mockedFetchApi.fetch).toHaveBeenCalledWith( + 'https://next.page/page2', + ); + expect(mockedFetchApi.fetch).toHaveBeenCalledWith( + 'https://next.page/page3', + ); + expect(actual).toStrictEqual(expected); + }); + + test.each([['LINK'], ['lINK']])( + 'It does not attempt pagination when the link header name is invalid (%p)', + async linkHeaderName => { + const mockedDiscoveryApi: DiscoveryApi = { + getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'), + }; + + const mockedFetchApi: FetchApi = { + fetch: jest.fn().mockResolvedValueOnce({ + ok: true, + json: async () => ({ applications: [] }), + headers: new Map([ + [ + linkHeaderName, + '; rel="next", ; rel="next"', + ], + ['otherheader', 'otherValue'], + ]), + }), + }; + + const client = new NewRelicClient({ + discoveryApi: mockedDiscoveryApi, + fetchApi: mockedFetchApi, + }); + await client.getApplications(); + + expect(mockedFetchApi.fetch).toHaveBeenCalledTimes(1); + expect(mockedFetchApi.fetch).toHaveBeenCalledWith( + 'https://test.test/newrelic/apm/api/applications.json', + ); + }, + ); + + test.each([ + [''], + ['<> rel=""'], + ['<>; rel=""'], + ['; rel=""'], + ['<>; rel:"value"'], + ['; rel: "next"'], + ['ABCDE'], + ])( + 'It does not attempt pagination when the link header value is invalid (%p)', + async linkHeaderValue => { + const mockedDiscoveryApi: DiscoveryApi = { + getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'), + }; + + const mockedFetchApi: FetchApi = { + fetch: jest.fn().mockResolvedValueOnce({ + ok: true, + json: async () => ({ applications: [] }), + headers: new Map([ + ['Link', linkHeaderValue], + ['otherheader', 'otherValue'], + ]), + }), + }; + + const client = new NewRelicClient({ + discoveryApi: mockedDiscoveryApi, + fetchApi: mockedFetchApi, + }); + await client.getApplications(); + + expect(mockedFetchApi.fetch).toHaveBeenCalledTimes(1); + expect(mockedFetchApi.fetch).toHaveBeenCalledWith( + 'https://test.test/newrelic/apm/api/applications.json', + ); + }, + ); + + test.each([ + [ + { + ok: false, + statusText: 'statusText', + json: async () => ({ error: { title: 'TESTING' } }), + }, + ], + [ + { + ok: false, + statusText: 'statusText', + json: async () => ({}), + }, + ], + ])( + 'It returns an empty array of applications when the fetch is not okay', + async fetchResult => { + const mockedDiscoveryApi: DiscoveryApi = { + getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'), + }; + + const mockedFetchApi: FetchApi = { + fetch: jest.fn().mockResolvedValueOnce({ + ok: true, + json: async () => fetchResult, + }), + }; + + const client = new NewRelicClient({ + discoveryApi: mockedDiscoveryApi, + fetchApi: mockedFetchApi, + }); + const actual = await client.getApplications(); + + expect(actual).toStrictEqual({ applications: [] }); + }, + ); + + it('Returns an empty array when the fetch itself throws an error', async () => { + const mockedDiscoveryApi: DiscoveryApi = { + getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'), + }; + + const mockedFetchApi: FetchApi = { + fetch: () => { + throw new Error('TESTING'); + }, + }; + + const client = new NewRelicClient({ + discoveryApi: mockedDiscoveryApi, + fetchApi: mockedFetchApi, + }); + const actual = await client.getApplications(); + + expect(actual).toStrictEqual({ applications: [] }); + }); +}); diff --git a/plugins/newrelic/src/api/index.ts b/plugins/newrelic/src/api/index.ts index 2734dd2b4f..9c95f3fe1f 100644 --- a/plugins/newrelic/src/api/index.ts +++ b/plugins/newrelic/src/api/index.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { createApiRef, DiscoveryApi } from '@backstage/core-plugin-api'; +import { + createApiRef, + DiscoveryApi, + FetchApi, +} from '@backstage/core-plugin-api'; export type NewRelicApplication = { id: number; @@ -61,6 +65,7 @@ const DEFAULT_PROXY_PATH_BASE = '/newrelic'; type Options = { discoveryApi: DiscoveryApi; + fetchApi: FetchApi; /** * Path to use for requests via the proxy, defaults to /newrelic */ @@ -71,27 +76,58 @@ export interface NewRelicApi { getApplications(): Promise; } +interface PaginationInformation { + nextPageLink: string; + relation: string; +} + +interface NewRelicPageReadResult { + hasMoreApplications: boolean; + pagination: PaginationInformation | undefined; + applicationsFromReadPage: NewRelicApplication[]; +} + export class NewRelicClient implements NewRelicApi { private readonly discoveryApi: DiscoveryApi; + private readonly fetchApi: FetchApi; private readonly proxyPathBase: string; constructor(options: Options) { this.discoveryApi = options.discoveryApi; + this.fetchApi = options.fetchApi; this.proxyPathBase = options.proxyPathBase ?? DEFAULT_PROXY_PATH_BASE; } async getApplications(): Promise { - const url = await this.getApiUrl('apm', 'applications.json'); - const response = await fetch(url); - let responseJson; + const proxyUrl = await this.discoveryApi.getBaseUrl('proxy'); + let targetUrl = `${proxyUrl}${this.proxyPathBase}/apm/api/applications.json`; + let hasNextPage = true; try { - responseJson = await response.json(); - } catch (e) { - responseJson = { applications: [] }; - } + let applications: NewRelicApplication[] = []; - if (response.status !== 200) { + do { + const { hasMoreApplications, pagination, applicationsFromReadPage } = + await this.fetchNewRelic(targetUrl); + + hasNextPage = hasMoreApplications; + targetUrl = pagination!.nextPageLink; + applications = applications.concat(applicationsFromReadPage); + } while (hasNextPage); + + return { applications }; + } catch (e) { + return { applications: [] }; + } + } + + private async fetchNewRelic( + targetUrl: string, + ): Promise { + const response = await this.fetchApi.fetch(targetUrl); + const responseJson = await response.json(); + + if (!response.ok) { throw new Error( `Error communicating with New Relic: ${ responseJson?.error?.title || response.statusText @@ -99,11 +135,40 @@ export class NewRelicClient implements NewRelicApi { ); } - return responseJson; + const readResponse = responseJson as NewRelicApplications; + const linkHeader = + response.headers.get('Link') || response.headers.get('link'); + const pagination = this.parseLinkHeader(linkHeader); + + return { + hasMoreApplications: !!(pagination && pagination.relation === 'next'), + pagination, + applicationsFromReadPage: readResponse.applications, + }; } - private async getApiUrl(product: string, path: string) { - const proxyUrl = await this.discoveryApi.getBaseUrl('proxy'); - return `${proxyUrl}${this.proxyPathBase}/${product}/api/${path}`; + private parseLinkHeader( + linkHeader: string | null, + ): PaginationInformation | undefined { + if (!linkHeader) { + return undefined; + } + + const nextRelevantLink = linkHeader.replaceAll(' ', '').split(',')[0]; + + // Link should be of the format ;rel="relation" + const linkParts = nextRelevantLink.match(/^<(.+)>;rel="(.+)"$/); + const nextPageLink = linkParts?.[1]; + const relation = linkParts?.[2]; + const isValidLink = !!(!!linkParts && nextPageLink && relation); + + if (!nextRelevantLink || !isValidLink) { + return undefined; + } + + return { + nextPageLink, + relation, + }; } } diff --git a/plugins/newrelic/src/plugin.ts b/plugins/newrelic/src/plugin.ts index 0387c50aa9..da7cae9b93 100644 --- a/plugins/newrelic/src/plugin.ts +++ b/plugins/newrelic/src/plugin.ts @@ -20,6 +20,7 @@ import { createPlugin, createRouteRef, discoveryApiRef, + fetchApiRef, createRoutableExtension, } from '@backstage/core-plugin-api'; @@ -33,8 +34,12 @@ export const newRelicPlugin = createPlugin({ apis: [ createApiFactory({ api: newRelicApiRef, - deps: { discoveryApi: discoveryApiRef }, - factory: ({ discoveryApi }) => new NewRelicClient({ discoveryApi }), + deps: { + discoveryApi: discoveryApiRef, + fetchApi: fetchApiRef, + }, + factory: ({ discoveryApi, fetchApi }) => + new NewRelicClient({ discoveryApi, fetchApi }), }), ], routes: {