Add pagination support to newrelic plugin

Signed-off-by: Robert Bunning <rbunning@webstaurantstore.com>

plugins/newrelic/README.md

@@ -18,6 +18,8 @@ APIs.
         target: https://api.newrelic.com/v2
         headers:
           X-Api-Key: ${NEW_RELIC_REST_API_KEY}
+        allowedHeaders:
+          - link
     ```
 
     There is some types of api key on new relic, to this use must be `User` type of key, In your production deployment of Backstage, you would also need to ensure that
@@ -33,6 +35,8 @@ APIs.
       '/newrelic/apm/api':
         headers:
           X-Api-Key: NRRA-YourActualApiKey
+        allowedHeaders:
+          - link
     ```
 
     Read more about how to find or generate this key in

plugins/newrelic/src/api/index.test.ts

@@ -0,0 +1,312 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { NewRelicClient } from '.';
+import { FetchApi } from '@backstage/core-plugin-api';
+import { DiscoveryApi } from '@backstage/core-plugin-api';
+
+beforeEach(() => {
+  jest.resetAllMocks();
+});
+
+describe('NewRelicClient', () => {
+  test.each([
+    ['https://test.test/BASEPATH/apm/api/applications.json', '/BASEPATH'],
+    ['https://test.test/BASEPATH2/apm/api/applications.json', '/BASEPATH2'],
+    ['https://test.testBASEPATH3/apm/api/applications.json', 'BASEPATH3'],
+    ['https://test.test/newrelic/apm/api/applications.json', undefined],
+  ])(
+    'It correctly forms the request url (%p) when proxyPathBase is %p',
+    async (expectedUrl, basePathOverride) => {
+      const mockedDiscoveryApi: DiscoveryApi = {
+        getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'),
+      };
+
+      const mockedFetchApi: FetchApi = {
+        fetch: jest.fn().mockResolvedValueOnce({
+          ok: true,
+          json: async () => [],
+          headers: new Map<string, string | null>(),
+        }),
+      };
+
+      const client = new NewRelicClient({
+        discoveryApi: mockedDiscoveryApi,
+        fetchApi: mockedFetchApi,
+        proxyPathBase: basePathOverride,
+      });
+      await client.getApplications();
+
+      expect(mockedFetchApi.fetch).toHaveBeenCalledWith(expectedUrl);
+    },
+  );
+
+  it('Correctly reads all pages of results and returns the expected results', async () => {
+    const mockedDiscoveryApi: DiscoveryApi = {
+      getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'),
+    };
+
+    const mockedApplicationOne = {
+      id: 1,
+      application_summary: {
+        apdex_score: 0,
+        error_rate: 100,
+        host_count: 500,
+        instance_count: 5000,
+        response_time: 20,
+        throughput: 500000,
+      },
+      name: 'Testing Application #1',
+      language: 'en-us',
+      health_status: 'Failing',
+      reporting: true,
+      settings: {
+        app_apdex_threshold: 0,
+        end_user_apdex_threshold: 0,
+        enable_real_user_monitoring: true,
+        use_server_side_config: true,
+      },
+    };
+
+    const mockedApplicationTwo = {
+      id: 2,
+      name: 'Testing Application #2',
+      language: 'en-us',
+      health_status: 'Working',
+      reporting: true,
+      settings: {
+        app_apdex_threshold: 0,
+        end_user_apdex_threshold: 0,
+        enable_real_user_monitoring: true,
+        use_server_side_config: true,
+      },
+    };
+
+    const mockedApplicationThree = {
+      id: 3,
+      application_summary: {
+        apdex_score: -900,
+        error_rate: 0,
+        host_count: 0,
+        instance_count: 0,
+        response_time: 0,
+        throughput: 0,
+      },
+      name: 'Testing Application #3',
+      language: 'en-us',
+      health_status: 'Waiting',
+      reporting: false,
+      settings: {
+        app_apdex_threshold: 1000,
+        end_user_apdex_threshold: 500,
+        enable_real_user_monitoring: false,
+        use_server_side_config: false,
+      },
+    };
+
+    const mockedFetchApi: FetchApi = {
+      fetch: jest
+        .fn()
+        .mockResolvedValueOnce({
+          ok: true,
+          json: async () => ({ applications: [mockedApplicationOne] }),
+          headers: new Map<string, string | null>([
+            [
+              'link',
+              '<https://next.page/page2>; rel="next", <https://badsite.dontgohere>; rel="next"',
+            ],
+            ['otherheader', 'otherValue'],
+          ]),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: async () => ({ applications: [] }),
+          headers: new Map<string, string | null>([
+            ['Link', '<https://next.page/page3>; rel="next",'],
+          ]),
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          json: async () => ({
+            applications: [mockedApplicationTwo, mockedApplicationThree],
+          }),
+          headers: new Map<string, string | null>([
+            [
+              'link',
+              '<https://first.page/page1>; rel="first", <https://next.page/page2>; rel="next"',
+            ],
+          ]),
+        }),
+    };
+
+    const client = new NewRelicClient({
+      discoveryApi: mockedDiscoveryApi,
+      fetchApi: mockedFetchApi,
+    });
+    const actual = await client.getApplications();
+    const expected = {
+      applications: [
+        mockedApplicationOne,
+        mockedApplicationTwo,
+        mockedApplicationThree,
+      ],
+    };
+
+    expect(mockedFetchApi.fetch).toHaveBeenCalledTimes(3);
+    expect(mockedFetchApi.fetch).toHaveBeenCalledWith(
+      'https://test.test/newrelic/apm/api/applications.json',
+    );
+    expect(mockedFetchApi.fetch).toHaveBeenCalledWith(
+      'https://next.page/page2',
+    );
+    expect(mockedFetchApi.fetch).toHaveBeenCalledWith(
+      'https://next.page/page3',
+    );
+    expect(actual).toStrictEqual(expected);
+  });
+
+  test.each([['LINK'], ['lINK']])(
+    'It does not attempt pagination when the link header name is invalid (%p)',
+    async linkHeaderName => {
+      const mockedDiscoveryApi: DiscoveryApi = {
+        getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'),
+      };
+
+      const mockedFetchApi: FetchApi = {
+        fetch: jest.fn().mockResolvedValueOnce({
+          ok: true,
+          json: async () => ({ applications: [] }),
+          headers: new Map<string, string | null>([
+            [
+              linkHeaderName,
+              '<https://next.page/page2>; rel="next", <https://badsite.dontgohere>; rel="next"',
+            ],
+            ['otherheader', 'otherValue'],
+          ]),
+        }),
+      };
+
+      const client = new NewRelicClient({
+        discoveryApi: mockedDiscoveryApi,
+        fetchApi: mockedFetchApi,
+      });
+      await client.getApplications();
+
+      expect(mockedFetchApi.fetch).toHaveBeenCalledTimes(1);
+      expect(mockedFetchApi.fetch).toHaveBeenCalledWith(
+        'https://test.test/newrelic/apm/api/applications.json',
+      );
+    },
+  );
+
+  test.each([
+    [''],
+    ['<> rel=""'],
+    ['<>; rel=""'],
+    ['<https://next.page>; rel=""'],
+    ['<>; rel:"value"'],
+    ['<https://next.page>; rel: "next"'],
+    ['ABCDE'],
+  ])(
+    'It does not attempt pagination when the link header value is invalid (%p)',
+    async linkHeaderValue => {
+      const mockedDiscoveryApi: DiscoveryApi = {
+        getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'),
+      };
+
+      const mockedFetchApi: FetchApi = {
+        fetch: jest.fn().mockResolvedValueOnce({
+          ok: true,
+          json: async () => ({ applications: [] }),
+          headers: new Map<string, string | null>([
+            ['Link', linkHeaderValue],
+            ['otherheader', 'otherValue'],
+          ]),
+        }),
+      };
+
+      const client = new NewRelicClient({
+        discoveryApi: mockedDiscoveryApi,
+        fetchApi: mockedFetchApi,
+      });
+      await client.getApplications();
+
+      expect(mockedFetchApi.fetch).toHaveBeenCalledTimes(1);
+      expect(mockedFetchApi.fetch).toHaveBeenCalledWith(
+        'https://test.test/newrelic/apm/api/applications.json',
+      );
+    },
+  );
+
+  test.each([
+    [
+      {
+        ok: false,
+        statusText: 'statusText',
+        json: async () => ({ error: { title: 'TESTING' } }),
+      },
+    ],
+    [
+      {
+        ok: false,
+        statusText: 'statusText',
+        json: async () => ({}),
+      },
+    ],
+  ])(
+    'It returns an empty array of applications when the fetch is not okay',
+    async fetchResult => {
+      const mockedDiscoveryApi: DiscoveryApi = {
+        getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'),
+      };
+
+      const mockedFetchApi: FetchApi = {
+        fetch: jest.fn().mockResolvedValueOnce({
+          ok: true,
+          json: async () => fetchResult,
+        }),
+      };
+
+      const client = new NewRelicClient({
+        discoveryApi: mockedDiscoveryApi,
+        fetchApi: mockedFetchApi,
+      });
+      const actual = await client.getApplications();
+
+      expect(actual).toStrictEqual({ applications: [] });
+    },
+  );
+
+  it('Returns an empty array when the fetch itself throws an error', async () => {
+    const mockedDiscoveryApi: DiscoveryApi = {
+      getBaseUrl: jest.fn().mockResolvedValueOnce('https://test.test'),
+    };
+
+    const mockedFetchApi: FetchApi = {
+      fetch: () => {
+        throw new Error('TESTING');
+      },
+    };
+
+    const client = new NewRelicClient({
+      discoveryApi: mockedDiscoveryApi,
+      fetchApi: mockedFetchApi,
+    });
+    const actual = await client.getApplications();
+
+    expect(actual).toStrictEqual({ applications: [] });
+  });
+});

plugins/newrelic/src/api/index.ts

@@ -14,7 +14,11 @@
  * limitations under the License.
  */
 
-import { createApiRef, DiscoveryApi } from '@backstage/core-plugin-api';
+import {
+  createApiRef,
+  DiscoveryApi,
+  FetchApi,
+} from '@backstage/core-plugin-api';
 
 export type NewRelicApplication = {
   id: number;
@@ -61,6 +65,7 @@ const DEFAULT_PROXY_PATH_BASE = '/newrelic';
 
 type Options = {
   discoveryApi: DiscoveryApi;
+  fetchApi: FetchApi;
   /**
    * Path to use for requests via the proxy, defaults to /newrelic
    */
@@ -71,27 +76,58 @@ export interface NewRelicApi {
   getApplications(): Promise<NewRelicApplications>;
 }
 
+interface PaginationInformation {
+  nextPageLink: string;
+  relation: string;
+}
+
+interface NewRelicPageReadResult {
+  hasMoreApplications: boolean;
+  pagination: PaginationInformation | undefined;
+  applicationsFromReadPage: NewRelicApplication[];
+}
+
 export class NewRelicClient implements NewRelicApi {
   private readonly discoveryApi: DiscoveryApi;
+  private readonly fetchApi: FetchApi;
   private readonly proxyPathBase: string;
 
   constructor(options: Options) {
     this.discoveryApi = options.discoveryApi;
+    this.fetchApi = options.fetchApi;
     this.proxyPathBase = options.proxyPathBase ?? DEFAULT_PROXY_PATH_BASE;
   }
 
   async getApplications(): Promise<NewRelicApplications> {
-    const url = await this.getApiUrl('apm', 'applications.json');
-    const response = await fetch(url);
-    let responseJson;
+    const proxyUrl = await this.discoveryApi.getBaseUrl('proxy');
+    let targetUrl = `${proxyUrl}${this.proxyPathBase}/apm/api/applications.json`;
+    let hasNextPage = true;
 
     try {
-      responseJson = await response.json();
-    } catch (e) {
-      responseJson = { applications: [] };
-    }
+      let applications: NewRelicApplication[] = [];
 
-    if (response.status !== 200) {
+      do {
+        const { hasMoreApplications, pagination, applicationsFromReadPage } =
+          await this.fetchNewRelic(targetUrl);
+
+        hasNextPage = hasMoreApplications;
+        targetUrl = pagination!.nextPageLink;
+        applications = applications.concat(applicationsFromReadPage);
+      } while (hasNextPage);
+
+      return { applications };
+    } catch (e) {
+      return { applications: [] };
+    }
+  }
+
+  private async fetchNewRelic(
+    targetUrl: string,
+  ): Promise<NewRelicPageReadResult> {
+    const response = await this.fetchApi.fetch(targetUrl);
+    const responseJson = await response.json();
+
+    if (!response.ok) {
       throw new Error(
         `Error communicating with New Relic: ${
           responseJson?.error?.title || response.statusText
@@ -99,11 +135,40 @@ export class NewRelicClient implements NewRelicApi {
       );
     }
 
-    return responseJson;
+    const readResponse = responseJson as NewRelicApplications;
+    const linkHeader =
+      response.headers.get('Link') || response.headers.get('link');
+    const pagination = this.parseLinkHeader(linkHeader);
+
+    return {
+      hasMoreApplications: !!(pagination && pagination.relation === 'next'),
+      pagination,
+      applicationsFromReadPage: readResponse.applications,
+    };
   }
 
-  private async getApiUrl(product: string, path: string) {
-    const proxyUrl = await this.discoveryApi.getBaseUrl('proxy');
-    return `${proxyUrl}${this.proxyPathBase}/${product}/api/${path}`;
+  private parseLinkHeader(
+    linkHeader: string | null,
+  ): PaginationInformation | undefined {
+    if (!linkHeader) {
+      return undefined;
+    }
+
+    const nextRelevantLink = linkHeader.replaceAll(' ', '').split(',')[0];
+
+    // Link should be of the format <nextPageLink>;rel="relation"
+    const linkParts = nextRelevantLink.match(/^<(.+)>;rel="(.+)"$/);
+    const nextPageLink = linkParts?.[1];
+    const relation = linkParts?.[2];
+    const isValidLink = !!(!!linkParts && nextPageLink && relation);
+
+    if (!nextRelevantLink || !isValidLink) {
+      return undefined;
+    }
+
+    return {
+      nextPageLink,
+      relation,
+    };
   }
 }

plugins/newrelic/src/plugin.ts

@@ -20,6 +20,7 @@ import {
   createPlugin,
   createRouteRef,
   discoveryApiRef,
+  fetchApiRef,
   createRoutableExtension,
 } from '@backstage/core-plugin-api';
 
@@ -33,8 +34,12 @@ export const newRelicPlugin = createPlugin({
   apis: [
     createApiFactory({
       api: newRelicApiRef,
-      deps: { discoveryApi: discoveryApiRef },
-      factory: ({ discoveryApi }) => new NewRelicClient({ discoveryApi }),
+      deps: {
+        discoveryApi: discoveryApiRef,
+        fetchApi: fetchApiRef,
+      },
+      factory: ({ discoveryApi, fetchApi }) =>
+        new NewRelicClient({ discoveryApi, fetchApi }),
     }),
   ],
   routes: {