Add paginated entities endpoint

Signed-off-by: Vincenzo Scamporlino <vincenzos@spotify.com>

plugins/catalog-backend/src/catalog/types.ts

@@ -174,6 +174,16 @@ export interface EntitiesCatalog {
    */
   entitiesBatch(request: EntitiesBatchRequest): Promise<EntitiesBatchResponse>;
 
+  /**
+   * Fetch entities and scroll back and forth between entities.
+   *
+   * @alpha
+   * @param request
+   */
+  paginatedEntities(
+    request?: PaginatedEntitiesRequest,
+  ): Promise<PaginatedEntitiesResponse>;
+
   /**
    * Removes a single entity.
    *
@@ -202,3 +212,108 @@ export interface EntitiesCatalog {
    */
   facets(request: EntityFacetsRequest): Promise<EntityFacetsResponse>;
 }
+
+/**
+ * The request shape for {@link EntitiesCatalog.paginatedEntities}.
+ *
+ * @alpha
+ */
+export type PaginatedEntitiesRequest =
+  | PaginatedEntitiesInitialRequest
+  | PaginatedEntitiesCursorRequest;
+
+/**
+ * The initial request for {@link EntitiesCatalog.paginatedEntities}.
+ * The request take immutable properties that are going to be bound
+ * for the current and the next pagination requests.
+ *
+ * @alpha
+ */
+export interface PaginatedEntitiesInitialRequest {
+  authorizationToken?: string;
+  fields?: (entity: Entity) => Entity;
+  limit?: number;
+  filter?: EntityFilter;
+  sortField?: string;
+  query?: string;
+  sortFieldOrder?: 'asc' | 'desc' | undefined;
+}
+
+/**
+ * Request for {@link EntitiesCatalog.paginatedEntities} used to
+ * move forward or backward on the data.
+ *
+ * @alpha
+ */
+export interface PaginatedEntitiesCursorRequest {
+  authorizationToken?: string;
+  fields?: (entity: Entity) => Entity;
+  limit?: number;
+  cursor: string;
+}
+
+/**
+ * The response shape for {@link EntitiesCatalog.paginatedEntities}.
+ *
+ * @alpha
+ */
+export interface PaginatedEntitiesResponse {
+  /**
+   * The entities for the current pagination request
+   */
+  entities: Entity[];
+  /**
+   * The cursor of the next pagination request.
+   */
+  nextCursor?: string;
+  /**
+   * The cursor of the previous pagination request.
+   */
+  prevCursor?: string;
+  /**
+   * the total number of entities matching the current filters.
+   */
+  totalItems: number;
+}
+
+/**
+ * The Cursor used internally by the catalog.
+ *
+ * @alpha
+ */
+export type Cursor = {
+  /**
+   * the id of the field used for sorting the data.
+   * For example, metadata.name
+   */
+  sortField: string;
+  /**
+   * The value of the last item returned
+   * by the request. This is used for performing
+   * cursor based pagination.
+   */
+  sortFieldId: string;
+  /**
+   * In which order the data should be paginated.
+   */
+  sortFieldOrder: 'asc' | 'desc';
+  filter?: EntityFilter;
+  /**
+   * true if the cursor is a previous cursor.
+   */
+  isPrevious: boolean;
+  /**
+   * Filter the data by name.
+   */
+  query?: string;
+  /**
+   * Sort field id of the first item.
+   * The catalog uses this field internally for understanding when the beginning
+   * of the list has been reached when performing cursor based pagination.
+   */
+  firstFieldId: string;
+  /**
+   * The number of items that match the provided filters
+   */
+  totalItems?: number;
+};

plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts

@@ -35,8 +35,11 @@ import {
   EntityFacetsRequest,
   EntityFacetsResponse,
   EntityFilter,
+  PaginatedEntitiesRequest,
+  PaginatedEntitiesResponse,
 } from '../catalog/types';
 import { basicEntityFilter } from './request/basicEntityFilter';
+import { isPaginatedEntitiesInitialRequest } from './util';
 
 export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
   constructor(
@@ -106,6 +109,41 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
     return this.entitiesCatalog.entitiesBatch(request);
   }
 
+  async paginatedEntities(
+    request?: PaginatedEntitiesRequest,
+  ): Promise<PaginatedEntitiesResponse> {
+    const authorizeDecision = (
+      await this.permissionApi.authorizeConditional(
+        [{ permission: catalogEntityReadPermission }],
+        { token: request?.authorizationToken },
+      )
+    )[0];
+
+    if (authorizeDecision.result === AuthorizeResult.DENY) {
+      return {
+        entities: [],
+        totalItems: 0,
+      };
+    }
+
+    if (authorizeDecision.result === AuthorizeResult.CONDITIONAL) {
+      const permissionFilter: EntityFilter = this.transformConditions(
+        authorizeDecision.conditions,
+      );
+
+      return this.entitiesCatalog.paginatedEntities({
+        ...request,
+        ...(isPaginatedEntitiesInitialRequest(request) && {
+          filter: request?.filter
+            ? { allOf: [permissionFilter, request.filter] }
+            : permissionFilter,
+        }),
+      });
+    }
+
+    return this.entitiesCatalog.paginatedEntities(request);
+  }
+
   async removeEntityByUid(
     uid: string,
     options?: { authorizationToken?: string },

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts

@@ -18,6 +18,10 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
 import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
 import { Knex } from 'knex';
 import { v4 as uuid } from 'uuid';
+import {
+  PaginatedEntitiesCursorRequest,
+  PaginatedEntitiesInitialRequest,
+} from '../catalog/types';
 import { applyDatabaseMigrations } from '../database/migrations';
 import {
   DbFinalEntitiesRow,
@@ -674,6 +678,376 @@ describe('DefaultEntitiesCatalog', () => {
     );
   });
 
+  describe('paginatedEntities', () => {
+    it.each(databases.eachSupportedId())(
+      'should return paginated entities and scroll the items accordingly, %p',
+      async databaseId => {
+        const { knex } = await createDatabase(databaseId);
+
+        function entityFrom(name: string) {
+          return {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name },
+            spec: { should_include_this: 'yes' },
+          };
+        }
+
+        const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E'];
+        const entities: Entity[] = names.map(entityFrom);
+
+        const notFoundEntities: Entity[] = [
+          {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name: 'something' },
+            spec: {},
+          },
+          {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name: 'something else' },
+            spec: {},
+          },
+        ];
+
+        await Promise.all(
+          entities
+            .concat(notFoundEntities)
+            .map(e => addEntityToSearch(knex, e)),
+        );
+
+        const catalog = new DefaultEntitiesCatalog(knex);
+
+        const filter = {
+          key: 'spec.should_include_this',
+        };
+
+        const limit = 2;
+
+        // initial request
+        const request1: PaginatedEntitiesInitialRequest = {
+          filter,
+          limit,
+          sortField: 'metadata.name',
+        };
+        const response1 = await catalog.paginatedEntities(request1);
+        expect(response1.entities).toEqual([entityFrom('A'), entityFrom('B')]);
+        expect(response1.nextCursor).toBeDefined();
+        expect(response1.prevCursor).toBeUndefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // second request (forward)
+        const request2: PaginatedEntitiesCursorRequest = {
+          cursor: response1.nextCursor!,
+          limit,
+        };
+        const response2 = await catalog.paginatedEntities(request2);
+        expect(response2.entities).toEqual([entityFrom('C'), entityFrom('D')]);
+        expect(response2.nextCursor).toBeDefined();
+        expect(response2.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // third request (forward)
+        const request3: PaginatedEntitiesCursorRequest = {
+          cursor: response2.nextCursor!,
+          limit,
+        };
+        const response3 = await catalog.paginatedEntities(request3);
+        expect(response3.entities).toEqual([entityFrom('E'), entityFrom('F')]);
+        expect(response3.nextCursor).toBeDefined();
+        expect(response3.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // fourth request (backwards)
+        const request4: PaginatedEntitiesCursorRequest = {
+          cursor: response3.prevCursor!,
+          limit,
+        };
+        const response4 = await catalog.paginatedEntities(request4);
+        expect(response4.entities).toEqual([entityFrom('C'), entityFrom('D')]);
+        expect(response4.nextCursor).toBeDefined();
+        expect(response4.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // fifth request (backwards)
+        const request5: PaginatedEntitiesCursorRequest = {
+          cursor: response4.prevCursor!,
+          limit,
+        };
+        const response5 = await catalog.paginatedEntities(request5);
+        expect(response5.entities).toEqual([entityFrom('A'), entityFrom('B')]);
+        expect(response5.nextCursor).toBeDefined();
+        expect(response5.prevCursor).toBeUndefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // sixth request (forward)
+        const request6: PaginatedEntitiesCursorRequest = {
+          cursor: response5.nextCursor!,
+          limit,
+        };
+        const response6 = await catalog.paginatedEntities(request6);
+        expect(response6.entities).toEqual([entityFrom('C'), entityFrom('D')]);
+        expect(response6.nextCursor).toBeDefined();
+        expect(response6.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // seventh request (forward)
+        const request7: PaginatedEntitiesCursorRequest = {
+          cursor: response6.nextCursor!,
+          limit,
+        };
+        const response7 = await catalog.paginatedEntities(request7);
+        expect(response7.entities).toEqual([entityFrom('E'), entityFrom('F')]);
+        expect(response7.nextCursor).toBeDefined();
+        expect(response7.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // seventh.2 request (forward with a different limit)
+        const request7bis: PaginatedEntitiesCursorRequest = {
+          cursor: response6.nextCursor!,
+          limit: limit + 1,
+        };
+        const response7bis = await catalog.paginatedEntities(request7bis);
+        expect(response7bis.entities).toEqual([
+          entityFrom('E'),
+          entityFrom('F'),
+          entityFrom('G'),
+        ]);
+        expect(response7bis.nextCursor).toBeUndefined();
+        expect(response7bis.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // last request (forward)
+        const request8: PaginatedEntitiesCursorRequest = {
+          cursor: response7.nextCursor!,
+          limit,
+        };
+        const response8 = await catalog.paginatedEntities(request8);
+        expect(response8.entities).toEqual([entityFrom('G')]);
+        expect(response8.nextCursor).toBeUndefined();
+        expect(response8.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+      },
+    );
+
+    it.each(databases.eachSupportedId())(
+      'should return paginated entities ordered in descending order and scroll the items accordingly, %p',
+      async databaseId => {
+        const { knex } = await createDatabase(databaseId);
+
+        function entityFrom(name: string) {
+          return {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name },
+            spec: { should_include_this: 'yes' },
+          };
+        }
+
+        const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E'];
+        const entities: Entity[] = names.map(entityFrom);
+
+        const notFoundEntities: Entity[] = [
+          {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name: 'something' },
+            spec: {},
+          },
+          {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name: 'something else' },
+            spec: {},
+          },
+        ];
+
+        await Promise.all(
+          entities
+            .concat(notFoundEntities)
+            .map(e => addEntityToSearch(knex, e)),
+        );
+
+        const catalog = new DefaultEntitiesCatalog(knex);
+
+        const filter = {
+          key: 'spec.should_include_this',
+        };
+
+        const limit = 2;
+
+        // initial request
+        const request1: PaginatedEntitiesInitialRequest = {
+          filter,
+          limit,
+          sortField: 'metadata.name',
+          sortFieldOrder: 'desc',
+        };
+        const response1 = await catalog.paginatedEntities(request1);
+        expect(response1.entities).toEqual([entityFrom('G'), entityFrom('F')]);
+        expect(response1.nextCursor).toBeDefined();
+        expect(response1.prevCursor).toBeUndefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // second request (forward)
+        const request2: PaginatedEntitiesCursorRequest = {
+          cursor: response1.nextCursor!,
+          limit,
+        };
+        const response2 = await catalog.paginatedEntities(request2);
+        expect(response2.entities).toEqual([entityFrom('E'), entityFrom('D')]);
+        expect(response2.nextCursor).toBeDefined();
+        expect(response2.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // third request (forward)
+        const request3: PaginatedEntitiesCursorRequest = {
+          cursor: response2.nextCursor!,
+          limit,
+        };
+        const response3 = await catalog.paginatedEntities(request3);
+        expect(response3.entities).toEqual([entityFrom('C'), entityFrom('B')]);
+        expect(response3.nextCursor).toBeDefined();
+        expect(response3.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // fourth request (backwards)
+        const request4: PaginatedEntitiesCursorRequest = {
+          cursor: response3.prevCursor!,
+          limit,
+        };
+        const response4 = await catalog.paginatedEntities(request4);
+
+        expect(response4.entities).toEqual([entityFrom('E'), entityFrom('D')]);
+        expect(response4.nextCursor).toBeDefined();
+        expect(response4.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // fifth request (backwards)
+        const request5: PaginatedEntitiesCursorRequest = {
+          cursor: response4.prevCursor!,
+          limit,
+        };
+        const response5 = await catalog.paginatedEntities(request5);
+        expect(response5.entities).toEqual([entityFrom('G'), entityFrom('F')]);
+        expect(response5.nextCursor).toBeDefined();
+        expect(response5.prevCursor).toBeUndefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // sixth request (forward)
+        const request6: PaginatedEntitiesCursorRequest = {
+          cursor: response5.nextCursor!,
+          limit,
+        };
+        const response6 = await catalog.paginatedEntities(request6);
+        expect(response6.entities).toEqual([entityFrom('E'), entityFrom('D')]);
+        expect(response6.nextCursor).toBeDefined();
+        expect(response6.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // seventh request (forward)
+        const request7: PaginatedEntitiesCursorRequest = {
+          cursor: response6.nextCursor!,
+          limit,
+        };
+        const response7 = await catalog.paginatedEntities(request7);
+        expect(response7.entities).toEqual([entityFrom('C'), entityFrom('B')]);
+        expect(response7.nextCursor).toBeDefined();
+        expect(response7.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // seventh.2 request (forward with a different limit)
+        const request7bis: PaginatedEntitiesCursorRequest = {
+          cursor: response6.nextCursor!,
+          limit: limit + 1,
+        };
+        const response7bis = await catalog.paginatedEntities(request7bis);
+        expect(response7bis.entities).toEqual([
+          entityFrom('C'),
+          entityFrom('B'),
+          entityFrom('A'),
+        ]);
+        expect(response7bis.nextCursor).toBeUndefined();
+        expect(response7bis.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+
+        // last request (forward)
+        const request8: PaginatedEntitiesCursorRequest = {
+          cursor: response7.nextCursor!,
+          limit,
+        };
+        const response8 = await catalog.paginatedEntities(request8);
+        expect(response8.entities).toEqual([entityFrom('A')]);
+        expect(response8.nextCursor).toBeUndefined();
+        expect(response8.prevCursor).toBeDefined();
+        expect(response1.totalItems).toBe(names.length);
+      },
+    );
+
+    it.each(databases.eachSupportedId())(
+      'should filter the results when query is provided, %p',
+      async databaseId => {
+        const { knex } = await createDatabase(databaseId);
+
+        function entityFrom(name: string) {
+          return {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name },
+            spec: { should_include_this: 'yes' },
+          };
+        }
+
+        const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's'];
+        const entities: Entity[] = names.map(entityFrom);
+
+        const notFoundEntities: Entity[] = [
+          {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name: 'something' },
+            spec: {},
+          },
+          {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name: 'something else' },
+            spec: {},
+          },
+        ];
+
+        await Promise.all(
+          entities
+            .concat(notFoundEntities)
+            .map(e => addEntityToSearch(knex, e)),
+        );
+
+        const catalog = new DefaultEntitiesCatalog(knex);
+
+        const filter = {
+          key: 'spec.should_include_this',
+        };
+
+        const request: PaginatedEntitiesInitialRequest = {
+          filter,
+          limit: 100,
+          sortField: 'metadata.name',
+          query: 'cAt ',
+        };
+        const response = await catalog.paginatedEntities(request);
+        expect(response.entities).toEqual([
+          entityFrom('atcatss'),
+          entityFrom('cat'),
+          entityFrom('dogcat'),
+        ]);
+        expect(response.nextCursor).toBeUndefined();
+        expect(response.prevCursor).toBeUndefined();
+        expect(response.totalItems).toBe(3);
+      },
+    );
+  });
+
   describe('removeEntityByUid', () => {
     it.each(databases.eachSupportedId())(
       'also clears parent hashes, %p',
@@ -889,5 +1263,89 @@ describe('DefaultEntitiesCatalog', () => {
       },
       60_000,
     );
+
+    it.each(databases.eachSupportedId())(
+      'can match relations',
+      async databaseId => {
+        const { knex } = await createDatabase(databaseId);
+
+        await addEntityToSearch(knex, {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: {
+            name: 'one',
+            tags: ['java', 'rust'],
+          },
+          spec: {},
+          relations: [
+            {
+              targetRef: 'targetRef',
+              type: 'ownedBy',
+              target: { kind: 'k', namespace: 'default', name: 'targetRef' },
+            },
+          ],
+        });
+        await addEntityToSearch(knex, {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: {
+            name: 'two',
+            tags: ['java', 'node'],
+          },
+          spec: {},
+          relations: [
+            {
+              targetRef: 'anotherTargetRef',
+              type: 'ownedBy',
+              target: {
+                kind: 'k',
+                namespace: 'default',
+                name: 'anotherTargetRef',
+              },
+            },
+            {
+              targetRef: 'tt',
+              type: 'somethingelse',
+              target: {
+                kind: 'k',
+                namespace: 'default',
+                name: 'tt',
+              },
+            },
+          ],
+        });
+        await addEntityToSearch(knex, {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: {
+            name: 'three',
+            tags: ['go'],
+          },
+          spec: {},
+          relations: [
+            {
+              targetRef: 'targetRef',
+              type: 'ownedBy',
+              target: { kind: 'k', namespace: 'default', name: 'targetRef' },
+            },
+          ],
+        });
+
+        const catalog = new DefaultEntitiesCatalog(knex);
+
+        await expect(
+          catalog.facets({
+            facets: ['relations.ownedBy'],
+          }),
+        ).resolves.toEqual({
+          facets: {
+            'relations.ownedBy': [
+              { count: 1, value: 'anotherTargetRef' },
+              { count: 2, value: 'targetRef' },
+            ],
+          },
+        });
+      },
+    );
   });
 });

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts

@@ -25,6 +25,7 @@ import { Knex } from 'knex';
 import {
   EntitiesBatchRequest,
   EntitiesBatchResponse,
+  Cursor,
   EntitiesCatalog,
   EntitiesRequest,
   EntitiesResponse,
@@ -34,6 +35,8 @@ import {
   EntityFacetsResponse,
   EntityFilter,
   EntityPagination,
+  PaginatedEntitiesRequest,
+  PaginatedEntitiesResponse,
 } from '../catalog/types';
 import {
   DbFinalEntitiesRow,
@@ -43,8 +46,17 @@ import {
   DbRelationsRow,
   DbSearchRow,
 } from '../database/tables';
+
 import { Stitcher } from '../stitching/Stitcher';
 
+import {
+  isPaginatedEntitiesCursorRequest,
+  isPaginatedEntitiesInitialRequest,
+} from './util';
+
+const defaultSortField = 'metadata.name';
+const defaultSortFieldOrder = 'asc';
+
 function parsePagination(input?: EntityPagination): {
   limit?: number;
   offset?: number;
@@ -97,7 +109,7 @@ function addCondition(
   // make a lot of sense. However, it had abysmal performance on sqlite
   // when datasets grew large, so we're using IN instead.
   const matchQuery = db<DbSearchRow>('search')
-    .select('search.entity_id')
+    .select(entityIdField)
     .where({ key: filter.key.toLowerCase() })
     .andWhere(function keyFilter() {
       if (filter.values) {
@@ -310,6 +322,141 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
     return { items };
   }
 
+  async paginatedEntities(
+    request?: PaginatedEntitiesRequest,
+  ): Promise<PaginatedEntitiesResponse> {
+    const db = this.database;
+    const limit = request?.limit ?? 20;
+
+    const cursor: Omit<Cursor, 'sortFieldId'> & { sortFieldId?: string } = {
+      firstFieldId: '',
+      sortField: defaultSortField,
+      sortFieldOrder: defaultSortFieldOrder,
+      isPrevious: false,
+      ...parseCursorFromRequest(request),
+    };
+
+    const allowedSortFields = ['metadata.name', 'metadata.uid'];
+    if (!allowedSortFields.includes(cursor.sortField)) {
+      throw new InputError(
+        `Invalid sortField. Allowed values are ${allowedSortFields.join(
+          ', ',
+        )}.`,
+      );
+    }
+
+    const isFetchingBackwards = cursor.isPrevious;
+
+    /**
+     * page number is used for quickly
+     * detecting the initial batch of items
+     * when navigating backwards without performing
+     * extra operations on the database.
+     */
+    /*
+    const currentPage = isFetchingBackwards
+      ? cursor.previousPage - 1
+      : cursor.previousPage + 1;
+      */
+
+    const dbQuery = db('search')
+      .join('final_entities', 'search.entity_id', 'final_entities.entity_id')
+      .where('key', cursor.sortField);
+
+    if (cursor.filter) {
+      parseFilter(cursor.filter, dbQuery, db, false, 'search.entity_id');
+    }
+
+    const normalizedQueryByName = cursor.query?.trim();
+    if (normalizedQueryByName) {
+      dbQuery.andWhereLike(
+        'value',
+        `%${normalizedQueryByName.toLocaleLowerCase('en-US')}%`,
+      );
+    }
+
+    const countQuery = dbQuery.clone();
+
+    const isOrderingDescending = cursor.sortFieldOrder === 'desc';
+    if (cursor.sortFieldId) {
+      dbQuery.andWhere(
+        'value',
+        isFetchingBackwards !== isOrderingDescending ? '<' : '>',
+        cursor.sortFieldId,
+      );
+    }
+
+    dbQuery
+      .orderBy(
+        'value',
+        isFetchingBackwards
+          ? invertOrder(cursor.sortFieldOrder)
+          : cursor.sortFieldOrder,
+      )
+      // fetch an extra item for
+      // checking if there are more items
+      .limit(isFetchingBackwards ? limit : limit + 1);
+
+    countQuery.count('search.entity_id', { as: 'count' });
+
+    const [rows, [{ count }]] = await Promise.all([
+      dbQuery,
+      // for performance reasons we invoke the countQuery
+      // only on the first request.
+      // The result is then embedded into the cursor
+      // for subsequent requests.
+      typeof cursor.totalItems === 'undefined'
+        ? countQuery
+        : [{ count: cursor.totalItems }],
+    ]);
+
+    const totalItems = Number(count);
+
+    if (isFetchingBackwards) {
+      rows.reverse();
+    }
+    const hasMoreResults =
+      limit > 0 && (isFetchingBackwards || rows.length > limit);
+
+    // discard the extra item only when fetching forward.
+    if (rows.length > limit) {
+      rows.length -= 1;
+    }
+
+    const isInitialRequest = cursor.firstFieldId === '';
+
+    const firstFieldId = cursor.firstFieldId || rows[0].value;
+
+    const nextCursor = hasMoreResults
+      ? encodeCursor({
+          ...cursor,
+          sortFieldId: rows[rows.length - 1].value,
+          firstFieldId,
+          isPrevious: false,
+          totalItems,
+        })
+      : undefined;
+
+    const prevCursor =
+      !isInitialRequest &&
+      rows.length > 0 &&
+      rows[0].value !== cursor.firstFieldId
+        ? encodeCursor({
+            ...cursor,
+            sortFieldId: rows[0].value,
+            firstFieldId: cursor.firstFieldId,
+            isPrevious: true,
+            totalItems,
+          })
+        : undefined;
+
+    const entities = rows
+      .map(e => JSON.parse(e.final_entity!))
+      .map(e => (request?.fields ? request.fields(e) : e));
+
+    return { entities, prevCursor, nextCursor, totalItems };
+  }
+
   async removeEntityByUid(uid: string): Promise<void> {
     const dbConfig = this.database.client.config;
 
@@ -490,3 +637,37 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
     return { facets };
   }
 }
+
+function encodeCursor(cursor: Cursor) {
+  const json = JSON.stringify(cursor);
+  return Buffer.from(json, 'utf8').toString('base64');
+}
+
+function parseCursorFromRequest(
+  request?: PaginatedEntitiesRequest,
+): Partial<Cursor> {
+  if (isPaginatedEntitiesInitialRequest(request)) {
+    const {
+      filter,
+      sortField = defaultSortField,
+      sortFieldOrder = defaultSortFieldOrder,
+      query,
+    } = request;
+    return { filter, sortField, sortFieldOrder, query };
+  }
+  if (isPaginatedEntitiesCursorRequest(request)) {
+    try {
+      const json = Buffer.from(request.cursor, 'base64').toString('utf8');
+      const cursor = JSON.parse(json);
+      // TODO(vinzscam): validate the shit
+      return cursor as unknown as Cursor;
+    } catch {
+      throw new InputError('Malformed cursor, could not be parsed');
+    }
+  }
+  return {};
+}
+
+function invertOrder(order: Cursor['sortFieldOrder']) {
+  return order === 'asc' ? 'desc' : 'asc';
+}

plugins/catalog-backend/src/service/createRouter.test.ts

@@ -52,6 +52,7 @@ describe('createRouter readonly disabled', () => {
       removeEntityByUid: jest.fn(),
       entityAncestry: jest.fn(),
       facets: jest.fn(),
+      paginatedEntities: jest.fn(),
     };
     locationService = {
       getLocation: jest.fn(),
@@ -135,6 +136,80 @@ describe('createRouter readonly disabled', () => {
     });
   });
 
+  describe('GET /v2beta1/entities', () => {
+    it('happy path: lists entities', async () => {
+      const entities: Entity[] = [
+        { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
+      ];
+
+      entitiesCatalog.paginatedEntities.mockResolvedValueOnce({
+        entities,
+        totalItems: 100,
+        nextCursor: 'something',
+      });
+
+      const response = await request(app).get('/v2beta1/entities');
+      expect(response.status).toEqual(200);
+      expect(response.body).toEqual({
+        entities,
+        totalItems: 100,
+        nextCursor: 'something',
+      });
+    });
+
+    it('parses initial request', async () => {
+      entitiesCatalog.paginatedEntities.mockResolvedValueOnce({
+        entities: [],
+        totalItems: 0,
+      });
+      const response = await request(app).get(
+        '/v2beta1/entities?filter=a=1,a=2,b=3&filter=c=4',
+      );
+
+      expect(response.status).toEqual(200);
+      expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledTimes(1);
+      expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledWith({
+        filter: {
+          anyOf: [
+            {
+              allOf: [
+                { key: 'a', values: ['1', '2'] },
+                { key: 'b', values: ['3'] },
+              ],
+            },
+            { allOf: [{ key: 'c', values: ['4'] }] },
+          ],
+        },
+      });
+    });
+
+    it('parses cursor request', async () => {
+      const entities: Entity[] = [
+        { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
+      ];
+
+      entitiesCatalog.paginatedEntities.mockResolvedValueOnce({
+        entities,
+        totalItems: 100,
+        nextCursor: 'next',
+      });
+
+      const response = await request(app).get(
+        '/v2beta1/entities?cursor=something',
+      );
+      expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledTimes(1);
+      expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledWith({
+        cursor: 'something',
+      });
+      expect(response.status).toEqual(200);
+      expect(response.body).toEqual({
+        entities,
+        totalItems: 100,
+        nextCursor: 'next',
+      });
+    });
+  });
+
   describe('GET /entities/by-uid/:uid', () => {
     it('can fetch entity by uid', async () => {
       const entity: Entity = {

plugins/catalog-backend/src/service/createRouter.ts

@@ -39,6 +39,7 @@ import {
   parseEntityFilterParams,
   parseEntityPaginationParams,
   parseEntityTransformParams,
+  parsePaginatedEntitiesParams,
 } from './request';
 import { parseEntityFacetParams } from './request/parseEntityFacetParams';
 import { parseEntityOrderParams } from './request/parseEntityOrderParams';
@@ -130,6 +131,14 @@ export async function createRouter(
         // TODO(freben): encode the pageInfo in the response
         res.json(entities);
       })
+      .get('/v2beta1/entities', async (req, res) => {
+        const response = await entitiesCatalog.paginatedEntities({
+          ...parsePaginatedEntitiesParams(req.query),
+          authorizationToken: getBearerToken(req.header('authorization')),
+        });
+
+        res.json(response);
+      })
       .get('/entities/by-uid/:uid', async (req, res) => {
         const { uid } = req.params;
         const { entities } = await entitiesCatalog.entities({

plugins/catalog-backend/src/service/request/index.ts

@@ -19,3 +19,4 @@ export { basicEntityFilter } from './basicEntityFilter';
 export { parseEntityFilterParams } from './parseEntityFilterParams';
 export { parseEntityPaginationParams } from './parseEntityPaginationParams';
 export { parseEntityTransformParams } from './parseEntityTransformParams';
+export { parsePaginatedEntitiesParams } from './parsePaginatedEntitiesParams';

plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts

@@ -0,0 +1,132 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  PaginatedEntitiesCursorRequest,
+  PaginatedEntitiesInitialRequest,
+} from '../../catalog/types';
+import { parsePaginatedEntitiesParams } from './parsePaginatedEntitiesParams';
+
+describe('parsePaginatedEntitiesParams', () => {
+  describe('initial request', () => {
+    it('should parse all the defined params', () => {
+      const validRequest = {
+        authorizationToken: 'to_not_be_returned',
+        fields: ['kind'],
+        limit: '3',
+        filter: ['a=1', 'b=2'],
+        sortField: 'sortField',
+        sortFieldOrder: 'desc',
+        query: 'query',
+      };
+      const parsedObj = parsePaginatedEntitiesParams(
+        validRequest,
+      ) as PaginatedEntitiesInitialRequest;
+      expect(parsedObj.limit).toBe(3);
+      expect(parsedObj.fields).toBeDefined();
+      expect(parsedObj.sortField).toBe('sortField');
+      expect(parsedObj.sortFieldOrder).toBe('desc');
+      expect(parsedObj.filter).toBeDefined();
+      expect(parsedObj.query).toBe('query');
+      expect(parsedObj).not.toHaveProperty('authorizationToken');
+      expect(parsedObj).not.toHaveProperty('cursor');
+    });
+    it('should ignore optional params', () => {
+      const parsedObj = parsePaginatedEntitiesParams(
+        {},
+      ) as PaginatedEntitiesInitialRequest;
+      expect(parsedObj.limit).toBeUndefined();
+      expect(parsedObj.fields).toBeUndefined();
+      expect(parsedObj.sortField).toBeUndefined();
+      expect(parsedObj.sortFieldOrder).toBeUndefined();
+      expect(parsedObj.filter).toBeUndefined();
+      expect(parsedObj.query).toBeUndefined();
+      expect(parsedObj).not.toHaveProperty('authorizationToken');
+      expect(parsedObj).not.toHaveProperty('cursor');
+    });
+
+    it.each([
+      {
+        limit: 'asd',
+      },
+      { filter: 3 },
+      { sortField: [] },
+      { sortFieldOrder: 'something' },
+      { fields: [4] },
+      { query: [] },
+    ])('should throw if some parameter is not valid %p', params => {
+      expect(() => parsePaginatedEntitiesParams(params)).toThrow();
+    });
+  });
+
+  describe('cursor request', () => {
+    it('should parse all the defined params', () => {
+      const validRequest = {
+        authorizationToken: 'to_not_be_returned',
+        fields: ['kind'],
+        limit: '3',
+        cursor: 'cursor',
+      };
+      const parsedObj = parsePaginatedEntitiesParams(
+        validRequest,
+      ) as PaginatedEntitiesCursorRequest;
+      expect(parsedObj.limit).toBe(3);
+      expect(parsedObj.fields).toBeDefined();
+      expect(parsedObj.cursor).toBe('cursor');
+    });
+
+    it('should ignore unknown params', () => {
+      const validRequest = {
+        authorizationToken: 'to_not_be_returned',
+        fields: ['kind'],
+        limit: '3',
+        cursor: 'cursor',
+        filter: ['a=1', 'b=2'],
+        sortField: 'sortField',
+        sortFieldOrder: 'desc',
+        query: 'query',
+      };
+      const parsedObj = parsePaginatedEntitiesParams(
+        validRequest,
+      ) as PaginatedEntitiesCursorRequest;
+      expect(parsedObj.limit).toBe(3);
+      expect(parsedObj.fields).toBeDefined();
+      expect(parsedObj.cursor).toBe('cursor');
+      expect(parsedObj).not.toHaveProperty('filter');
+      expect(parsedObj).not.toHaveProperty('sortField');
+      expect(parsedObj).not.toHaveProperty('sortFieldOrder');
+      expect(parsedObj).not.toHaveProperty('query');
+    });
+
+    it('should ignore optional params', () => {
+      const parsedObj = parsePaginatedEntitiesParams(
+        {},
+      ) as PaginatedEntitiesCursorRequest;
+      expect(parsedObj.limit).toBeUndefined();
+      expect(parsedObj.fields).toBeUndefined();
+    });
+
+    it.each([
+      {
+        limit: 'asd',
+      },
+      { cursor: [] },
+      { fields: [4] },
+    ])('should throw if some parameter is not valid %p', params => {
+      expect(() => parsePaginatedEntitiesParams(params)).toThrow();
+    });
+  });
+});

plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { InputError } from '@backstage/errors';
+import {
+  PaginatedEntitiesCursorRequest,
+  PaginatedEntitiesInitialRequest,
+  PaginatedEntitiesRequest,
+} from '../../catalog/types';
+import { parseIntegerParam, parseStringParam } from './common';
+import { parseEntityFilterParams } from './parseEntityFilterParams';
+import { parseEntityTransformParams } from './parseEntityTransformParams';
+
+export function parsePaginatedEntitiesParams(
+  params: Record<string, unknown>,
+): Omit<PaginatedEntitiesRequest, 'authorizationToken'> {
+  const fields = parseEntityTransformParams(params);
+  const limit = parseIntegerParam(params.limit, 'limit');
+  const cursor = parseStringParam(params.cursor, 'cursor');
+  if (cursor) {
+    const response: Omit<PaginatedEntitiesCursorRequest, 'authorizationToken'> =
+      {
+        cursor,
+        fields,
+        limit,
+      };
+    return response;
+  }
+
+  const filter = parseEntityFilterParams(params);
+  const query = parseStringParam(params.query, 'query');
+  const sortField = parseStringParam(params.sortField, 'sortField');
+  const sortFieldOrder = parseSortFieldOrder(
+    params.sortFieldOrder,
+    'sortFieldOrder',
+  );
+
+  const response: Omit<PaginatedEntitiesInitialRequest, 'authorizationToken'> =
+    {
+      fields,
+      filter,
+      limit,
+      sortField,
+      sortFieldOrder,
+      query,
+    };
+
+  return response;
+}
+
+function parseSortFieldOrder(sortFieldOrder: unknown, ctx: string) {
+  const isSortFieldOrder =
+    sortFieldOrder === undefined ||
+    sortFieldOrder === 'asc' ||
+    sortFieldOrder === 'desc';
+
+  if (isSortFieldOrder) {
+    return sortFieldOrder;
+  }
+  throw new InputError(`Invalid ${ctx}, not asc or desc`);
+}

plugins/catalog-backend/src/service/util.ts

@@ -18,6 +18,11 @@ import { InputError, NotAllowedError } from '@backstage/errors';
 import { Request } from 'express';
 import lodash from 'lodash';
 import { z } from 'zod';
+import {
+  EntitiesRequest,
+  PaginatedEntitiesCursorRequest,
+  PaginatedEntitiesInitialRequest,
+} from '../catalog/types';
 
 export async function requireRequestBody(req: Request): Promise<unknown> {
   const contentType = req.header('content-type');
@@ -65,3 +70,23 @@ export function disallowReadonlyMode(readonly: boolean) {
     throw new NotAllowedError('This operation not allowed in readonly mode');
   }
 }
+
+export function isPaginatedEntitiesInitialRequest(
+  input: EntitiesRequest | undefined,
+): input is PaginatedEntitiesInitialRequest {
+  if (!input) {
+    return false;
+  }
+  // TODO(vinzscam) expand this
+  return !isPaginatedEntitiesCursorRequest(input);
+}
+
+export function isPaginatedEntitiesCursorRequest(
+  input: EntitiesRequest | undefined,
+): input is PaginatedEntitiesCursorRequest {
+  if (!input) {
+    return false;
+  }
+  // TODO(vinzscam) expand this
+  return input?.hasOwnProperty('cursor') ?? false;
+}