Merge pull request #16921 from sennyeya/query-issues

fix(catalog-backend): Support full text searches across multiple fields

.changeset/orange-rabbits-jam.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Add full text search support to the `by-query` endpoint.

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts

@@ -1119,6 +1119,193 @@ describe('DefaultEntitiesCatalog', () => {
       },
     );
 
+    it.each(databases.eachSupportedId())(
+      'should filter the text results when sortOrder is not provided, %p',
+      async databaseId => {
+        await createDatabase(databaseId);
+
+        const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's'];
+        const entities: Entity[] = names.map((name, index) =>
+          // Need a stable search since default filtering is by uid, and those get generated on the fly
+          //  during the test case.
+          entityFrom(`${index}`, { uid: `id${index}`, title: name }),
+        );
+
+        const notFoundEntities: Entity[] = [
+          {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name: 'something' },
+            spec: {},
+          },
+          {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name: 'something else' },
+            spec: {},
+          },
+        ];
+
+        await Promise.all(
+          entities.concat(notFoundEntities).map(e => addEntityToSearch(e)),
+        );
+
+        const catalog = new DefaultEntitiesCatalog({
+          database: knex,
+          logger: getVoidLogger(),
+          stitcher,
+        });
+
+        const filter = {
+          key: 'spec.should_include_this',
+        };
+
+        const request: QueryEntitiesInitialRequest = {
+          filter,
+          limit: 100,
+          fullTextFilter: { term: 'cAt ', fields: ['metadata.title'] },
+        };
+        const response = await catalog.queryEntities(request);
+        expect(response.items).toEqual([
+          entityFrom('1', { uid: 'id1', title: 'cat' }),
+          entityFrom('2', { uid: 'id2', title: 'atcatss' }),
+          entityFrom('4', { uid: 'id4', title: 'dogcat' }),
+        ]);
+        expect(response.pageInfo.nextCursor).toBeUndefined();
+        expect(response.pageInfo.prevCursor).toBeUndefined();
+        expect(response.totalItems).toBe(3);
+
+        const paginatedResponse = await catalog.queryEntities({
+          ...request,
+          limit: 2,
+        });
+        expect(paginatedResponse.items).toEqual([
+          entityFrom('1', { uid: 'id1', title: 'cat' }),
+          entityFrom('2', { uid: 'id2', title: 'atcatss' }),
+        ]);
+        expect(paginatedResponse.pageInfo.nextCursor).not.toBeUndefined();
+        expect(paginatedResponse.pageInfo.prevCursor).toBeUndefined();
+        expect(paginatedResponse.totalItems).toBe(3);
+
+        const paginatedResponseNext = await catalog.queryEntities({
+          cursor: paginatedResponse.pageInfo.nextCursor!,
+        });
+        expect(paginatedResponseNext.items).toEqual([
+          entityFrom('4', { uid: 'id4', title: 'dogcat' }),
+        ]);
+        expect(paginatedResponseNext.pageInfo.nextCursor).toBeUndefined();
+        expect(paginatedResponseNext.pageInfo.prevCursor).not.toBeUndefined();
+        expect(paginatedResponseNext.totalItems).toBe(3);
+
+        const paginatedResponsePrev = await catalog.queryEntities({
+          cursor: paginatedResponseNext.pageInfo.prevCursor!,
+        });
+        expect(paginatedResponsePrev).toMatchObject(paginatedResponse);
+      },
+    );
+
+    it.each(databases.eachSupportedId())(
+      'should filter the text results by multiple search fields if provided, %p',
+      async databaseId => {
+        await createDatabase(databaseId);
+
+        const defs = [
+          {
+            title: 'lion',
+            name: 'KingOfTheJungle',
+          },
+          { title: 'cat', name: 'NotKingOfTheJungle' },
+          { title: 'atcatss', name: 'NotACatKing' },
+          { title: 'king', name: '123' },
+          { title: 'dogcat', name: 'dogcat' },
+          { title: 'aa', name: 'test123' },
+          { title: 's', name: 'idk' },
+        ];
+        const entities: Entity[] = defs.map(({ title, name }, index) =>
+          // Need a stable search since default filtering is by uid, and those get generated on the fly
+          //  during the test case.
+          entityFrom(name, { uid: `id${index}`, title }),
+        );
+
+        const notFoundEntities: Entity[] = [
+          {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name: 'something' },
+            spec: {},
+          },
+          {
+            apiVersion: 'a',
+            kind: 'k',
+            metadata: { name: 'something else' },
+            spec: {},
+          },
+        ];
+
+        await Promise.all(
+          entities.concat(notFoundEntities).map(e => addEntityToSearch(e)),
+        );
+
+        const catalog = new DefaultEntitiesCatalog({
+          database: knex,
+          logger: getVoidLogger(),
+          stitcher,
+        });
+
+        const filter = {
+          key: 'spec.should_include_this',
+        };
+
+        const request: QueryEntitiesInitialRequest = {
+          filter,
+          limit: 100,
+          fullTextFilter: {
+            term: 'KiNg ',
+            fields: ['metadata.title', 'metadata.name'],
+          },
+        };
+        const response = await catalog.queryEntities(request);
+
+        expect(response.items).toEqual([
+          entityFrom('KingOfTheJungle', { uid: 'id0', title: 'lion' }),
+          entityFrom('NotKingOfTheJungle', { uid: 'id1', title: 'cat' }),
+          entityFrom('NotACatKing', { uid: 'id2', title: 'atcatss' }),
+          entityFrom('123', { uid: 'id3', title: 'king' }),
+        ]);
+        expect(response.pageInfo.nextCursor).toBeUndefined();
+        expect(response.pageInfo.prevCursor).toBeUndefined();
+        expect(response.totalItems).toBe(4);
+
+        const paginatedResponse = await catalog.queryEntities({
+          ...request,
+          limit: 2,
+        });
+        expect(paginatedResponse.items).toEqual([
+          entityFrom('KingOfTheJungle', { uid: 'id0', title: 'lion' }),
+          entityFrom('NotKingOfTheJungle', { uid: 'id1', title: 'cat' }),
+        ]);
+        expect(paginatedResponse.pageInfo.nextCursor).not.toBeUndefined();
+        expect(paginatedResponse.pageInfo.prevCursor).toBeUndefined();
+        expect(paginatedResponse.totalItems).toBe(4);
+
+        const paginatedResponseNext = await catalog.queryEntities({
+          cursor: paginatedResponse.pageInfo.nextCursor!,
+        });
+        expect(paginatedResponseNext.items).toEqual([
+          entityFrom('NotACatKing', { uid: 'id2', title: 'atcatss' }),
+          entityFrom('123', { uid: 'id3', title: 'king' }),
+        ]);
+        expect(paginatedResponseNext.pageInfo.nextCursor).toBeUndefined();
+        expect(paginatedResponseNext.pageInfo.prevCursor).not.toBeUndefined();
+        expect(paginatedResponseNext.totalItems).toBe(4);
+
+        const paginatedResponsePrev = await catalog.queryEntities({
+          cursor: paginatedResponseNext.pageInfo.prevCursor!,
+        });
+        expect(paginatedResponsePrev).toMatchObject(paginatedResponse);
+      },
+    );
+
     it.each(databases.eachSupportedId())(
       'should include totalItems and empty entities in the response in case limit is zero, %p',
       async databaseId => {
@@ -1577,7 +1764,11 @@ describe('DefaultEntitiesCatalog', () => {
 
 function entityFrom(
   name: string,
-  { uid, namespace }: { uid?: string; namespace?: string } = {},
+  {
+    uid,
+    namespace,
+    title,
+  }: { uid?: string; namespace?: string; title?: string } = {},
 ) {
   return {
     apiVersion: 'a',
@@ -1586,6 +1777,7 @@ function entityFrom(
       name,
       ...(!!namespace && { namespace }),
       ...(!!uid && { uid }),
+      ...(!!title && { title }),
     },
     spec: { should_include_this: 'yes' },
   };

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts

@@ -382,11 +382,30 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
     }
 
     const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim();
+    const textFilterFields = cursor.fullTextFilter?.fields ?? [sortField.field];
     if (normalizedFullTextFilterTerm) {
-      dbQuery.andWhereRaw(
-        'value like ?',
-        `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`,
-      );
+      if (
+        textFilterFields.length === 1 &&
+        textFilterFields[0] === sortField.field
+      ) {
+        // If there is one item, apply the like query to the top level query which is already
+        //   filtered based on the singular sortField.
+        dbQuery.andWhereRaw(
+          'value like ?',
+          `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`,
+        );
+      } else {
+        const matchQuery = db<DbSearchRow>('search')
+          .select('search.entity_id')
+          .whereIn('key', textFilterFields)
+          .andWhere(function keyFilter() {
+            this.andWhereRaw(
+              'value like ?',
+              `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`,
+            );
+          });
+        dbQuery.andWhere('search.entity_id', 'in', matchQuery);
+      }
     }
 
     const countQuery = dbQuery.clone();