feat(catalog-backend): support filtering on property existence

Signed-off-by: Phil Kuang <pkuang@factset.com>

.changeset/chilly-frogs-kiss.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Support filtering entities via property existence. For example you can now query with `/entities?filter=metadata.annotations.blah` to fetch all entities that has the particular property defined.

.changeset/five-dolls-complain.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-react': patch
+---
+
+Export `CATALOG_FILTER_EXISTS` symbol

.changeset/lucky-boats-rescue.md

@@ -0,0 +1,5 @@
+---
+'@backstage/catalog-client': patch
+---
+
+Support filtering entities via property existence

.changeset/selfish-phones-double.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-techdocs': patch
+---
+
+Optimize load times by only fetching entities with the `backstage.io/techdocs-ref` annotation

packages/catalog-client/api-report.md

@@ -25,6 +25,11 @@ export type AddLocationResponse = {
   entities: Entity[];
 };
 
+// Warning: (ae-missing-release-tag) "CATALOG_FILTER_EXISTS" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export const CATALOG_FILTER_EXISTS: unique symbol;
+
 // Warning: (ae-missing-release-tag) "CatalogApi" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -123,8 +128,8 @@ export class CatalogClient implements CatalogApi {
 // @public (undocumented)
 export type CatalogEntitiesRequest = {
   filter?:
-    | Record<string, string | string[]>[]
-    | Record<string, string | string[]>
+    | Record<string, string | symbol | (string | symbol)[]>[]
+    | Record<string, string | symbol | (string | symbol)[]>
     | undefined;
   fields?: string[] | undefined;
 };

packages/catalog-client/src/CatalogClient.test.ts

@@ -18,7 +18,7 @@ import { Entity } from '@backstage/catalog-model';
 import { rest } from 'msw';
 import { setupServer } from 'msw/node';
 import { CatalogClient } from './CatalogClient';
-import { CatalogListResponse } from './types/api';
+import { CATALOG_FILTER_EXISTS, CatalogListResponse } from './types/api';
 import { DiscoveryApi } from './types/discovery';
 
 const server = setupServer();
@@ -83,7 +83,7 @@ describe('CatalogClient', () => {
       server.use(
         rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => {
           expect(req.url.search).toBe(
-            '?filter=a=1,b=2,b=3,%C3%B6=%3D&filter=a=2',
+            '?filter=a=1,b=2,b=3,%C3%B6=%3D&filter=a=2&filter=c',
           );
           return res(ctx.json([]));
         }),
@@ -100,6 +100,9 @@ describe('CatalogClient', () => {
             {
               a: '2',
             },
+            {
+              c: CATALOG_FILTER_EXISTS,
+            },
           ],
         },
         { token },
@@ -113,7 +116,7 @@ describe('CatalogClient', () => {
 
       server.use(
         rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => {
-          expect(req.url.search).toBe('?filter=a=1,b=2,b=3,%C3%B6=%3D');
+          expect(req.url.search).toBe('?filter=a=1,b=2,b=3,%C3%B6=%3D,c');
           return res(ctx.json([]));
         }),
       );
@@ -124,6 +127,7 @@ describe('CatalogClient', () => {
             a: '1',
             b: ['2', '3'],
             ö: '=',
+            c: CATALOG_FILTER_EXISTS,
           },
         },
         { token },

packages/catalog-client/src/CatalogClient.ts

@@ -26,6 +26,7 @@ import {
 import { ResponseError } from '@backstage/errors';
 import fetch from 'cross-fetch';
 import {
+  CATALOG_FILTER_EXISTS,
   AddLocationRequest,
   AddLocationResponse,
   CatalogApi,
@@ -69,9 +70,13 @@ export class CatalogClient implements CatalogApi {
       const filterParts: string[] = [];
       for (const [key, value] of Object.entries(filterItem)) {
         for (const v of [value].flat()) {
-          filterParts.push(
-            `${encodeURIComponent(key)}=${encodeURIComponent(v)}`,
-          );
+          if (v === CATALOG_FILTER_EXISTS) {
+            filterParts.push(encodeURIComponent(key));
+          } else if (typeof v === 'string') {
+            filterParts.push(
+              `${encodeURIComponent(key)}=${encodeURIComponent(v)}`,
+            );
+          }
         }
       }
 

packages/catalog-client/src/types/api.ts

@@ -16,10 +16,12 @@
 
 import { Entity, EntityName, Location } from '@backstage/catalog-model';
 
+export const CATALOG_FILTER_EXISTS = Symbol();
+
 export type CatalogEntitiesRequest = {
   filter?:
-    | Record<string, string | string[]>[]
-    | Record<string, string | string[]>
+    | Record<string, string | symbol | (string | symbol)[]>[]
+    | Record<string, string | symbol | (string | symbol)[]>
     | undefined;
   fields?: string[] | undefined;
 };

packages/catalog-client/src/types/index.ts

@@ -22,4 +22,5 @@ export type {
   CatalogListResponse,
   CatalogRequestOptions,
 } from './api';
+export { CATALOG_FILTER_EXISTS } from './api';
 export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status';

plugins/catalog-backend/api-report.md

@@ -636,6 +636,7 @@ export type EntitiesCatalog = {
 export type EntitiesSearchFilter = {
   key: string;
   matchValueIn?: string[];
+  matchValueExists?: boolean;
 };
 
 // Warning: (ae-missing-release-tag) "entity" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -1077,16 +1078,16 @@ export class UrlReaderProcessor implements CatalogProcessor {
 // src/catalog/types.d.ts:44:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
 // src/catalog/types.d.ts:45:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
 // src/catalog/types.d.ts:75:5 - (ae-forgotten-export) The symbol "LocationUpdateLogEvent" needs to be exported by the entry point index.d.ts
-// src/database/types.d.ts:121:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:127:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:128:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:142:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:143:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:144:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/database/types.d.ts:125:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/database/types.d.ts:131:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/database/types.d.ts:132:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 // src/database/types.d.ts:146:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:159:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:160:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:161:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/database/types.d.ts:147:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/database/types.d.ts:148:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/database/types.d.ts:150:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/database/types.d.ts:163:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/database/types.d.ts:164:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/database/types.d.ts:165:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 // src/ingestion/processors/GithubMultiOrgReaderProcessor.d.ts:23:9 - (ae-forgotten-export) The symbol "GithubMultiOrgConfig" needs to be exported by the entry point index.d.ts
 // src/ingestion/processors/types.d.ts:7:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 // src/ingestion/processors/types.d.ts:8:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen

plugins/catalog-backend/src/database/CommonDatabase.test.ts

@@ -491,6 +491,98 @@ describe('CommonDatabase', () => {
         ]),
       );
     });
+
+    it('can get all specific entities for matching existence filters', async () => {
+      const entities: Entity[] = [
+        {
+          apiVersion: 'A',
+          kind: 'K1',
+          metadata: {
+            name: 'N',
+            annotations: {
+              foo: 'bar',
+            },
+          },
+          spec: { c: 'SOME' },
+        },
+        {
+          apiVersion: 'a',
+          kind: 'k2',
+          metadata: {
+            name: 'N',
+            annotations: {
+              foo: 'bar',
+            },
+          },
+          spec: { c: 'Some' },
+        },
+        {
+          apiVersion: 'a',
+          kind: 'k3',
+          metadata: { name: 'n' },
+          spec: { c: 'somE' },
+        },
+      ];
+
+      await db.transaction(async tx => {
+        await db.addEntities(
+          tx,
+          entities.map(entity => ({ entity, relations: [] })),
+        );
+      });
+
+      const existRows = await db.transaction(async tx =>
+        db.entities(tx, {
+          filter: {
+            anyOf: [
+              {
+                allOf: [
+                  { key: 'metadata.annotations.foo', matchValueExists: true },
+                ],
+              },
+            ],
+          },
+        }),
+      );
+
+      expect(existRows.entities.length).toEqual(2);
+      expect(existRows.entities).toEqual(
+        expect.arrayContaining([
+          {
+            locationId: undefined,
+            entity: expect.objectContaining({ kind: 'K1' }),
+          },
+          {
+            locationId: undefined,
+            entity: expect.objectContaining({ kind: 'k2' }),
+          },
+        ]),
+      );
+
+      const nonExistRows = await db.transaction(async tx =>
+        db.entities(tx, {
+          filter: {
+            anyOf: [
+              {
+                allOf: [
+                  { key: 'metadata.annotations.foo', matchValueExists: false },
+                ],
+              },
+            ],
+          },
+        }),
+      );
+
+      expect(nonExistRows.entities.length).toEqual(1);
+      expect(nonExistRows.entities).toEqual(
+        expect.arrayContaining([
+          {
+            locationId: undefined,
+            entity: expect.objectContaining({ kind: 'k3' }),
+          },
+        ]),
+      );
+    });
   });
 
   describe('setRelations', () => {

plugins/catalog-backend/src/database/CommonDatabase.ts

@@ -218,7 +218,11 @@ export class CommonDatabase implements Database {
 
     for (const singleFilter of request?.filter?.anyOf ?? []) {
       entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() {
-        for (const { key, matchValueIn } of singleFilter.allOf) {
+        for (const {
+          key,
+          matchValueIn,
+          matchValueExists,
+        } of singleFilter.allOf) {
           // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
           // make a lot of sense. However, it had abysmal performance on sqlite
           // when datasets grew large, so we're using IN instead.
@@ -226,7 +230,7 @@ export class CommonDatabase implements Database {
             .select('entity_id')
             .where(function keyFilter() {
               this.andWhere({ key: key.toLowerCase() });
-              if (matchValueIn) {
+              if (matchValueExists !== false && matchValueIn) {
                 if (matchValueIn.length === 1) {
                   this.andWhere({ value: matchValueIn[0].toLowerCase() });
                 } else if (matchValueIn.length > 1) {
@@ -238,7 +242,12 @@ export class CommonDatabase implements Database {
                 }
               }
             });
-          this.andWhere('id', 'in', matchQuery);
+          // Explicitly evaluate matchValueExists as a boolean since it may be undefined
+          this.andWhere(
+            'id',
+            matchValueExists === false ? 'not in' : 'in',
+            matchQuery,
+          );
         }
       });
     }

plugins/catalog-backend/src/database/types.ts

@@ -118,6 +118,11 @@ export type EntitiesSearchFilter = {
    * case insensitive.
    */
   matchValueIn?: string[];
+
+  /**
+   * Match on existence of key.
+   */
+  matchValueExists?: boolean;
 };
 
 /**

plugins/catalog-backend/src/next/NextEntitiesCatalog.ts

@@ -78,7 +78,11 @@ export class NextEntitiesCatalog implements EntitiesCatalog {
 
     for (const singleFilter of request?.filter?.anyOf ?? []) {
       entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() {
-        for (const { key, matchValueIn } of singleFilter.allOf) {
+        for (const {
+          key,
+          matchValueIn,
+          matchValueExists,
+        } of singleFilter.allOf) {
           // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
           // make a lot of sense. However, it had abysmal performance on sqlite
           // when datasets grew large, so we're using IN instead.
@@ -86,7 +90,7 @@ export class NextEntitiesCatalog implements EntitiesCatalog {
             .select('entity_id')
             .where(function keyFilter() {
               this.andWhere({ key: key.toLowerCase() });
-              if (matchValueIn) {
+              if (matchValueExists !== false && matchValueIn) {
                 if (matchValueIn.length === 1) {
                   this.andWhere({ value: matchValueIn[0].toLowerCase() });
                 } else if (matchValueIn.length > 1) {
@@ -98,7 +102,12 @@ export class NextEntitiesCatalog implements EntitiesCatalog {
                 }
               }
             });
-          this.andWhere('entity_id', 'in', matchQuery);
+          // Explicitly evaluate matchValueExists as a boolean since it may be undefined
+          this.andWhere(
+            'entity_id',
+            matchValueExists === false ? 'not in' : 'in',
+            matchQuery,
+          );
         }
       });
     }

plugins/catalog-backend/src/service/request/parseEntityFilterParams.test.ts

@@ -69,9 +69,11 @@ describe('parseEntityFilterParams', () => {
 describe('parseEntityFilterString', () => {
   it('works for the happy path', () => {
     expect(parseEntityFilterString('')).toBeUndefined();
-    expect(parseEntityFilterString('a=1,b=2,a=3')).toEqual([
+    expect(parseEntityFilterString('a=1,b=2,a=3,c,d=')).toEqual([
       { key: 'a', matchValueIn: ['1', '3'] },
       { key: 'b', matchValueIn: ['2'] },
+      { key: 'c', matchValueExists: true },
+      { key: 'd', matchValueIn: [''] },
     ]);
   });
 
@@ -83,17 +85,11 @@ describe('parseEntityFilterString', () => {
   });
 
   it('rejects malformed strings', () => {
-    expect(() => parseEntityFilterString('x=2,a=')).toThrow(
-      "Invalid filter, 'a=' is not a valid statement (expected a string on the form a=b)",
-    );
     expect(() => parseEntityFilterString('x=2,=a')).toThrow(
-      "Invalid filter, '=a' is not a valid statement (expected a string on the form a=b)",
+      "Invalid filter, '=a' is not a valid statement (expected a string on the form a=b or a= or a)",
     );
     expect(() => parseEntityFilterString('x=2,=')).toThrow(
-      "Invalid filter, '=' is not a valid statement (expected a string on the form a=b)",
-    );
-    expect(() => parseEntityFilterString('x=2,a')).toThrow(
-      "Invalid filter, 'a' is not a valid statement (expected a string on the form a=b)",
+      "Invalid filter, '=' is not a valid statement (expected a string on the form a=b or a= or a)",
     );
   });
 });

plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts

@@ -61,25 +61,26 @@ export function parseEntityFilterString(
 
   for (const statement of statements) {
     const equalsIndex = statement.indexOf('=');
-    if (equalsIndex < 1) {
-      throw new InputError(
-        `Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b)`,
-      );
-    }
 
-    const key = statement.substr(0, equalsIndex).trim();
-    const value = statement.substr(equalsIndex + 1).trim();
-    if (!key || !value) {
+    const key =
+      equalsIndex === -1 ? statement : statement.substr(0, equalsIndex).trim();
+    const value =
+      equalsIndex === -1 ? undefined : statement.substr(equalsIndex + 1).trim();
+    if (!key) {
       throw new InputError(
-        `Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b)`,
+        `Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b or a= or a)`,
       );
     }
 
     const f =
-      key in filtersByKey
-        ? filtersByKey[key]
-        : (filtersByKey[key] = { key, matchValueIn: [] });
-    f.matchValueIn!.push(value);
+      key in filtersByKey ? filtersByKey[key] : (filtersByKey[key] = { key });
+
+    if (value === undefined) {
+      f.matchValueExists = true;
+    } else {
+      f.matchValueIn = f.matchValueIn || [];
+      f.matchValueIn.push(value);
+    }
   }
 
   return Object.values(filtersByKey);

plugins/catalog-react/api-report.md

@@ -7,6 +7,7 @@
 
 import { ApiRef } from '@backstage/core-plugin-api';
 import { AsyncState } from 'react-use/lib/useAsync';
+import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client';
 import { CatalogApi } from '@backstage/catalog-client';
 import { ComponentEntity } from '@backstage/catalog-model';
 import { Context } from 'react';
@@ -22,6 +23,8 @@ import { SystemEntity } from '@backstage/catalog-model';
 import { TableColumn } from '@backstage/core-components';
 import { UserEntity } from '@backstage/catalog-model';
 
+export { CATALOG_FILTER_EXISTS };
+
 export { CatalogApi };
 
 // Warning: (ae-missing-release-tag) "catalogApiRef" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)

plugins/catalog-react/src/index.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 export type { CatalogApi } from '@backstage/catalog-client';
+export { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client';
 export { catalogApiRef } from './api';
 export * from './components';
 export * from './hooks';

plugins/techdocs/src/home/components/TechDocsCustomHome.tsx

@@ -19,6 +19,7 @@ import { useAsync } from 'react-use';
 import { makeStyles } from '@material-ui/core';
 import { CSSProperties } from '@material-ui/styles';
 import {
+  CATALOG_FILTER_EXISTS,
   catalogApiRef,
   CatalogApi,
   isOwnerOf,
@@ -125,6 +126,9 @@ export const TechDocsCustomHome = ({
 
   const { value: entities, loading, error } = useAsync(async () => {
     const response = await catalogApi.getEntities({
+      filter: {
+        'metadata.annotations.backstage.io/techdocs-ref': CATALOG_FILTER_EXISTS,
+      },
       fields: [
         'apiVersion',
         'kind',