Merge pull request #20433 from gittesserakt/master

Refactor techdocs collator

.changeset/red-beers-roll.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-search-backend-module-techdocs': patch
+---
+
+The process of adding or modifying fields in the techdocs search index has been simplified. For more details, see [How to customize fields in the Software Catalog or TechDocs index](../docs/features/search/how-to-guides.md#how-to-customize-fields-in-the-software-catalog-or-techdocs-index).

docs/features/search/how-to-guides.md

@@ -114,18 +114,18 @@ of the `SearchType` component.
 
 > Check out the documentation around [integrating search into plugins](../../plugins/integrating-search-into-plugins.md#create-a-collator) for how to create your own collator.
 
-## How to customize fields in the Software Catalog index
+## How to customize fields in the Software Catalog or TechDocs index
 
-Sometimes you will might want to have ability to control
-which data passes to search index in catalog collator, or to customize data for specific kind.
-You can easily do that by passing `entityTransformer` callback to `DefaultCatalogCollatorFactory`.
-You can either just simply amend default behaviour, or even to write completely new document
-(which should follow some required basic structure though).
+Sometimes, you might want to have the ability to control which data passes into the search index
+in the catalog collator or customize data for a specific kind. You can easily achieve this
+by passing an `entityTransformer` callback to the `DefaultCatalogCollatorFactory`. This behavior
+is also possible for the `DefaultTechDocsCollatorFactory`. You can either simply amend the default behavior
+or even write an entirely new document (which should still follow some required basic structure).
 
 > `authorization` and `location` cannot be modified via a `entityTransformer`, `location` can be modified only through `locationTemplate`.
 
 ```ts title="packages/backend/src/plugins/search.ts"
-const entityTransformer: CatalogCollatorEntityTransformer = (
+const catalogEntityTransformer: CatalogCollatorEntityTransformer = (
   entity: Entity,
 ) => {
   if (entity.kind === 'SomeKind') {
@@ -146,7 +146,26 @@ indexBuilder.addCollator({
     discovery: env.discovery,
     tokenManager: env.tokenManager,
     /* highlight-add-next-line */
-    entityTransformer,
+    entityTransformer: catalogEntityTransformer,
+  }),
+});
+
+const techDocsEntityTransformer: TechDocsCollatorEntityTransformer = (
+  entity: Entity,
+) => {
+  return {
+    // add more fields to the index
+    ...defaultTechDocsCollatorEntityTransformer(entity),
+    tags: entity.metadata.tags,
+  };
+};
+
+indexBuilder.addCollator({
+  collator: DefaultTechDocsCollatorFactory.fromConfig(env.config, {
+    discovery: env.discovery,
+    tokenManager: env.tokenManager,
+    /* highlight-add-next-line */
+    entityTransformer: techDocsEntityTransformer,
   }),
 });
 ```

plugins/search-backend-module-techdocs/alpha-api-report.md

@@ -4,10 +4,21 @@
 
 ```ts
 import { BackendFeature } from '@backstage/backend-plugin-api';
+import { ExtensionPoint } from '@backstage/backend-plugin-api';
+import { TechDocsCollatorEntityTransformer } from '@backstage/plugin-search-backend-module-techdocs';
 
 // @alpha
 const _default: () => BackendFeature;
 export default _default;
 
+// @alpha (undocumented)
+export interface TechDocsCollatorEntityTransformerExtensionPoint {
+  // (undocumented)
+  setTransformer(transformer: TechDocsCollatorEntityTransformer): void;
+}
+
+// @alpha
+export const techdocsCollatorEntityTransformerExtensionPoint: ExtensionPoint<TechDocsCollatorEntityTransformerExtensionPoint>;
+
 // (No @packageDocumentation comment for this package)
 ```

plugins/search-backend-module-techdocs/api-report.md

@@ -8,12 +8,17 @@
 import { CatalogApi } from '@backstage/catalog-client';
 import { Config } from '@backstage/config';
 import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
+import { Entity } from '@backstage/catalog-model';
 import { Logger } from 'winston';
 import { Permission } from '@backstage/plugin-permission-common';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
 import { Readable } from 'stream';
+import { TechDocsDocument } from '@backstage/plugin-techdocs-node';
 import { TokenManager } from '@backstage/backend-common';
 
+// @public (undocumented)
+export const defaultTechDocsCollatorEntityTransformer: TechDocsCollatorEntityTransformer;
+
 // @public
 export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
   // (undocumented)
@@ -29,6 +34,11 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
   readonly visibilityPermission: Permission;
 }
 
+// @public (undocumented)
+export type TechDocsCollatorEntityTransformer = (
+  entity: Entity,
+) => Omit<TechDocsDocument, 'location' | 'authorization'>;
+
 // @public
 export type TechDocsCollatorFactoryOptions = {
   discovery: PluginEndpointDiscovery;
@@ -38,5 +48,6 @@ export type TechDocsCollatorFactoryOptions = {
   catalogClient?: CatalogApi;
   parallelismLimit?: number;
   legacyPathCasing?: boolean;
+  entityTransformer?: TechDocsCollatorEntityTransformer;
 };
 ```

plugins/search-backend-module-techdocs/src/alpha.ts

@@ -23,11 +23,28 @@ import { loggerToWinstonLogger } from '@backstage/backend-common';
 import {
   coreServices,
   createBackendModule,
+  createExtensionPoint,
 } from '@backstage/backend-plugin-api';
 import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks';
 import { catalogServiceRef } from '@backstage/plugin-catalog-node/alpha';
 import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-search-backend-module-techdocs';
 import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha';
+import { TechDocsCollatorEntityTransformer } from '@backstage/plugin-search-backend-module-techdocs';
+
+/** @alpha */
+export interface TechDocsCollatorEntityTransformerExtensionPoint {
+  setTransformer(transformer: TechDocsCollatorEntityTransformer): void;
+}
+
+/**
+ * Extension point used to customize the TechDocs collator entity transformer.
+ *
+ * @alpha
+ */
+export const techdocsCollatorEntityTransformerExtensionPoint =
+  createExtensionPoint<TechDocsCollatorEntityTransformerExtensionPoint>({
+    id: 'search.techdocsCollator.transformer',
+  });
 
 /**
  * @alpha
@@ -37,6 +54,22 @@ export default createBackendModule({
   moduleId: 'techDocsCollator',
   pluginId: 'search',
   register(env) {
+    let transformer: TechDocsCollatorEntityTransformer | undefined;
+
+    env.registerExtensionPoint(
+      techdocsCollatorEntityTransformerExtensionPoint,
+      {
+        setTransformer(newTransformer) {
+          if (transformer) {
+            throw new Error(
+              'TechDocs collator entity transformer may only be set once',
+            );
+          }
+          transformer = newTransformer;
+        },
+      },
+    );
+
     env.registerInit({
       deps: {
         config: coreServices.rootConfig,
@@ -75,6 +108,7 @@ export default createBackendModule({
             tokenManager,
             logger: loggerToWinstonLogger(logger),
             catalogClient: catalog,
+            entityTransformer: transformer,
           }),
         });
       },

plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 The Backstage Authors
+ * Copyright 2023 The Backstage Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,6 +26,8 @@ import { rest } from 'msw';
 import { setupServer } from 'msw/node';
 import { Readable } from 'stream';
 import { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory';
+import { defaultTechDocsCollatorEntityTransformer } from './defaultTechDocsCollatorEntityTransformer';
+import { TechDocsCollatorEntityTransformer } from './TechDocsCollatorEntityTransformer';
 
 const logger = getVoidLogger();
 
@@ -66,6 +68,7 @@ const expectedEntities: Entity[] = [
       annotations: {
         'backstage.io/techdocs-ref': './',
       },
+      tags: ['tag1', 'tag2'],
     },
     spec: {
       type: 'dog',
@@ -256,5 +259,42 @@ describe('DefaultTechDocsCollatorFactory', () => {
         });
       });
     });
+
+    it('should transform the entity using the entityTransformer function', async () => {
+      const entityTransformer: TechDocsCollatorEntityTransformer = (
+        entity: Entity,
+      ) => {
+        return {
+          ...defaultTechDocsCollatorEntityTransformer(entity),
+          tags: entity.metadata.tags,
+        };
+      };
+
+      factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
+        ...options,
+        entityTransformer,
+      });
+
+      collator = await factory.getCollator();
+
+      const pipeline = TestPipeline.fromCollator(collator);
+      const { documents } = await pipeline.execute();
+      const entity = expectedEntities[0];
+      documents.forEach((document, idx) => {
+        expect(document).toMatchObject({
+          title: mockSearchDocIndex.docs[idx].title,
+          location: `/docs/default/component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
+          text: mockSearchDocIndex.docs[idx].text,
+          namespace: 'default',
+          entityTitle: entity!.metadata.title,
+          componentType: entity!.spec!.type,
+          lifecycle: entity!.spec!.lifecycle,
+          owner: '',
+          kind: entity.kind.toLocaleLowerCase('en-US'),
+          name: entity.metadata.name,
+          tags: entity.metadata.tags,
+        });
+      });
+    });
   });
 });

plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 The Backstage Authors
+ * Copyright 2023 The Backstage Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -39,6 +39,8 @@ import fetch from 'node-fetch';
 import pLimit from 'p-limit';
 import { Readable } from 'stream';
 import { Logger } from 'winston';
+import { TechDocsCollatorEntityTransformer } from './TechDocsCollatorEntityTransformer';
+import { defaultTechDocsCollatorEntityTransformer } from './defaultTechDocsCollatorEntityTransformer';
 
 interface MkSearchIndexDoc {
   title: string;
@@ -59,6 +61,7 @@ export type TechDocsCollatorFactoryOptions = {
   catalogClient?: CatalogApi;
   parallelismLimit?: number;
   legacyPathCasing?: boolean;
+  entityTransformer?: TechDocsCollatorEntityTransformer;
 };
 
 type EntityInfo = {
@@ -85,6 +88,7 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
   private readonly tokenManager: TokenManager;
   private readonly parallelismLimit: number;
   private readonly legacyPathCasing: boolean;
+  private entityTransformer: TechDocsCollatorEntityTransformer;
 
   private constructor(options: TechDocsCollatorFactoryOptions) {
     this.discovery = options.discovery;
@@ -97,6 +101,8 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
     this.parallelismLimit = options.parallelismLimit ?? 10;
     this.legacyPathCasing = options.legacyPathCasing ?? false;
     this.tokenManager = options.tokenManager;
+    this.entityTransformer =
+      options.entityTransformer ?? defaultTechDocsCollatorEntityTransformer;
   }
 
   static fromConfig(config: Config, options: TechDocsCollatorFactoryOptions) {
@@ -142,17 +148,6 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
               'metadata.annotations.backstage.io/techdocs-ref':
                 CATALOG_FILTER_EXISTS,
             },
-            fields: [
-              'kind',
-              'namespace',
-              'metadata.annotations',
-              'metadata.name',
-              'metadata.title',
-              'metadata.namespace',
-              'spec.type',
-              'spec.lifecycle',
-              'relations',
-            ],
             limit: batchSize,
             offset: entitiesRetrieved,
           },
@@ -204,6 +199,7 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
               ]);
 
               return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
+                ...this.entityTransformer(entity),
                 title: unescape(doc.title),
                 text: unescape(doc.text || ''),
                 location: this.applyArgsToFormat(

plugins/search-backend-module-techdocs/src/collators/TechDocsCollatorEntityTransformer.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Entity } from '@backstage/catalog-model';
+import { TechDocsDocument } from '@backstage/plugin-techdocs-node';
+
+/** @public */
+export type TechDocsCollatorEntityTransformer = (
+  entity: Entity,
+) => Omit<TechDocsDocument, 'location' | 'authorization'>;

plugins/search-backend-module-techdocs/src/collators/defaultTechDocsCollatorEntityTransformer.test.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Entity } from '@backstage/catalog-model';
+import { defaultTechDocsCollatorEntityTransformer } from './defaultTechDocsCollatorEntityTransformer';
+
+describe('defaultTechDocsCollatorEntityTransformer', () => {
+  it('should transform the entity with the correct properties', () => {
+    const entity: Entity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      metadata: {
+        name: 'test-component',
+        description: 'A test component',
+        annotations: {
+          'backstage.io/techdocs-ref': 'docs',
+        },
+      },
+      spec: {
+        type: 'service',
+        owner: 'test@example.com',
+      },
+    };
+
+    const transformedEntity = defaultTechDocsCollatorEntityTransformer(entity);
+
+    expect(transformedEntity).toEqual({
+      kind: entity.kind,
+      namespace: entity.metadata.namespace || 'default',
+      annotations: entity.metadata.annotations || '',
+      name: entity.metadata.name || '',
+      title: entity.metadata.title || '',
+      text: 'A test component',
+      componentType: entity.spec?.type?.toString() || 'other',
+      type: entity.spec?.type?.toString() || 'other',
+      lifecycle: (entity.spec?.lifecycle as string) || '',
+      owner: (entity.spec?.owner as string) || '',
+      path: '',
+    });
+  });
+});

plugins/search-backend-module-techdocs/src/collators/defaultTechDocsCollatorEntityTransformer.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Entity, isGroupEntity, isUserEntity } from '@backstage/catalog-model';
+import { TechDocsCollatorEntityTransformer } from './TechDocsCollatorEntityTransformer';
+
+const getDocumentText = (entity: Entity): string => {
+  const documentTexts: string[] = [];
+  documentTexts.push(entity.metadata.description || '');
+
+  if (isUserEntity(entity) || isGroupEntity(entity)) {
+    if (entity.spec?.profile?.displayName) {
+      documentTexts.push(entity.spec.profile.displayName);
+    }
+  }
+
+  if (isUserEntity(entity)) {
+    if (entity.spec?.profile?.email) {
+      documentTexts.push(entity.spec.profile.email);
+    }
+  }
+
+  return documentTexts.join(' : ');
+};
+
+/** @public */
+export const defaultTechDocsCollatorEntityTransformer: TechDocsCollatorEntityTransformer =
+  (entity: Entity) => {
+    return {
+      kind: entity.kind,
+      namespace: entity.metadata.namespace || 'default',
+      annotations: entity.metadata.annotations || '',
+      name: entity.metadata.name || '',
+      title: entity.metadata.title || '',
+      text: getDocumentText(entity),
+      componentType: entity.spec?.type?.toString() || 'other',
+      type: entity.spec?.type?.toString() || 'other',
+      lifecycle: (entity.spec?.lifecycle as string) || '',
+      owner: (entity.spec?.owner as string) || '',
+      path: '',
+    };
+  };

plugins/search-backend-module-techdocs/src/collators/index.ts

@@ -17,3 +17,7 @@
 export { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory';
 
 export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFactory';
+
+export { defaultTechDocsCollatorEntityTransformer } from './defaultTechDocsCollatorEntityTransformer';
+
+export type { TechDocsCollatorEntityTransformer } from './TechDocsCollatorEntityTransformer';