paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 24 Packages Projects Releases Wiki Activity

Merge pull request #15142 from IlyaSavich/catalog-collator-refactoring

Browse Source 
Refactor catalog collator
This commit is contained in:
Fredrik Adelöw
2023-01-25 10:47:58 +01:00
committed by GitHub
parent 3cb84125f0 860b1c0902
commit a0871a3c0a
10 changed files with 343 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/clean-queens-judge.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': patch
---
The process of adding or modifying fields in the software-catalog search index has been simplified. For more details, see [how to customize fields in the Software Catalog index](../docs/features/search/how-to-guides.md#how-to-customize-fields-in-the-software-catalog-index).
docs/features/search/how-to-guides.md
+36
View File
@@ -114,6 +114,42 @@ of the `SearchType` component.
> Check out the documentation around [integrating search into plugins](../../plugins/integrating-search-into-plugins.md#create-a-collator) for how to create your own collator.
## How to customize fields in the Software Catalog index
Sometimes you will might want to have ability to control
which data passes to search index in catalog collator, or to customize data for specific kind.
You can easily do that by passing `entityTransformer` callback to `DefaultCatalogCollatorFactory`.
You can either just simply amend default behaviour, or even to write completely new document
(which should follow some required basic structure though).
> `authorization` and `location` cannot be modified via a `entityTransformer`, `location` can be modified only through `locationTemplate`.
```diff
// packages/backend/src/plugins/search.ts
const entityTransformer: CatalogCollatorEntityTransformer = (entity: Entity) => {
if (entity.kind === 'SomeKind') {
return {
// customize here output for 'SomeKind' kind
};
}
return {
// and customize default output
...defaultCatalogCollatorEntityTransformer(entity),
text: 'my super cool text',
};
};
indexBuilder.addCollator({
collator: DefaultCatalogCollatorFactory.fromConfig(env.config, {
discovery: env.discovery,
tokenManager: env.tokenManager,
+ entityTransformer,
}),
});
```
## How to limit what can be searched in the Software Catalog
The Software Catalog includes a wealth of information about the components,
plugins/catalog-backend/api-report.md
+10 -1
View File
@@ -167,6 +167,11 @@ export class CatalogBuilder {
useLegacySingleProcessorValidation(): this;
}
// @public (undocumented)
export type CatalogCollatorEntityTransformer = (
entity: Entity,
) => Omit<CatalogEntityDocument, 'location' | 'authorization'>;
// @alpha
export const catalogConditions: Conditions<{
hasAnnotation: PermissionRule<
@@ -350,6 +355,9 @@ export class DefaultCatalogCollator {
readonly visibilityPermission: Permission;
}
// @public (undocumented)
export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer;
// @public (undocumented)
export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
// (undocumented)
@@ -360,7 +368,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
// (undocumented)
getCollator(): Promise<Readable>;
// (undocumented)
readonly type: string;
readonly type = 'software-catalog';
// (undocumented)
readonly visibilityPermission: Permission;
}
@@ -373,6 +381,7 @@ export type DefaultCatalogCollatorFactoryOptions = {
filter?: GetEntitiesRequest['filter'];
batchSize?: number;
catalogClient?: CatalogApi;
entityTransformer?: CatalogCollatorEntityTransformer;
};
export { DeferredEntity };
plugins/catalog-backend/src/search/util.ts → plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts
+6 -12
View File
@@ -14,16 +14,10 @@
* limitations under the License.
*/
import { Entity, isUserEntity, isGroupEntity } from '@backstage/catalog-model';
import { Entity } from '@backstage/catalog-model';
import { CatalogEntityDocument } from '@backstage/plugin-catalog-common';
export function getDocumentText(entity: Entity): string {
const documentTexts: string[] = [];
documentTexts.push(entity.metadata.description || '');
if (isUserEntity(entity) || isGroupEntity(entity)) {
if (entity.spec?.profile?.displayName) {
documentTexts.push(entity.spec.profile.displayName);
}
}
return documentTexts.join(' : ');
}
/** @public */
export type CatalogCollatorEntityTransformer = (
entity: Entity,
) => Omit<CatalogEntityDocument, 'location' | 'authorization'>;
plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts
+61 -7
View File
@@ -95,11 +95,6 @@ describe('DefaultCatalogCollatorFactory', () => {
);
});
it('has expected type', () => {
const factory = DefaultCatalogCollatorFactory.fromConfig(config, options);
expect(factory.type).toBe('software-catalog');
});
describe('getCollator', () => {
let factory: DefaultCatalogCollatorFactory;
let collator: Readable;
@@ -124,24 +119,28 @@ describe('DefaultCatalogCollatorFactory', () => {
const pipeline = TestPipeline.fromCollator(collator);
const { documents } = await pipeline.execute();
expect(documents[0]).toMatchObject({
expect(documents[0]).toEqual({
title: expectedEntities[0].metadata.name,
location: '/catalog/default/component/test-entity',
text: expectedEntities[0].metadata.description,
namespace: 'default',
componentType: expectedEntities[0]!.spec!.type,
kind: expectedEntities[0]!.kind,
type: expectedEntities[0]!.spec!.type,
lifecycle: expectedEntities[0]!.spec!.lifecycle,
owner: expectedEntities[0]!.spec!.owner,
authorization: {
resourceRef: 'component:default/test-entity',
},
});
expect(documents[1]).toMatchObject({
expect(documents[1]).toEqual({
title: expectedEntities[1].metadata.title,
location: '/catalog/default/component/test-entity-2',
text: expectedEntities[1].metadata.description,
namespace: 'default',
componentType: expectedEntities[1]!.spec!.type,
kind: expectedEntities[1]!.kind,
type: expectedEntities[1]!.spec!.type,
lifecycle: expectedEntities[1]!.spec!.lifecycle,
owner: expectedEntities[1]!.spec!.owner,
authorization: {
@@ -150,6 +149,61 @@ describe('DefaultCatalogCollatorFactory', () => {
});
});
it('maps a returned entity to an expected CatalogEntityDocument with custom transformer', async () => {
const customFactory = DefaultCatalogCollatorFactory.fromConfig(config, {
...options,
entityTransformer: entity => ({
title: `custom-title-${
entity.metadata.title ?? entity.metadata.name
}`,
namespace: 'custom/namespace',
text: 'custom-text',
type: 'custom-type',
componentType: 'custom-component-type',
kind: 'custom-kind',
lifecycle: 'custom-lifecycle',
owner: 'custom-owner',
authorization: {
resourceRef: 'custom:resource/ref',
},
location: '/custom/location',
}),
});
const customCollator = await customFactory.getCollator();
const pipeline = TestPipeline.fromCollator(customCollator);
const { documents } = await pipeline.execute();
expect(documents[0]).toEqual({
title: 'custom-title-test-entity',
location: '/catalog/default/component/test-entity',
text: 'custom-text',
namespace: 'custom/namespace',
componentType: 'custom-component-type',
kind: 'custom-kind',
type: 'custom-type',
lifecycle: 'custom-lifecycle',
owner: 'custom-owner',
authorization: {
resourceRef: 'component:default/test-entity',
},
});
expect(documents[1]).toEqual({
title: 'custom-title-Test Entity',
location: '/catalog/default/component/test-entity-2',
text: 'custom-text',
namespace: 'custom/namespace',
componentType: 'custom-component-type',
kind: 'custom-kind',
type: 'custom-type',
lifecycle: 'custom-lifecycle',
owner: 'custom-owner',
authorization: {
resourceRef: 'component:default/test-entity-2',
},
});
});
it('maps a returned entity with a custom locationTemplate', async () => {
// Provide an alternate location template.
factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), {
plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts
+27 -26
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
import { Config } from '@backstage/config';
import {
PluginEndpointDiscovery,
TokenManager,
@@ -23,8 +24,6 @@ import {
CatalogClient,
GetEntitiesRequest,
} from '@backstage/catalog-client';
import { stringifyEntityRef } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
import {
catalogEntityReadPermission,
@@ -32,7 +31,9 @@ import {
} from '@backstage/plugin-catalog-common';
import { Permission } from '@backstage/plugin-permission-common';
import { Readable } from 'stream';
import { getDocumentText } from './util';
import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer';
import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer';
import { stringifyEntityRef } from '@backstage/catalog-model';
/** @public */
export type DefaultCatalogCollatorFactoryOptions = {
@@ -42,11 +43,12 @@ export type DefaultCatalogCollatorFactoryOptions = {
filter?: GetEntitiesRequest['filter'];
batchSize?: number;
catalogClient?: CatalogApi;
entityTransformer?: CatalogCollatorEntityTransformer;
};
/** @public */
export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
public readonly type: string = 'software-catalog';
public readonly type = 'software-catalog';
public readonly visibilityPermission: Permission =
catalogEntityReadPermission;
@@ -55,6 +57,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
private batchSize: number;
private readonly catalogClient: CatalogApi;
private tokenManager: TokenManager;
private entityTransformer: CatalogCollatorEntityTransformer;
static fromConfig(
_config: Config,
@@ -71,6 +74,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
filter,
catalogClient,
tokenManager,
entityTransformer,
} = options;
this.locationTemplate =
@@ -80,23 +84,14 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
this.catalogClient =
catalogClient || new CatalogClient({ discoveryApi: discovery });
this.tokenManager = tokenManager;
this.entityTransformer =
entityTransformer ?? defaultCatalogCollatorEntityTransformer;
}
async getCollator(): Promise<Readable> {
return Readable.from(this.execute());
}
private applyArgsToFormat(
format: string,
args: Record<string, string>,
): string {
let formatted = format;
for (const [key, value] of Object.entries(args)) {
formatted = formatted.replace(`:${key}`, value);
}
return formatted.toLowerCase();
}
private async *execute(): AsyncGenerator<CatalogEntityDocument> {
const { token } = await this.tokenManager.getToken();
let entitiesRetrieved = 0;
@@ -123,24 +118,30 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
for (const entity of entities) {
yield {
title: entity.metadata.title ?? entity.metadata.name,
...this.entityTransformer(entity),
authorization: {
resourceRef: stringifyEntityRef(entity),
},
location: this.applyArgsToFormat(this.locationTemplate, {
namespace: entity.metadata.namespace || 'default',
kind: entity.kind,
name: entity.metadata.name,
}),
text: getDocumentText(entity),
componentType: entity.spec?.type?.toString() || 'other',
type: entity.spec?.type?.toString() || 'other',
namespace: entity.metadata.namespace || 'default',
kind: entity.kind,
lifecycle: (entity.spec?.lifecycle as string) || '',
owner: (entity.spec?.owner as string) || '',
authorization: {
resourceRef: stringifyEntityRef(entity),
},
};
}
}
}
private applyArgsToFormat(
format: string,
args: Record<string, string>,
): string {
let formatted = format;
for (const [key, value] of Object.entries(args)) {
formatted = formatted.replace(`:${key}`, value);
}
return formatted.toLowerCase();
}
}
plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts
+150
View File
@@ -0,0 +1,150 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer';
const entity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
title: 'Test Entity',
name: 'test-entity',
description: 'The expected description',
namespace: 'namespace',
},
spec: {
type: 'some-type',
lifecycle: 'experimental',
owner: 'someone',
},
};
const userEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name: 'test-user-entity',
description: 'The expected user description',
},
spec: {
profile: {
displayName: 'User 1',
},
},
};
describe('DefaultCatalogCollatorEntityTransformer', () => {
describe('transform', () => {
it('maps a returned entity', async () => {
const document = defaultCatalogCollatorEntityTransformer(entity);
expect(document).toMatchObject({
title: entity.metadata.title,
text: entity.metadata.description,
namespace: entity.metadata.namespace,
componentType: entity.spec.type,
lifecycle: entity.spec.lifecycle,
owner: entity.spec.owner,
});
});
it('maps a returned entity with default fallback', async () => {
const entityWithoutTitle = {
...entity,
metadata: {
...entity.metadata,
title: undefined,
namespace: undefined,
},
spec: {
type: undefined,
lifecycle: undefined,
owner: undefined,
},
};
const document =
defaultCatalogCollatorEntityTransformer(entityWithoutTitle);
expect(document).toMatchObject({
title: entity.metadata.name,
text: entity.metadata.description,
namespace: 'default',
componentType: 'other',
lifecycle: '',
owner: '',
});
});
it('maps a returned user entity', async () => {
const document = defaultCatalogCollatorEntityTransformer(userEntity);
expect(document).toMatchObject({
title: userEntity.metadata.name,
text: `${userEntity.metadata.description} : ${userEntity.spec.profile.displayName}`,
namespace: 'default',
componentType: 'other',
lifecycle: '',
owner: '',
});
});
it('maps a returned user entity without display name', async () => {
const testEntity = {
...userEntity,
spec: undefined,
};
const document = defaultCatalogCollatorEntityTransformer(testEntity);
expect(document).toMatchObject({
title: userEntity.metadata.name,
text: userEntity.metadata.description,
namespace: 'default',
componentType: 'other',
lifecycle: '',
owner: '',
});
});
it('maps a returned group entity', async () => {
const groupEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
name: 'test-group-entity',
description: 'The expected group description',
},
spec: {
profile: {
displayName: 'Group 1',
},
},
};
const document = defaultCatalogCollatorEntityTransformer(groupEntity);
expect(document).toMatchObject({
title: groupEntity.metadata.name,
text: `${groupEntity.metadata.description} : ${groupEntity.spec.profile.displayName}`,
namespace: 'default',
componentType: 'other',
lifecycle: '',
owner: '',
});
});
});
});
plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts
+46
View File
@@ -0,0 +1,46 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, isGroupEntity, isUserEntity } from '@backstage/catalog-model';
import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer';
const getDocumentText = (entity: Entity): string => {
const documentTexts: string[] = [];
documentTexts.push(entity.metadata.description || '');
if (isUserEntity(entity) || isGroupEntity(entity)) {
if (entity.spec?.profile?.displayName) {
documentTexts.push(entity.spec.profile.displayName);
}
}
return documentTexts.join(' : ');
};
/** @public */
export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer =
(entity: Entity) => {
return {
title: entity.metadata.title ?? entity.metadata.name,
text: getDocumentText(entity),
componentType: entity.spec?.type?.toString() || 'other',
type: entity.spec?.type?.toString() || 'other',
namespace: entity.metadata.namespace || 'default',
kind: entity.kind,
lifecycle: (entity.spec?.lifecycle as string) || '',
owner: (entity.spec?.owner as string) || '',
};
};
plugins/catalog-backend/src/search/index.ts
+2
View File
@@ -16,6 +16,8 @@
export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory';
export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory';
export type { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer';
export type { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer';
/**
* todo(backstage/techdocs-core): stop exporting this in a future release.
plugins/catalog-backend/src/search/util.test.ts
-146
View File
@@ -1,146 +0,0 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
ComponentEntity,
GroupEntity,
UserEntity,
} from '@backstage/catalog-model';
import { getDocumentText } from './util';
describe('getDocumentText', () => {
describe('kind is not User or Group', () => {
test('contains description if set', () => {
const entity = createComponent();
entity.metadata.description = 'The expected description';
const actual = getDocumentText(entity);
expect(actual).toContain(entity.metadata.description);
});
test('is empty if description is not set', () => {
const entity = createComponent();
const actual = getDocumentText(entity);
expect(actual).toEqual('');
});
});
describe('kind is User', () => {
test('contains display name if set', () => {
const entity = createUser();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.spec.profile?.displayName);
});
test('contains description if set', () => {
const entity = createUser();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.metadata.description);
});
test('contains both description and display name if both are set', () => {
const entity = createUser();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.spec.profile?.displayName);
expect(actual).toContain(entity.metadata.description);
});
test('is empty if description and display name are not set', () => {
const entity = createUser();
delete entity.metadata.description;
delete entity.spec.profile?.displayName;
const actual = getDocumentText(entity);
expect(actual).toEqual('');
});
});
describe('kind is Group', () => {
test('contains display name if set', () => {
const entity = createGroup();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.spec.profile?.displayName);
});
test('contains description if set', () => {
const entity = createGroup();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.metadata.description);
});
test('contains both description and display name if both are set', () => {
const entity = createGroup();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.spec.profile?.displayName);
expect(actual).toContain(entity.metadata.description);
});
test('is empty if description and display name are not set', () => {
const entity = createGroup();
delete entity.metadata.description;
delete entity.spec.profile?.displayName;
const actual = getDocumentText(entity);
expect(actual).toEqual('');
});
});
});
function createGroup(): GroupEntity {
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
name: 'group-1',
description: 'The expected description',
},
spec: {
type: 'team',
profile: {
displayName: 'Group 1',
},
children: [],
},
};
}
function createUser(): UserEntity {
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name: 'user-1',
description: 'The expected description',
},
spec: {
profile: {
displayName: 'User 1',
},
},
};
}
function createComponent(): ComponentEntity {
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'component-1',
},
spec: {
lifecycle: 'experimental',
owner: 'someone',
type: 'service',
},
};
}