From d2ea0473bfe565292bb31f82e5980105ef55f60b Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Wed, 16 Feb 2022 16:21:08 +0700 Subject: [PATCH 1/9] Add filter parameter in DefaultTechDocsCollator to help limit scanning all entities Signed-off-by: Dede Hamzah --- .../src/search/DefaultTechDocsCollator.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts index 2c8786d92e..41196b77e7 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts @@ -31,7 +31,11 @@ import { Logger } from 'winston'; import pLimit from 'p-limit'; import { Config } from '@backstage/config'; import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; -import { CatalogApi, CatalogClient } from '@backstage/catalog-client'; +import { + CatalogApi, + CatalogClient, + GetEntitiesRequest, +} from '@backstage/catalog-client'; import { TechDocsDocument } from '@backstage/techdocs-common'; interface MkSearchIndexDoc { @@ -50,6 +54,7 @@ export type TechDocsCollatorOptions = { logger: Logger; tokenManager: TokenManager; locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; catalogClient?: CatalogApi; parallelismLimit?: number; legacyPathCasing?: boolean; @@ -88,6 +93,7 @@ export class DefaultTechDocsCollator implements DocumentCollator { parallelismLimit, discovery, tokenManager, + filter, catalogClient, locationTemplate, logger, @@ -99,6 +105,7 @@ export class DefaultTechDocsCollator implements DocumentCollator { catalogClient ?? new CatalogClient({ discoveryApi: discovery }) ).getEntities( { + filter, fields: [ 'kind', 'namespace', From bef351550fdba9b375703605d39dd67774ea36d6 Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Wed, 16 Feb 2022 16:45:31 +0700 Subject: [PATCH 2/9] generate api docs Signed-off-by: Dede Hamzah --- plugins/techdocs-backend/api-report.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index 5d106cfafb..7399ad9126 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -8,6 +8,7 @@ import { Config } from '@backstage/config'; import { DocumentCollator } from '@backstage/search-common'; import express from 'express'; import { GeneratorBuilder } from '@backstage/techdocs-common'; +import { GetEntitiesRequest } from '@backstage/catalog-client'; import { Knex } from 'knex'; import { Logger as Logger_2 } from 'winston'; import { Permission } from '@backstage/plugin-permission-common'; @@ -31,6 +32,8 @@ export class DefaultTechDocsCollator implements DocumentCollator { // (undocumented) execute(): Promise; // (undocumented) + protected filter?: GetEntitiesRequest['filter']; + // (undocumented) static fromConfig( config: Config, options: TechDocsCollatorOptions, @@ -73,6 +76,7 @@ export type TechDocsCollatorOptions = { logger: Logger_2; tokenManager: TokenManager; locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; catalogClient?: CatalogApi; parallelismLimit?: number; legacyPathCasing?: boolean; From 91eb01b5cfdbbd141da0692052f342cab9f54f87 Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Thu, 17 Feb 2022 09:15:00 +0700 Subject: [PATCH 3/9] add changeset Signed-off-by: Dede Hamzah --- .changeset/rotten-trees-mate.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/rotten-trees-mate.md diff --git a/.changeset/rotten-trees-mate.md b/.changeset/rotten-trees-mate.md new file mode 100644 index 0000000000..cf51780421 --- /dev/null +++ b/.changeset/rotten-trees-mate.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs-backend': patch +--- + +Provide optional filter parameter in DefaultTechDocsCollator to control where to scan the techdocs entities annotation. From d386ea658ad5e00698e969830a61f448418ea311 Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Thu, 17 Feb 2022 09:17:21 +0700 Subject: [PATCH 4/9] update changeset Signed-off-by: Dede Hamzah --- .changeset/rotten-trees-mate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/rotten-trees-mate.md b/.changeset/rotten-trees-mate.md index cf51780421..1e7d6abd9b 100644 --- a/.changeset/rotten-trees-mate.md +++ b/.changeset/rotten-trees-mate.md @@ -2,4 +2,4 @@ '@backstage/plugin-techdocs-backend': patch --- -Provide optional filter parameter in DefaultTechDocsCollator to control where to scan the techdocs entities annotation. +Provide optional filter parameter in DefaultTechDocsCollator to help limit scanning all entities. From 8e91f7a35996bff4bd093f355a7b364ec7c96f7e Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Wed, 23 Feb 2022 10:12:33 +0700 Subject: [PATCH 5/9] updating api-report Signed-off-by: Dede Hamzah --- plugins/techdocs-backend/api-report.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index 7399ad9126..ba218056e5 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -32,8 +32,6 @@ export class DefaultTechDocsCollator implements DocumentCollator { // (undocumented) execute(): Promise; // (undocumented) - protected filter?: GetEntitiesRequest['filter']; - // (undocumented) static fromConfig( config: Config, options: TechDocsCollatorOptions, From 66afe2e014196f66904271671f6b9801051576e4 Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Wed, 23 Feb 2022 10:40:27 +0700 Subject: [PATCH 6/9] Update how to guide search on limiting where techdocs collator is searching the documents Signed-off-by: Dede Hamzah --- docs/features/search/how-to-guides.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index a3c42c682c..51a71ee314 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -131,3 +131,26 @@ indexBuilder.addCollator({ As shown above, you can add a catalog entity filter to narrow down what catalog entities are indexed by the search engine. + +## How to limit where TechDocs Collator scanning the Software Catalog + +The DefaultTechDocsCollator is responsible for indexing the TechDocs into search. +The way it does is by getting all the entities every refresh interval from the +catalog and find the entities that has annotation `backstage.io/techdocs-ref`, +the process of getting all the entities is quite expensive if you have a large +amount of entity catalog data. To relieve the process, you can filter where the +DefaultTechDocsCollator is scanning your entities. + +```typescript +indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, + collator: DefaultTechDocsCollator.fromConfig(config, { + discovery, + logger, + tokenManager, ++ filter: { ++ kind: ['API', 'Component', 'Domain', 'System'], ++ }, + }), +}); +``` From d721f8a458fd03e9a860a3de16c7eb01c29664c4 Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Thu, 24 Feb 2022 10:02:27 +0700 Subject: [PATCH 7/9] Update to filter techdocs annotation directly in get entities Signed-off-by: Dede Hamzah --- plugins/techdocs-backend/api-report.md | 2 - .../src/search/DefaultTechDocsCollator.ts | 107 +++++++++--------- 2 files changed, 53 insertions(+), 56 deletions(-) diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index ba218056e5..5d106cfafb 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -8,7 +8,6 @@ import { Config } from '@backstage/config'; import { DocumentCollator } from '@backstage/search-common'; import express from 'express'; import { GeneratorBuilder } from '@backstage/techdocs-common'; -import { GetEntitiesRequest } from '@backstage/catalog-client'; import { Knex } from 'knex'; import { Logger as Logger_2 } from 'winston'; import { Permission } from '@backstage/plugin-permission-common'; @@ -74,7 +73,6 @@ export type TechDocsCollatorOptions = { logger: Logger_2; tokenManager: TokenManager; locationTemplate?: string; - filter?: GetEntitiesRequest['filter']; catalogClient?: CatalogApi; parallelismLimit?: number; legacyPathCasing?: boolean; diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts index 41196b77e7..73c030cc51 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts @@ -34,7 +34,7 @@ import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; import { CatalogApi, CatalogClient, - GetEntitiesRequest, + CATALOG_FILTER_EXISTS, } from '@backstage/catalog-client'; import { TechDocsDocument } from '@backstage/techdocs-common'; @@ -54,7 +54,6 @@ export type TechDocsCollatorOptions = { logger: Logger; tokenManager: TokenManager; locationTemplate?: string; - filter?: GetEntitiesRequest['filter']; catalogClient?: CatalogApi; parallelismLimit?: number; legacyPathCasing?: boolean; @@ -93,7 +92,6 @@ export class DefaultTechDocsCollator implements DocumentCollator { parallelismLimit, discovery, tokenManager, - filter, catalogClient, locationTemplate, logger, @@ -105,7 +103,10 @@ export class DefaultTechDocsCollator implements DocumentCollator { catalogClient ?? new CatalogClient({ discoveryApi: discovery }) ).getEntities( { - filter, + filter: { + 'metadata.annotations.backstage.io/techdocs-ref': + CATALOG_FILTER_EXISTS, + }, fields: [ 'kind', 'namespace', @@ -120,62 +121,60 @@ export class DefaultTechDocsCollator implements DocumentCollator { }, { token }, ); - const docPromises = entities.items - .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref']) - .map((entity: Entity) => - limit(async (): Promise => { - const entityInfo = DefaultTechDocsCollator.handleEntityInfoCasing( - this.legacyPathCasing ?? false, + const docPromises = entities.items.map((entity: Entity) => + limit(async (): Promise => { + const entityInfo = DefaultTechDocsCollator.handleEntityInfoCasing( + this.legacyPathCasing ?? false, + { + kind: entity.kind, + namespace: entity.metadata.namespace || 'default', + name: entity.metadata.name, + }, + ); + + try { + const searchIndexResponse = await fetch( + DefaultTechDocsCollator.constructDocsIndexUrl( + techDocsBaseUrl, + entityInfo, + ), { - kind: entity.kind, - namespace: entity.metadata.namespace || 'default', - name: entity.metadata.name, + headers: { + Authorization: `Bearer ${token}`, + }, }, ); + const searchIndex = await searchIndexResponse.json(); - try { - const searchIndexResponse = await fetch( - DefaultTechDocsCollator.constructDocsIndexUrl( - techDocsBaseUrl, - entityInfo, - ), + return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({ + title: unescape(doc.title), + text: unescape(doc.text || ''), + location: this.applyArgsToFormat( + locationTemplate || '/docs/:namespace/:kind/:name/:path', { - headers: { - Authorization: `Bearer ${token}`, - }, + ...entityInfo, + path: doc.location, }, - ); - const searchIndex = await searchIndexResponse.json(); - - return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({ - title: unescape(doc.title), - text: unescape(doc.text || ''), - location: this.applyArgsToFormat( - locationTemplate || '/docs/:namespace/:kind/:name/:path', - { - ...entityInfo, - path: doc.location, - }, - ), - path: doc.location, - ...entityInfo, - entityTitle: entity.metadata.title, - componentType: entity.spec?.type?.toString() || 'other', - lifecycle: (entity.spec?.lifecycle as string) || '', - owner: getSimpleEntityOwnerString(entity), - authorization: { - resourceRef: stringifyEntityRef(entity), - }, - })); - } catch (e) { - logger.debug( - `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`, - e, - ); - return []; - } - }), - ); + ), + path: doc.location, + ...entityInfo, + entityTitle: entity.metadata.title, + componentType: entity.spec?.type?.toString() || 'other', + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: getSimpleEntityOwnerString(entity), + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + })); + } catch (e) { + logger.debug( + `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`, + e, + ); + return []; + } + }), + ); return (await Promise.all(docPromises)).flat(); } From 3170026aeadd7899908a77561dd46c5d49e88a70 Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Thu, 24 Feb 2022 10:04:58 +0700 Subject: [PATCH 8/9] revert docs Signed-off-by: Dede Hamzah --- docs/features/search/how-to-guides.md | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index 51a71ee314..a3c42c682c 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -131,26 +131,3 @@ indexBuilder.addCollator({ As shown above, you can add a catalog entity filter to narrow down what catalog entities are indexed by the search engine. - -## How to limit where TechDocs Collator scanning the Software Catalog - -The DefaultTechDocsCollator is responsible for indexing the TechDocs into search. -The way it does is by getting all the entities every refresh interval from the -catalog and find the entities that has annotation `backstage.io/techdocs-ref`, -the process of getting all the entities is quite expensive if you have a large -amount of entity catalog data. To relieve the process, you can filter where the -DefaultTechDocsCollator is scanning your entities. - -```typescript -indexBuilder.addCollator({ - defaultRefreshIntervalSeconds: 600, - collator: DefaultTechDocsCollator.fromConfig(config, { - discovery, - logger, - tokenManager, -+ filter: { -+ kind: ['API', 'Component', 'Domain', 'System'], -+ }, - }), -}); -``` From d2d293d6b16c5b54c4e9902a9e5eb37e7c38c20e Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Thu, 24 Feb 2022 10:11:40 +0700 Subject: [PATCH 9/9] update changeset Signed-off-by: Dede Hamzah --- .changeset/rotten-trees-mate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/rotten-trees-mate.md b/.changeset/rotten-trees-mate.md index 1e7d6abd9b..f9d164d0fe 100644 --- a/.changeset/rotten-trees-mate.md +++ b/.changeset/rotten-trees-mate.md @@ -2,4 +2,4 @@ '@backstage/plugin-techdocs-backend': patch --- -Provide optional filter parameter in DefaultTechDocsCollator to help limit scanning all entities. +Optimize DefaultTechDocsCollator get entities.