Merge branch 'master' of github.com:Niksko/backstage into docs-build-strategy

plugins/techdocs-backend/CHANGELOG.md

@@ -1,5 +1,72 @@
 # @backstage/plugin-techdocs-backend
 
+## 0.14.0
+
+### Minor Changes
+
+- a925ba8385: BREAKING: constructor based initialization of DefaultTechDocsCollator now deprecated. Use static fromConfig method instead.
+
+  ```diff
+  indexBuilder.addCollator({
+    defaultRefreshIntervalSeconds: 600,
+  -   collator: new DefaultTechDocsCollator({
+  +   collator: DefaultTechDocsCollator.fromConfig(config, {
+      discovery,
+      logger,
+      tokenManager,
+    }),
+  });
+  ```
+
+  Note: in an upcoming release, TechDocs backend's /sync/:namespace/:kind/:name endpoint will only respond to text/event-stream-based requests. Update any custom code at your organization accordingly.
+
+### Patch Changes
+
+- 91eb01b5cf: Optimize DefaultTechDocsCollator get entities.
+- 919cf2f836: Minor updates to match the new `targetRef` field of relations, and to stop consuming the `target` field
+- Updated dependencies
+  - @backstage/backend-common@0.11.0
+  - @backstage/catalog-model@0.11.0
+  - @backstage/catalog-client@0.7.2
+  - @backstage/techdocs-common@0.11.10
+  - @backstage/integration@0.7.5
+
+## 0.13.5
+
+### Patch Changes
+
+- Fix for the previous release with missing type declarations.
+- Updated dependencies
+  - @backstage/backend-common@0.10.9
+  - @backstage/catalog-client@0.7.1
+  - @backstage/catalog-model@0.10.1
+  - @backstage/config@0.1.15
+  - @backstage/errors@0.2.2
+  - @backstage/integration@0.7.4
+  - @backstage/search-common@0.2.4
+  - @backstage/techdocs-common@0.11.9
+  - @backstage/plugin-catalog-common@0.1.4
+
+## 0.13.4
+
+### Patch Changes
+
+- 453145abba: Do not use cross-fetch in the backend
+- 1ed305728b: Bump `node-fetch` to version 2.6.7 and `cross-fetch` to version 3.1.5
+- c77c5c7eb6: Added `backstage.role` to `package.json`
+- 811c710a21: Fix bug where tech docs collator stores search indices with wrong entity ref casing. Make the collator to conform legacyPathCasing configuration option.
+- 7aeb491394: Replace use of deprecated `ENTITY_DEFAULT_NAMESPACE` constant with `DEFAULT_NAMESPACE`.
+- Updated dependencies
+  - @backstage/backend-common@0.10.8
+  - @backstage/catalog-client@0.7.0
+  - @backstage/errors@0.2.1
+  - @backstage/integration@0.7.3
+  - @backstage/catalog-model@0.10.0
+  - @backstage/config@0.1.14
+  - @backstage/search-common@0.2.3
+  - @backstage/techdocs-common@0.11.8
+  - @backstage/plugin-catalog-common@0.1.3
+
 ## 0.13.3
 
 ### Patch Changes

plugins/techdocs-backend/api-report.md

@@ -18,25 +18,17 @@ import { PublisherBase } from '@backstage/techdocs-common';
 import { TechDocsDocument } from '@backstage/techdocs-common';
 import { TokenManager } from '@backstage/backend-common';
 
-// Warning: (ae-missing-release-tag) "createRouter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public (undocumented)
+// @public
 export function createRouter(options: RouterOptions): Promise<express.Router>;
 
-// Warning: (ae-missing-release-tag) "DefaultTechDocsCollator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public (undocumented)
+// @public
 export class DefaultTechDocsCollator implements DocumentCollator {
-  // @deprecated
-  constructor(options: TechDocsCollatorOptions);
   // (undocumented)
   protected applyArgsToFormat(
     format: string,
     args: Record<string, string>,
   ): string;
   // (undocumented)
-  protected discovery: PluginEndpointDiscovery;
-  // (undocumented)
   execute(): Promise<TechDocsDocument[]>;
   // (undocumented)
   static fromConfig(
@@ -44,15 +36,11 @@ export class DefaultTechDocsCollator implements DocumentCollator {
     options: TechDocsCollatorOptions,
   ): DefaultTechDocsCollator;
   // (undocumented)
-  protected locationTemplate: string;
-  // (undocumented)
   readonly type: string;
   // (undocumented)
   readonly visibilityPermission: Permission;
 }
 
-// Warning: (ae-missing-release-tag) "OutOfTheBoxDeploymentOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
 // @public
 export type OutOfTheBoxDeploymentOptions = {
   preparers: PreparerBuilder;
@@ -65,8 +53,6 @@ export type OutOfTheBoxDeploymentOptions = {
   cache: PluginCacheManager;
 };
 
-// Warning: (ae-missing-release-tag) "RecommendedDeploymentOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
 // @public
 export type RecommendedDeploymentOptions = {
   publisher: PublisherBase;
@@ -76,16 +62,12 @@ export type RecommendedDeploymentOptions = {
   cache: PluginCacheManager;
 };
 
-// Warning: (ae-missing-release-tag) "RouterOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
 // @public
 export type RouterOptions =
   | RecommendedDeploymentOptions
   | OutOfTheBoxDeploymentOptions;
 
-// Warning: (ae-missing-release-tag) "TechDocsCollatorOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public (undocumented)
+// @public
 export type TechDocsCollatorOptions = {
   discovery: PluginEndpointDiscovery;
   logger: Logger_2;

plugins/techdocs-backend/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@backstage/plugin-techdocs-backend",
   "description": "The Backstage backend plugin that renders technical documentation for your components",
-  "version": "0.13.3",
+  "version": "0.14.0",
   "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
@@ -11,6 +11,9 @@
     "main": "dist/index.cjs.js",
     "types": "dist/index.d.ts"
   },
+  "backstage": {
+    "role": "backend-plugin"
+  },
   "homepage": "https://backstage.io",
   "repository": {
     "type": "git",
@@ -22,24 +25,24 @@
     "techdocs"
   ],
   "scripts": {
-    "start": "backstage-cli backend:dev",
-    "build": "backstage-cli backend:build",
-    "lint": "backstage-cli lint",
-    "test": "backstage-cli test",
-    "prepack": "backstage-cli prepack",
-    "postpack": "backstage-cli postpack",
-    "clean": "backstage-cli clean"
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "clean": "backstage-cli package clean"
   },
   "dependencies": {
-    "@backstage/backend-common": "^0.10.7",
-    "@backstage/catalog-client": "^0.6.0",
-    "@backstage/catalog-model": "^0.9.10",
-    "@backstage/config": "^0.1.13",
-    "@backstage/errors": "^0.2.0",
-    "@backstage/integration": "^0.7.2",
-    "@backstage/plugin-catalog-common": "^0.1.2",
-    "@backstage/search-common": "^0.2.2",
-    "@backstage/techdocs-common": "^0.11.7",
+    "@backstage/backend-common": "^0.11.0",
+    "@backstage/catalog-client": "^0.7.2",
+    "@backstage/catalog-model": "^0.11.0",
+    "@backstage/config": "^0.1.15",
+    "@backstage/errors": "^0.2.2",
+    "@backstage/integration": "^0.7.5",
+    "@backstage/plugin-catalog-common": "^0.1.4",
+    "@backstage/search-common": "^0.2.4",
+    "@backstage/techdocs-common": "^0.11.10",
     "@types/express": "^4.17.6",
     "dockerode": "^3.3.1",
     "express": "^4.17.1",
@@ -52,8 +55,8 @@
     "winston": "^3.2.1"
   },
   "devDependencies": {
-    "@backstage/cli": "^0.13.2",
-    "@backstage/test-utils": "^0.2.4",
+    "@backstage/cli": "^0.14.1",
+    "@backstage/test-utils": "^0.2.6",
     "@types/dockerode": "^3.3.0",
     "msw": "^0.35.0",
     "supertest": "^6.1.3"

plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts

@@ -208,8 +208,13 @@ describe('DefaultTechDocsCollator', () => {
   });
 
   it('maps a returned entity with a custom locationTemplate', async () => {
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        legacyUseCaseSensitiveTripletPaths: true,
+      },
+    });
     // Provide an alternate location template.
-    collator = new DefaultTechDocsCollator({
+    collator = DefaultTechDocsCollator.fromConfig(mockConfig, {
       discovery: mockDiscoveryApi,
       tokenManager: mockTokenManager,
       locationTemplate: '/software/:name',

plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts

@@ -20,6 +20,7 @@ import {
 } from '@backstage/backend-common';
 import {
   Entity,
+  parseEntityRef,
   RELATION_OWNED_BY,
   stringifyEntityRef,
 } from '@backstage/catalog-model';
@@ -30,7 +31,11 @@ import { Logger } from 'winston';
 import pLimit from 'p-limit';
 import { Config } from '@backstage/config';
 import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common';
-import { CatalogApi, CatalogClient } from '@backstage/catalog-client';
+import {
+  CatalogApi,
+  CatalogClient,
+  CATALOG_FILTER_EXISTS,
+} from '@backstage/catalog-client';
 import { TechDocsDocument } from '@backstage/techdocs-common';
 
 interface MkSearchIndexDoc {
@@ -39,6 +44,11 @@ interface MkSearchIndexDoc {
   location: string;
 }
 
+/**
+ * Options to configure the TechDocs collator
+ *
+ * @public
+ */
 export type TechDocsCollatorOptions = {
   discovery: PluginEndpointDiscovery;
   logger: Logger;
@@ -55,47 +65,48 @@ type EntityInfo = {
   kind: string;
 };
 
+/**
+ * A search collator responsible for gathering and transforming TechDocs documents.
+ *
+ * @public
+ */
 export class DefaultTechDocsCollator implements DocumentCollator {
-  protected discovery: PluginEndpointDiscovery;
-  protected locationTemplate: string;
-  private readonly logger: Logger;
-  private readonly catalogClient: CatalogApi;
-  private readonly tokenManager: TokenManager;
-  private readonly parallelismLimit: number;
-  private readonly legacyPathCasing: boolean;
   public readonly type: string = 'techdocs';
   public readonly visibilityPermission = catalogEntityReadPermission;
 
-  /**
-   * @deprecated use static fromConfig method instead.
-   */
-  constructor(options: TechDocsCollatorOptions) {
-    this.discovery = options.discovery;
-    this.locationTemplate =
-      options.locationTemplate || '/docs/:namespace/:kind/:name/:path';
-    this.logger = options.logger;
-    this.catalogClient =
-      options.catalogClient ||
-      new CatalogClient({ discoveryApi: options.discovery });
-    this.parallelismLimit = options.parallelismLimit ?? 10;
-    this.legacyPathCasing = options.legacyPathCasing ?? false;
-    this.tokenManager = options.tokenManager;
-  }
+  private constructor(
+    private readonly legacyPathCasing: boolean,
+    private readonly options: TechDocsCollatorOptions,
+  ) {}
 
   static fromConfig(config: Config, options: TechDocsCollatorOptions) {
     const legacyPathCasing =
       config.getOptionalBoolean(
         'techdocs.legacyUseCaseSensitiveTripletPaths',
       ) || false;
-    return new DefaultTechDocsCollator({ ...options, legacyPathCasing });
+    return new DefaultTechDocsCollator(legacyPathCasing, options);
   }
 
   async execute() {
-    const limit = pLimit(this.parallelismLimit);
-    const techDocsBaseUrl = await this.discovery.getBaseUrl('techdocs');
-    const { token } = await this.tokenManager.getToken();
-    const entities = await this.catalogClient.getEntities(
+    const {
+      parallelismLimit,
+      discovery,
+      tokenManager,
+      catalogClient,
+      locationTemplate,
+      logger,
+    } = this.options;
+    const limit = pLimit(parallelismLimit ?? 10);
+    const techDocsBaseUrl = await discovery.getBaseUrl('techdocs');
+    const { token } = await tokenManager.getToken();
+    const entities = await (
+      catalogClient ?? new CatalogClient({ discoveryApi: discovery })
+    ).getEntities(
       {
+        filter: {
+          'metadata.annotations.backstage.io/techdocs-ref':
+            CATALOG_FILTER_EXISTS,
+        },
         fields: [
           'kind',
           'namespace',
@@ -110,61 +121,60 @@ export class DefaultTechDocsCollator implements DocumentCollator {
       },
       { token },
     );
-    const docPromises = entities.items
-      .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref'])
-      .map((entity: Entity) =>
-        limit(async (): Promise<TechDocsDocument[]> => {
-          const entityInfo = DefaultTechDocsCollator.handleEntityInfoCasing(
-            this.legacyPathCasing,
+    const docPromises = entities.items.map((entity: Entity) =>
+      limit(async (): Promise<TechDocsDocument[]> => {
+        const entityInfo = DefaultTechDocsCollator.handleEntityInfoCasing(
+          this.legacyPathCasing ?? false,
+          {
+            kind: entity.kind,
+            namespace: entity.metadata.namespace || 'default',
+            name: entity.metadata.name,
+          },
+        );
+
+        try {
+          const searchIndexResponse = await fetch(
+            DefaultTechDocsCollator.constructDocsIndexUrl(
+              techDocsBaseUrl,
+              entityInfo,
+            ),
             {
-              kind: entity.kind,
-              namespace: entity.metadata.namespace || 'default',
-              name: entity.metadata.name,
+              headers: {
+                Authorization: `Bearer ${token}`,
+              },
             },
           );
+          const searchIndex = await searchIndexResponse.json();
 
-          try {
-            const searchIndexResponse = await fetch(
-              DefaultTechDocsCollator.constructDocsIndexUrl(
-                techDocsBaseUrl,
-                entityInfo,
-              ),
+          return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
+            title: unescape(doc.title),
+            text: unescape(doc.text || ''),
+            location: this.applyArgsToFormat(
+              locationTemplate || '/docs/:namespace/:kind/:name/:path',
               {
-                headers: {
-                  Authorization: `Bearer ${token}`,
-                },
-              },
-            );
-            const searchIndex = await searchIndexResponse.json();
-
-            return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
-              title: unescape(doc.title),
-              text: unescape(doc.text || ''),
-              location: this.applyArgsToFormat(this.locationTemplate, {
                 ...entityInfo,
                 path: doc.location,
-              }),
-              path: doc.location,
-              ...entityInfo,
-              entityTitle: entity.metadata.title,
-              componentType: entity.spec?.type?.toString() || 'other',
-              lifecycle: (entity.spec?.lifecycle as string) || '',
-              owner:
-                entity.relations?.find(r => r.type === RELATION_OWNED_BY)
-                  ?.target?.name || '',
-              authorization: {
-                resourceRef: stringifyEntityRef(entity),
               },
-            }));
-          } catch (e) {
-            this.logger.debug(
-              `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
-              e,
-            );
-            return [];
-          }
-        }),
-      );
+            ),
+            path: doc.location,
+            ...entityInfo,
+            entityTitle: entity.metadata.title,
+            componentType: entity.spec?.type?.toString() || 'other',
+            lifecycle: (entity.spec?.lifecycle as string) || '',
+            owner: getSimpleEntityOwnerString(entity),
+            authorization: {
+              resourceRef: stringifyEntityRef(entity),
+            },
+          }));
+        } catch (e) {
+          logger.debug(
+            `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
+            e,
+          );
+          return [];
+        }
+      }),
+    );
     return (await Promise.all(docPromises)).flat();
   }
 
@@ -197,3 +207,14 @@ export class DefaultTechDocsCollator implements DocumentCollator {
         }, {} as EntityInfo);
   }
 }
+
+function getSimpleEntityOwnerString(entity: Entity): string {
+  if (entity.relations) {
+    const owner = entity.relations.find(r => r.type === RELATION_OWNED_BY);
+    if (owner) {
+      const { name } = parseEntityRef(owner.targetRef);
+      return name;
+    }
+  }
+  return '';
+}

plugins/techdocs-backend/src/service/router.ts

@@ -41,8 +41,10 @@ import {
 } from './DocsBuildStrategy';
 
 /**
- * All of the required dependencies for running TechDocs in the "out-of-the-box"
+ * Required dependencies for running TechDocs in the "out-of-the-box"
  * deployment configuration (prepare/generate/publish all in the Backend).
+ *
+ * @public
  */
 export type OutOfTheBoxDeploymentOptions = {
   preparers: PreparerBuilder;
@@ -59,6 +61,8 @@ export type OutOfTheBoxDeploymentOptions = {
 /**
  * Required dependencies for running TechDocs in the "recommended" deployment
  * configuration (prepare/generate handled externally in CI/CD).
+ *
+ * @public
  */
 export type RecommendedDeploymentOptions = {
   publisher: PublisherBase;
@@ -71,6 +75,8 @@ export type RecommendedDeploymentOptions = {
 
 /**
  * One of the two deployment configurations must be provided.
+ *
+ * @public
  */
 export type RouterOptions =
   | RecommendedDeploymentOptions
@@ -79,6 +85,8 @@ export type RouterOptions =
 /**
  * Typeguard to help createRouter() understand when we are in a "recommended"
  * deployment vs. when we are in an out-of-the-box deployment configuration.
+ *
+ * * @public
  */
 function isOutOfTheBoxOption(
   opt: RouterOptions,
@@ -86,6 +94,11 @@ function isOutOfTheBoxOption(
   return (opt as OutOfTheBoxDeploymentOptions).preparers !== undefined;
 }
 
+/**
+ * Creates a techdocs router.
+ *
+ * @public
+ */
 export async function createRouter(
   options: RouterOptions,
 ): Promise<express.Router> {
@@ -332,12 +345,8 @@ export function createEventStream(
 }
 
 /**
- * Create a HTTP response. This is used for the legacy non-event-stream implementation of the sync endpoint.
- *
- * @param res - the response to write the event-stream to
- * @returns A tuple of <log, error, finish> callbacks to emit messages. A call to 'error' or 'finish'
- *          will close the event-stream.
- */
+ *  @deprecated use event-stream implementation of the sync endpoint
+ * */
 export function createHttpResponse(
   res: Response<any, any>,
 ): DocsSynchronizerSyncOpts {