Merge remote-tracking branch 'upstream/master' into feat/relative-ref-tmp

plugins/techdocs-backend/api-report.md

@@ -3,9 +3,12 @@
 > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
 
 ```ts
+import { CatalogApi } from '@backstage/catalog-client';
 import { Config } from '@backstage/config';
+import { DocumentCollator } from '@backstage/search-common';
 import express from 'express';
 import { GeneratorBuilder } from '@backstage/techdocs-common';
+import { IndexableDocument } from '@backstage/search-common';
 import { Knex } from 'knex';
 import { Logger as Logger_2 } from 'winston';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
@@ -18,6 +21,54 @@ import { PublisherBase } from '@backstage/techdocs-common';
 // @public (undocumented)
 export function createRouter(options: RouterOptions): Promise<express.Router>;
 
+// Warning: (ae-missing-release-tag) "DefaultTechDocsCollator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class DefaultTechDocsCollator implements DocumentCollator {
+  constructor({
+    discovery,
+    locationTemplate,
+    logger,
+    catalogClient,
+    parallelismLimit,
+  }: {
+    discovery: PluginEndpointDiscovery;
+    logger: Logger_2;
+    locationTemplate?: string;
+    catalogClient?: CatalogApi;
+    parallelismLimit?: number;
+  });
+  // (undocumented)
+  protected applyArgsToFormat(
+    format: string,
+    args: Record<string, string>,
+  ): string;
+  // (undocumented)
+  protected discovery: PluginEndpointDiscovery;
+  // (undocumented)
+  execute(): Promise<TechDocsDocument[]>;
+  // (undocumented)
+  protected locationTemplate: string;
+  // (undocumented)
+  readonly type: string;
+}
+
+// Warning: (ae-missing-release-tag) "TechDocsDocument" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export interface TechDocsDocument extends IndexableDocument {
+  // (undocumented)
+  kind: string;
+  // (undocumented)
+  lifecycle: string;
+  // (undocumented)
+  name: string;
+  // (undocumented)
+  namespace: string;
+  // (undocumented)
+  owner: string;
+}
+
 export * from '@backstage/techdocs-common';
 
 // (No @packageDocumentation comment for this package)

plugins/techdocs-backend/package.json

@@ -36,6 +36,7 @@
     "@backstage/config": "^0.1.5",
     "@backstage/errors": "^0.1.1",
     "@backstage/integration": "^0.5.8",
+    "@backstage/search-common": "^0.1.2",
     "@backstage/techdocs-common": "^0.6.8",
     "@types/express": "^4.17.6",
     "cross-fetch": "^3.0.6",
@@ -44,10 +45,13 @@
     "express-promise-router": "^4.1.0",
     "fs-extra": "9.1.0",
     "knex": "^0.95.1",
+    "lodash": "^4.17.21",
+    "p-limit": "^3.1.0",
     "winston": "^3.2.1"
   },
   "devDependencies": {
     "@backstage/cli": "^0.7.4",
+    "@backstage/test-utils": "^0.1.14",
     "@types/dockerode": "^3.2.1",
     "msw": "^0.29.0",
     "supertest": "^6.1.3"

plugins/techdocs-backend/src/index.ts

@@ -15,4 +15,5 @@
  */
 
 export { createRouter } from './service/router';
+export * from './search';
 export * from '@backstage/techdocs-common';

plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  PluginEndpointDiscovery,
+  getVoidLogger,
+} from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
+import { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
+import { msw } from '@backstage/test-utils';
+import { setupServer } from 'msw/node';
+import { rest } from 'msw';
+
+const logger = getVoidLogger();
+
+const mockSearchDocIndex = {
+  config: {
+    lang: ['en'],
+    min_search_length: 3,
+    prebuild_index: false,
+    separator: '[\\s\\-]+',
+  },
+  docs: [
+    {
+      location: '',
+      text: 'docs docs docs',
+      title: 'Home',
+    },
+    {
+      location: 'local-development/',
+      text: 'Docs for first subtitle',
+      title: 'Local development',
+    },
+    {
+      location: 'local-development/#development',
+      text: 'Docs for sub-subtitle',
+      title: 'Development',
+    },
+  ],
+};
+
+const expectedEntities: Entity[] = [
+  {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Component',
+    metadata: {
+      name: 'test-entity-with-docs',
+      description: 'Documented description',
+      annotations: {
+        'backstage.io/techdocs-ref': './',
+      },
+    },
+    spec: {
+      type: 'dog',
+      lifecycle: 'experimental',
+      owner: 'someone',
+    },
+  },
+  {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Component',
+    metadata: {
+      name: 'test-entity',
+      description: 'The expected description',
+    },
+    spec: {
+      type: 'some-type',
+      lifecycle: 'experimental',
+    },
+  },
+];
+
+describe('DefaultTechDocsCollator', () => {
+  let mockDiscoveryApi: jest.Mocked<PluginEndpointDiscovery>;
+  let collator: DefaultTechDocsCollator;
+
+  const worker = setupServer();
+  msw.setupDefaultHandlers(worker);
+  beforeEach(() => {
+    mockDiscoveryApi = {
+      getBaseUrl: jest.fn().mockResolvedValue('http://test-backend'),
+      getExternalBaseUrl: jest.fn(),
+    };
+    collator = new DefaultTechDocsCollator({
+      discovery: mockDiscoveryApi,
+      logger,
+    });
+
+    worker.use(
+      rest.get(
+        'http://test-backend/static/docs/default/Component/test-entity-with-docs/search/search_index.json',
+        (_, res, ctx) => res(ctx.status(200), ctx.json(mockSearchDocIndex)),
+      ),
+      rest.get('http://test-backend/entities', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json(expectedEntities)),
+      ),
+    );
+  });
+
+  it('fetches from the configured catalog and tech docs services', async () => {
+    const documents = await collator.execute();
+    expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog');
+    expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('techdocs');
+    expect(documents).toHaveLength(mockSearchDocIndex.docs.length);
+  });
+
+  it('should create documents for each tech docs search index', async () => {
+    const documents = await collator.execute();
+    const entity = expectedEntities[0];
+    documents.forEach((document, idx) => {
+      expect(document).toMatchObject({
+        title: mockSearchDocIndex.docs[idx].title,
+        location: `/docs/default/Component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
+        text: mockSearchDocIndex.docs[idx].text,
+        namespace: 'default',
+        componentType: entity!.spec!.type,
+        lifecycle: entity!.spec!.lifecycle,
+        owner: '',
+      });
+    });
+  });
+
+  it('maps a returned entity with a custom locationTemplate', async () => {
+    // Provide an alternate location template.
+    collator = new DefaultTechDocsCollator({
+      discovery: mockDiscoveryApi,
+      locationTemplate: '/software/:name',
+      logger,
+    });
+
+    const documents = await collator.execute();
+    expect(documents[0]).toMatchObject({
+      location: '/software/test-entity-with-docs',
+    });
+  });
+});

plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { Entity, RELATION_OWNED_BY } from '@backstage/catalog-model';
+import { IndexableDocument, DocumentCollator } from '@backstage/search-common';
+import fetch from 'cross-fetch';
+import unescape from 'lodash/unescape';
+import { Logger } from 'winston';
+import pLimit from 'p-limit';
+import { CatalogApi, CatalogClient } from '@backstage/catalog-client';
+
+interface MkSearchIndexDoc {
+  title: string;
+  text: string;
+  location: string;
+}
+
+export interface TechDocsDocument extends IndexableDocument {
+  kind: string;
+  namespace: string;
+  name: string;
+  lifecycle: string;
+  owner: string;
+}
+
+export class DefaultTechDocsCollator implements DocumentCollator {
+  protected discovery: PluginEndpointDiscovery;
+  protected locationTemplate: string;
+  private readonly logger: Logger;
+  private readonly catalogClient: CatalogApi;
+  private readonly parallelismLimit: number;
+  public readonly type: string = 'techdocs';
+
+  constructor({
+    discovery,
+    locationTemplate,
+    logger,
+    catalogClient,
+    parallelismLimit = 10,
+  }: {
+    discovery: PluginEndpointDiscovery;
+    logger: Logger;
+    locationTemplate?: string;
+    catalogClient?: CatalogApi;
+    parallelismLimit?: number;
+  }) {
+    this.discovery = discovery;
+    this.locationTemplate =
+      locationTemplate || '/docs/:namespace/:kind/:name/:path';
+    this.logger = logger;
+    this.catalogClient =
+      catalogClient || new CatalogClient({ discoveryApi: discovery });
+    this.parallelismLimit = parallelismLimit;
+  }
+
+  async execute() {
+    const limit = pLimit(this.parallelismLimit);
+    const techDocsBaseUrl = await this.discovery.getBaseUrl('techdocs');
+    const entities = await this.catalogClient.getEntities({
+      fields: [
+        'kind',
+        'namespace',
+        'metadata.annotations',
+        'metadata.name',
+        'metadata.namespace',
+        'spec.type',
+        'spec.lifecycle',
+        'relations',
+      ],
+    });
+    const docPromises = entities.items
+      .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref'])
+      .map((entity: Entity) =>
+        limit(
+          async (): Promise<TechDocsDocument[]> => {
+            const entityInfo = {
+              kind: entity.kind,
+              namespace: entity.metadata.namespace || 'default',
+              name: entity.metadata.name,
+            };
+
+            try {
+              const searchIndexResponse = await fetch(
+                DefaultTechDocsCollator.constructDocsIndexUrl(
+                  techDocsBaseUrl,
+                  entityInfo,
+                ),
+              );
+              const searchIndex = await searchIndexResponse.json();
+
+              return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
+                title: unescape(doc.title),
+                text: unescape(doc.text || ''),
+                location: this.applyArgsToFormat(this.locationTemplate, {
+                  ...entityInfo,
+                  path: doc.location,
+                }),
+                ...entityInfo,
+                componentType: entity.spec?.type?.toString() || 'other',
+                lifecycle: (entity.spec?.lifecycle as string) || '',
+                owner:
+                  entity.relations?.find(r => r.type === RELATION_OWNED_BY)
+                    ?.target?.name || '',
+              }));
+            } catch (e) {
+              this.logger.warn(
+                `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
+                e,
+              );
+              return [];
+            }
+          },
+        ),
+      );
+    return (await Promise.all(docPromises)).flat();
+  }
+
+  protected applyArgsToFormat(
+    format: string,
+    args: Record<string, string>,
+  ): string {
+    let formatted = format;
+    for (const [key, value] of Object.entries(args)) {
+      formatted = formatted.replace(`:${key}`, value);
+    }
+    return formatted;
+  }
+
+  private static constructDocsIndexUrl(
+    techDocsBaseUrl: string,
+    entityInfo: { kind: string; namespace: string; name: string },
+  ) {
+    return `${techDocsBaseUrl}/static/docs/${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}/search/search_index.json`;
+  }
+}

plugins/techdocs-backend/src/search/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
+export type { TechDocsDocument } from './DefaultTechDocsCollator';