Techdocs: Updates preparers to make it work in actual usecases (#1957)

* feature(techdocs): JIT generation of techdocs

* Fix linting issues

* Add Techdocs tab to entity page

* Added missing dep

* Added missing dep

* Removed duplicate dep

* Better tab navigation

* Update label on entity page to say docs

* Fix lint issue

* Move building of docs from entity to a function

* feature(techdocs): JIT generation of techdocs

* Fix linting issues

* Add Techdocs tab to entity page

* Added missing dep

* Added missing dep

* Removed duplicate dep

* Better tab navigation

* Update label on entity page to say docs

* Fix lint issue

* Move building of docs from entity to a function

* attempt to add back react as a dep

* Fixed failing test

* fix(techdocs): Hopefully fixed tests

* Fixed another failing test

* Initial apiRef for techdocs storage

* Cleaned up some code in reader

* test(headertabs): add tests to HeaderTabs component

* fix(headertabs): change tab mock id

* fix(techdocs-generator): fix problem with macOS symlink to tmp folder

* fix(lint): remove unused configApiRef

* wip

* Ongoing cleanups

* Cleanups and fix tests

* Initial github preparer. Doesn't work properly yet

* Updated dir preparer to handle github managed-by-location and added github preparer

* Removed old file

* Fixed feedback and added better logging to techdocs build process

* Updated create-app template

Co-authored-by: Fredrik Adelöw <freben@gmail.com>
Co-authored-by: Emma Indal <emmai@spotify.com>

plugins/techdocs-backend/examples/documented-component/documented-component.yaml

@@ -4,7 +4,7 @@ metadata:
   name: documented-component
   description: A Service with TechDocs documentation
   annotations:
-    backstage.io/techdocs-ref: 'dir:./'
+    backstage.io/techdocs-ref: 'github:https://github.com/spotify/backstage/blob/master/plugins/techdocs-backend/examples/documented-component'
 spec:
   type: service
   lifecycle: experimental

plugins/techdocs-backend/package.json

@@ -30,8 +30,10 @@
     "express": "^4.17.1",
     "express-promise-router": "^3.0.3",
     "fs-extra": "^9.0.1",
+    "git-url-parse": "^11.1.3",
     "knex": "^0.21.1",
     "node-fetch": "^2.6.0",
+    "nodegit": "^0.27.0",
     "winston": "^3.2.1"
   },
   "devDependencies": {

plugins/techdocs-backend/src/service/router.test.ts

@@ -24,12 +24,13 @@ import { createRouter } from './router';
 
 describe('createRouter', () => {
   let app: express.Express;
+  const logger = getVoidLogger();
 
   beforeAll(async () => {
     const router = await createRouter({
       preparers: new Preparers(),
       generators: new Generators(),
-      publisher: new LocalPublish(),
+      publisher: new LocalPublish(logger),
       logger: getVoidLogger(),
       dockerClient: new Docker(),
       config: ConfigReader.fromConfigs([]),

plugins/techdocs-backend/src/service/router.ts

@@ -45,18 +45,34 @@ export async function createRouter({
   publisher,
   config,
   dockerClient,
+  logger,
 }: RouterOptions): Promise<express.Router> {
   const router = Router();
 
+  const getEntityId = (entity: Entity) => {
+    return `${entity.kind}:${entity.metadata.namespace ?? ''}:${
+      entity.metadata.name
+    }`;
+  };
+
   const buildDocsForEntity = async (entity: Entity) => {
     const preparer = preparers.get(entity);
     const generator = generators.get(entity);
 
+    logger.info(`[TechDocs] Running preparer on entity ${getEntityId(entity)}`);
+    const preparedDir = await preparer.prepare(entity);
+
+    logger.info(
+      `[TechDocs] Running generator on entity ${getEntityId(entity)}`,
+    );
     const { resultDir } = await generator.run({
-      directory: await preparer.prepare(entity),
+      directory: preparedDir,
       dockerClient,
     });
 
+    logger.info(
+      `[TechDocs] Running publisher on entity ${getEntityId(entity)}`,
+    );
     await publisher.publish({
       entity,
       directory: resultDir,

plugins/techdocs-backend/src/service/standaloneServer.ts

@@ -41,14 +41,14 @@ export async function startStandaloneServer(
 
   logger.debug('Creating application...');
   const preparers = new Preparers();
-  const directoryPreparer = new DirectoryPreparer();
+  const directoryPreparer = new DirectoryPreparer(logger);
   preparers.register('dir', directoryPreparer);
 
   const generators = new Generators();
-  const techdocsGenerator = new TechdocsGenerator();
+  const techdocsGenerator = new TechdocsGenerator(logger);
   generators.register('techdocs', techdocsGenerator);
 
-  const publisher = new LocalPublish();
+  const publisher = new LocalPublish(logger);
 
   const dockerClient = new Docker();
 

plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts

@@ -13,17 +13,26 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import fs from 'fs-extra';
+import path from 'path';
+import os from 'os';
+import { Logger } from 'winston';
+
 import {
   GeneratorBase,
   GeneratorRunOptions,
   GeneratorRunResult,
 } from './types';
 import { runDockerContainer } from './helpers';
-import fs from 'fs-extra';
-import path from 'path';
-import os from 'os';
 
 export class TechdocsGenerator implements GeneratorBase {
+  private readonly logger: Logger;
+
+  constructor(logger: Logger) {
+    this.logger = logger;
+  }
+
   public async run({
     directory,
     logStream,
@@ -36,18 +45,23 @@ export class TechdocsGenerator implements GeneratorBase {
       path.join(tmpdirResolvedPath, 'techdocs-tmp-'),
     );
 
-    await runDockerContainer({
-      imageName: 'spotify/techdocs',
-      args: ['build', '-d', '/result'],
-      logStream,
-      docsDir: directory,
-      resultDir,
-      dockerClient,
-    });
-
-    console.log(
-      `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir}`,
-    );
+    try {
+      await runDockerContainer({
+        imageName: 'spotify/techdocs',
+        args: ['build', '-d', '/result'],
+        logStream,
+        docsDir: directory,
+        resultDir,
+        dockerClient,
+      });
+      this.logger.info(
+        `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir}`,
+      );
+    } catch (error) {
+      this.logger.debug(
+        `[TechDocs]: Failed to generate docs from ${directory} into ${resultDir}`,
+      );
+    }
 
     return { resultDir };
   }

plugins/techdocs-backend/src/techdocs/stages/prepare/dir.test.ts

@@ -14,6 +14,9 @@
  * limitations under the License.
  */
 import { DirectoryPreparer } from './dir';
+import { getVoidLogger } from '@backstage/backend-common';
+
+const logger = getVoidLogger();
 
 const createMockEntity = (annotations: {}) => {
   return {
@@ -30,7 +33,7 @@ const createMockEntity = (annotations: {}) => {
 
 describe('directory preparer', () => {
   it('should merge managed-by-location and techdocs-ref when techdocs-ref is relative', async () => {
-    const directoryPreparer = new DirectoryPreparer();
+    const directoryPreparer = new DirectoryPreparer(logger);
 
     const mockEntity = createMockEntity({
       'backstage.io/managed-by-location':
@@ -44,7 +47,7 @@ describe('directory preparer', () => {
   });
 
   it('should merge managed-by-location and techdocs-ref when techdocs-ref is absolute', async () => {
-    const directoryPreparer = new DirectoryPreparer();
+    const directoryPreparer = new DirectoryPreparer(logger);
 
     const mockEntity = createMockEntity({
       'backstage.io/managed-by-location':

plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts

@@ -13,26 +13,95 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import fs from 'fs-extra';
+import os from 'os';
 import { PreparerBase } from './types';
 import { Entity } from '@backstage/catalog-model';
 import path from 'path';
 import { parseReferenceAnnotation } from './helpers';
+import { InputError } from '@backstage/backend-common';
+import { Clone } from 'nodegit';
+import parseGitUrl from 'git-url-parse';
+import { Logger } from 'winston';
 
 export class DirectoryPreparer implements PreparerBase {
-  prepare(entity: Entity): Promise<string> {
-    const { location: managedByLocation } = parseReferenceAnnotation(
+  private readonly logger: Logger;
+
+  constructor(logger: Logger) {
+    this.logger = logger;
+  }
+
+  private async cloneGithubRepo(entity: Entity) {
+    const { type, target } = parseReferenceAnnotation(
       'backstage.io/managed-by-location',
       entity,
     );
-    const { location: techdocsLocation } = parseReferenceAnnotation(
+
+    if (type !== 'github') {
+      throw new InputError(`Wrong target type: ${type}, should be 'github'`);
+    }
+
+    const parsedGitLocation = parseGitUrl(target);
+    const repositoryTmpPath = path.join(
+      os.tmpdir(),
+      'backstage-repo',
+      parsedGitLocation.source,
+      parsedGitLocation.owner,
+      parsedGitLocation.name,
+      parsedGitLocation.ref,
+    );
+    if (fs.existsSync(repositoryTmpPath)) {
+      return repositoryTmpPath;
+    }
+    const repositoryCheckoutUrl = parsedGitLocation.toString('https');
+
+    this.logger.debug(
+      `[TechDocs] Checking out repository ${repositoryCheckoutUrl} to ${repositoryTmpPath}`,
+    );
+
+    fs.mkdirSync(repositoryTmpPath, { recursive: true });
+    await Clone.clone(repositoryCheckoutUrl, repositoryTmpPath, {});
+
+    return repositoryTmpPath;
+  }
+
+  private async resolveManagedByLocationToDir(entity: Entity) {
+    const { type, target } = parseReferenceAnnotation(
+      'backstage.io/managed-by-location',
+      entity,
+    );
+
+    this.logger.debug(
+      `[TechDocs] Building docs for entity with type 'dir' and managed-by-location '${type}'`,
+    );
+    switch (type) {
+      case 'github': {
+        const parsedGitLocation = parseGitUrl(target);
+        const repoLocation = await this.cloneGithubRepo(entity);
+
+        return path.dirname(
+          path.join(repoLocation, parsedGitLocation.filepath),
+        );
+      }
+      case 'file':
+        return path.dirname(target);
+      default:
+        throw new InputError(`Unable to resolve location type ${type}`);
+    }
+  }
+
+  async prepare(entity: Entity): Promise<string> {
+    const { target } = parseReferenceAnnotation(
       'backstage.io/techdocs-ref',
       entity,
     );
 
-    const managedByLocationDirectory = path.dirname(managedByLocation);
+    const managedByLocationDirectory = await this.resolveManagedByLocationToDir(
+      entity,
+    );
 
     return new Promise(resolve => {
-      resolve(path.resolve(managedByLocationDirectory, techdocsLocation));
+      resolve(path.resolve(managedByLocationDirectory, target));
     });
   }
 }

plugins/techdocs-backend/src/techdocs/stages/prepare/github.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs-extra';
+import path from 'path';
+import os from 'os';
+import { Entity } from '@backstage/catalog-model';
+import { InputError } from '@backstage/backend-common';
+import { PreparerBase } from './types';
+import parseGitUrl from 'git-url-parse';
+import { Clone } from 'nodegit';
+import { parseReferenceAnnotation } from './helpers';
+import { Logger } from 'winston';
+
+export class GithubPreparer implements PreparerBase {
+  private readonly logger: Logger;
+
+  constructor(logger: Logger) {
+    this.logger = logger;
+  }
+
+  async prepare(entity: Entity): Promise<string> {
+    const { type, target } = parseReferenceAnnotation(
+      'backstage.io/techdocs-ref',
+      entity,
+    );
+
+    if (type !== 'github') {
+      throw new InputError(`Wrong target type: ${type}, should be 'github'`);
+    }
+
+    const parsedGitLocation = parseGitUrl(target);
+    const repositoryTmpPath = path.join(
+      os.tmpdir(),
+      'backstage-repo',
+      parsedGitLocation.source,
+      parsedGitLocation.owner,
+      parsedGitLocation.name,
+      parsedGitLocation.ref,
+    );
+
+    if (fs.existsSync(repositoryTmpPath)) {
+      this.logger.debug(
+        `[TechDocs] Found repository already checked out at ${repositoryTmpPath}`,
+      );
+      return path.join(repositoryTmpPath, parsedGitLocation.filepath);
+    }
+    const repositoryCheckoutUrl = parsedGitLocation.toString('https');
+
+    this.logger.debug(
+      `[TechDocs] Checking out repository ${repositoryCheckoutUrl} to ${repositoryTmpPath}`,
+    );
+
+    fs.mkdirSync(repositoryTmpPath, { recursive: true });
+    await Clone.clone(repositoryCheckoutUrl, repositoryTmpPath, {});
+
+    return path.join(repositoryTmpPath, parsedGitLocation.filepath);
+  }
+}

plugins/techdocs-backend/src/techdocs/stages/prepare/helpers.ts

@@ -18,8 +18,8 @@ import { InputError } from '@backstage/backend-common';
 import { RemoteProtocol } from './types';
 
 export type ParsedLocationAnnotation = {
-  protocol: RemoteProtocol;
-  location: string;
+  type: RemoteProtocol;
+  target: string;
 };
 
 export const parseReferenceAnnotation = (
@@ -36,19 +36,19 @@ export const parseReferenceAnnotation = (
 
   // split on the first colon for the protocol and the rest after the first split
   // is the location.
-  const [protocol, location] = annotation.split(/:(.+)/) as [
+  const [type, target] = annotation.split(/:(.+)/) as [
     RemoteProtocol?,
     string?,
   ];
 
-  if (!protocol || !location) {
+  if (!type || !target) {
     throw new InputError(
       `Failure to parse either protocol or location for entity: ${entity.metadata.name}`,
     );
   }
 
   return {
-    protocol,
-    location,
+    type,
+    target,
   };
 };

plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts

@@ -14,5 +14,6 @@
  * limitations under the License.
  */
 export { DirectoryPreparer } from './dir';
+export { GithubPreparer } from './github';
 export { Preparers } from './preparers';
 export type { PreparerBuilder } from './types';

plugins/techdocs-backend/src/techdocs/stages/prepare/preparers.ts

@@ -26,13 +26,16 @@ export class Preparers implements PreparerBuilder {
   }
 
   get(entity: Entity): PreparerBase {
-    const { protocol } = parseReferenceAnnotation('backstage.io/techdocs-ref', entity);
-    const preparer = this.preparerMap.get(protocol);
+    const { type } = parseReferenceAnnotation(
+      'backstage.io/techdocs-ref',
+      entity,
+    );
+    const preparer = this.preparerMap.get(type);
 
     if (!preparer) {
-      throw new Error(`No preparer registered for type: "${protocol}"`);
+      throw new Error(`No preparer registered for type: "${type}"`);
     }
 
     return preparer;
   }
-}
+}

plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts

@@ -13,9 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { LocalPublish } from './local';
 import fs from 'fs-extra';
 import path from 'path';
+import { getVoidLogger } from '@backstage/backend-common';
+import { LocalPublish } from './local';
 
 const createMockEntity = (annotations = {}) => {
   return {
@@ -30,9 +31,11 @@ const createMockEntity = (annotations = {}) => {
   };
 };
 
+const logger = getVoidLogger();
+
 describe('local publisher', () => {
   it('should publish generated documentation dir', async () => {
-    const publisher = new LocalPublish();
+    const publisher = new LocalPublish(logger);
 
     const mockEntity = createMockEntity();
 

plugins/techdocs-backend/src/techdocs/stages/publish/local.ts

@@ -13,12 +13,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { PublisherBase } from './types';
-import { Entity } from '@backstage/catalog-model';
-import path from 'path';
 import fs from 'fs-extra';
+import path from 'path';
+import { Logger } from 'winston';
+import { Entity } from '@backstage/catalog-model';
+import { PublisherBase } from './types';
 
 export class LocalPublish implements PublisherBase {
+  private readonly logger: Logger;
+
+  constructor(logger: Logger) {
+    this.logger = logger;
+  }
+
   publish({
     entity,
     directory,
@@ -37,16 +44,22 @@ export class LocalPublish implements PublisherBase {
       '../../../../static/docs/',
       entity.kind,
       entityNamespace,
-      entity.metadata.name
+      entity.metadata.name,
     );
 
     if (!fs.existsSync(publishDir)) {
+      this.logger.info(
+        `[TechDocs]: Could not find ${publishDir}, creates the directory.`,
+      );
       fs.mkdirSync(publishDir, { recursive: true });
     }
 
     return new Promise((resolve, reject) => {
       fs.copy(directory, publishDir, err => {
         if (err) {
+          this.logger.debug(
+            `[TechDocs]: Failed to copy docs from ${directory} to ${publishDir}`,
+          );
           reject(err);
         }