TechDocs: Access entity techdocs from service catalog (#1835)

* feature(techdocs): JIT generation of techdocs

* Fix linting issues

* Add Techdocs tab to entity page

* Added missing dep

* Added missing dep

* Removed duplicate dep

* Better tab navigation

* Update label on entity page to say docs

* Fix lint issue

* Move building of docs from entity to a function

* feature(techdocs): JIT generation of techdocs

* Fix linting issues

* Add Techdocs tab to entity page

* Added missing dep

* Added missing dep

* Removed duplicate dep

* Better tab navigation

* Update label on entity page to say docs

* Fix lint issue

* Move building of docs from entity to a function

* attempt to add back react as a dep

* Fixed failing test

* fix(techdocs): Hopefully fixed tests

* Fixed another failing test

* Initial apiRef for techdocs storage

* Cleaned up some code in reader

* test(headertabs): add tests to HeaderTabs component

* fix(headertabs): change tab mock id

* fix(techdocs-generator): fix problem with macOS symlink to tmp folder

* fix(lint): remove unused configApiRef

* wip

* Ongoing cleanups

* Cleanups and fix tests

* WIP cleanups

* Clean up some things

* Clean up some things

* Added missing notice header

* ts issue fixed

* Add show contition to api tab

Co-authored-by: Fredrik Adelöw <freben@gmail.com>
Co-authored-by: Emma Indal <emmai@spotify.com>

plugins/techdocs-backend/src/service/router.ts

@@ -48,6 +48,21 @@ export async function createRouter({
 }: RouterOptions): Promise<express.Router> {
   const router = Router();
 
+  const buildDocsForEntity = async (entity: Entity) => {
+    const preparer = preparers.get(entity);
+    const generator = generators.get(entity);
+
+    const { resultDir } = await generator.run({
+      directory: await preparer.prepare(entity),
+      dockerClient,
+    });
+
+    await publisher.publish({
+      entity,
+      directory: resultDir,
+    });
+  };
+
   router.get('/', async (_, res) => {
     res.status(200).send('Hello TechDocs Backend');
   });
@@ -64,18 +79,7 @@ export async function createRouter({
     );
 
     entitiesWithDocs.forEach(async entity => {
-      const preparer = preparers.get(entity);
-      const generator = generators.get(entity);
-
-      const { resultDir } = await generator.run({
-        directory: await preparer.prepare(entity),
-        dockerClient,
-      });
-
-      publisher.publish({
-        entity,
-        directory: resultDir,
-      });
+      await buildDocsForEntity(entity);
     });
 
     res.send('Successfully generated documentation');
@@ -86,6 +90,23 @@ export async function createRouter({
       '/static/docs/',
       express.static(path.resolve(__dirname, `../../static/docs`)),
     );
+    router.use(
+      '/static/docs/:kind/:namespace/:name',
+      async (req, res, next) => {
+        const baseUrl = config.getString('backend.baseUrl');
+        const { kind, namespace, name } = req.params;
+
+        const entityResponse = await fetch(
+          `${baseUrl}/catalog/entities/by-name/${kind}/${namespace}/${name}`,
+        );
+        if (!entityResponse.ok) next();
+        const entity = (await entityResponse.json()) as Entity;
+
+        await buildDocsForEntity(entity);
+
+        res.redirect(req.originalUrl);
+      },
+    );
   }
 
   return router;

plugins/techdocs-backend/src/techdocs/stages/generate/helpers.ts

@@ -47,6 +47,16 @@ export async function runDockerContainer({
   dockerClient,
   createOptions,
 }: RunDockerContainerOptions) {
+  await new Promise((resolve, reject) => {
+    dockerClient.pull(imageName, {}, (err, stream) => {
+      if (err) return reject(err);
+      stream.pipe(logStream, { end: false });
+      stream.on('end', () => resolve());
+      stream.on('error', (error: Error) => reject(error));
+      return undefined;
+    });
+  });
+
   const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run(
     imageName,
     args,

plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts

@@ -19,7 +19,7 @@ import {
   GeneratorRunResult,
 } from './types';
 import { runDockerContainer } from './helpers';
-import fs from 'fs';
+import fs from 'fs-extra';
 import path from 'path';
 import os from 'os';
 
@@ -29,7 +29,12 @@ export class TechdocsGenerator implements GeneratorBase {
     logStream,
     dockerClient,
   }: GeneratorRunOptions): Promise<GeneratorRunResult> {
-    const resultDir = fs.mkdtempSync(path.join(os.tmpdir(), `techdocs-tmp-`));
+    const tmpdirPath = os.tmpdir();
+    // Fixes a problem with macOS returning a path that is a symlink
+    const tmpdirResolvedPath = fs.realpathSync(tmpdirPath);
+    const resultDir = fs.mkdtempSync(
+      path.join(tmpdirResolvedPath, 'techdocs-tmp-'),
+    );
 
     await runDockerContainer({
       imageName: 'spotify/techdocs',
@@ -43,6 +48,7 @@ export class TechdocsGenerator implements GeneratorBase {
     console.log(
       `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir}`,
     );
+
     return { resultDir };
   }
 }

plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts

@@ -48,8 +48,13 @@ describe('local publisher', () => {
       `../../../../static/docs/${mockEntity.metadata.name}`,
     );
 
-    expect(fs.existsSync(publishDir)).toBeTruthy();
-    expect(fs.existsSync(path.join(publishDir, '/mock-file'))).toBeTruthy();
+    const resultDir = path.resolve(
+      __dirname,
+      `../../../../static/docs/${mockEntity.kind}/default/${mockEntity.metadata.name}`,
+    );
+
+    expect(fs.existsSync(resultDir)).toBeTruthy();
+    expect(fs.existsSync(path.join(resultDir, '/mock-file'))).toBeTruthy();
 
     fs.removeSync(publishDir);
     fs.removeSync(tempDir);

plugins/techdocs-backend/src/techdocs/stages/publish/local.ts

@@ -30,9 +30,14 @@ export class LocalPublish implements PublisherBase {
         remoteUrl: string;
       }>
     | { remoteUrl: string } {
+    const entityNamespace = entity.metadata.namespace ?? 'default';
+
     const publishDir = path.resolve(
       __dirname,
-      `../../../../static/docs/${entity.metadata.name}`,
+      '../../../../static/docs/',
+      entity.kind,
+      entityNamespace,
+      entity.metadata.name
     );
 
     if (!fs.existsSync(publishDir)) {