Merge branch 'master' of github.com:spotify/backstage into shmidt-i/load-scaffolder-templates-from-api

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -26,6 +26,7 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn
 import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor';
 import { FileReaderProcessor } from './processors/FileReaderProcessor';
 import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
+import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
 import { LocationRefProcessor } from './processors/LocationEntityProcessor';
 import * as result from './processors/results';
 import {
@@ -56,6 +57,7 @@ export class LocationReaders implements LocationReader {
     return [
       new FileReaderProcessor(),
       new GithubReaderProcessor(),
+      new GitlabReaderProcessor(),
       new YamlProcessor(),
       new EntityPolicyProcessor(entityPolicy),
       new LocationRefProcessor(),

plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import fetch from 'node-fetch';
+import * as result from './results';
+import { LocationProcessor, LocationProcessorEmit } from './types';
+
+export class GitlabReaderProcessor implements LocationProcessor {
+  async readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: LocationProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'gitlab') {
+      return false;
+    }
+
+    try {
+      const url = this.buildRawUrl(location.target);
+
+      const response = await fetch(url.toString());
+
+      if (response.ok) {
+        const data = await response.buffer();
+        emit(result.data(location, data));
+      } else {
+        const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`;
+        if (response.status === 404) {
+          if (!optional) {
+            throw result.notFoundError(location, message);
+          }
+        } else {
+          throw result.generalError(location, message);
+        }
+      }
+    } catch (e) {
+      const message = `Unable to read ${location.type} ${location.target}, ${e}`;
+      emit(result.generalError(location, message));
+    }
+
+    return true;
+  }
+
+  // Converts
+  // from: https://gitlab.example.com/a/b/blob/master/c.yaml
+  // to:   https://gitlab.example.com/a/b/raw/master/c.yaml
+  private buildRawUrl(target: string): URL {
+    try {
+      const url = new URL(target);
+
+      const [
+        empty,
+        userOrOrg,
+        repoName,
+        blobKeyword,
+        ...restOfPath
+      ] = url.pathname.split('/');
+
+      if (
+        empty !== '' ||
+        userOrOrg === '' ||
+        repoName === '' ||
+        blobKeyword !== 'blob' ||
+        !restOfPath.join('/').match(/\.yaml$/)
+      ) {
+        throw new Error('Wrong Gitlab URL');
+      }
+
+      // Replace 'blob' with 'raw'
+      url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join(
+        '/',
+      );
+
+      return url;
+    } catch (e) {
+      throw new Error(`Incorrect url: ${target}, ${e}`);
+    }
+  }
+}

plugins/scaffolder-backend/package.json

@@ -23,7 +23,9 @@
   "dependencies": {
     "@backstage/backend-common": "^0.1.1-alpha.12",
     "@backstage/catalog-model": "^0.1.1-alpha.12",
+    "@backstage/config": "^0.1.1-alpha.12",
     "@types/express": "^4.17.6",
+    "@types/dockerode": "^2.5.32",
     "compression": "^1.7.4",
     "cors": "^2.8.5",
     "dockerode": "^3.2.0",

plugins/scaffolder-backend/sample-templates/react-ssr-template/cookiecutter.json

@@ -0,0 +1,3 @@
+{
+  "_copy_without_render": [".github/workflows/*"]
+}

plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh

@@ -1,12 +1,7 @@
-#!/bin/bash
+#!/bin/sh
 
-# package name is "__component_id__" so that yarn doesn't throw an error
-# about invalid characters when running yarn commands. here we replace it with the actual name
-sed -i -e "s/__component_id__/{{ cookiecutter.component_id }}/g" package.json
-
-# node_modules was moved out of the template folder, during the pre_gen hook,
-# to avoid cookie_cutter from copying all of them. time to move it back
-mv ../../node_modules.tmp ../../\{\{cookiecutter.component_id\}\}/node_modules 2>/dev/null ||:
-
-# move back the build directory that was moved out in the pre_gen hook (if it exists)
-mv ../../build.tmp ../../\{\{cookiecutter.component_id\}\}/build 2>/dev/null ||:
+# Move all template files to the root folder
+mv ./* ../
+cd ..
+rm -rf {{cookiecutter.component_id}}
+# # # # # # # # 

plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/pre_gen_project.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-# no way to ignore files in cookiecutter, so move node_modules out while building
-# to avoid cookiecutter from copying all of them
-mv ../../\{\{cookiecutter.component_id\}\}/node_modules ../../node_modules.tmp  2>/dev/null ||:
-
-# cookicutter really doesn't like the next.js build directory, so if the app has
-# been built from inside the template folder, that folders needs to be moved out as well
-mv ../../\{\{cookiecutter.component_id\}\}/build ../../build.tmp  2>/dev/null ||:

plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "__component_id__",
+  "name": "{{ cookiecutter.component_id }}",
   "version": "0.0.0",
   "description": "{{ cookiecutter.description }}",
   "license": "UNLICENSED",

plugins/scaffolder-backend/scripts/Dockerfile

@@ -0,0 +1,11 @@
+FROM alpine:3.7
+
+RUN apk add --update \
+	git \
+	python \
+	python-dev \
+	py-pip \
+	g++ && \
+	pip install cookiecutter && \
+	apk del g++ py-pip python-dev && \
+	rm -rf /var/cache/apk/* 

plugins/scaffolder-backend/src/index.ts

@@ -16,3 +16,4 @@
 
 export * from './scaffolder';
 export * from './service/router';
+

plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts

@@ -0,0 +1,151 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+jest.mock('./helpers', () => ({ runDockerContainer: jest.fn() }));
+
+import { CookieCutter } from './cookiecutter';
+import fs from 'fs-extra';
+import os from 'os';
+import { RunDockerContainerOptions } from './helpers';
+import { PassThrough } from 'stream';
+import Docker from 'dockerode';
+
+describe('CookieCutter Templater', () => {
+  const cookie = new CookieCutter();
+  const mockDocker = {} as Docker;
+  const {
+    runDockerContainer,
+  }: {
+    runDockerContainer: jest.Mock<RunDockerContainerOptions>;
+  } = require('./helpers');
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+
+    await fs.remove(`${os.tmpdir()}/cookiecutter.json`);
+  });
+
+  it('should write a cookiecutter.json file with the values from the entitiy', async () => {
+    const tempdir = os.tmpdir();
+
+    const values = {
+      component_id: 'test',
+      description: 'description',
+    };
+
+    await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
+
+    const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`);
+
+    expect(cookieCutterJson).toEqual(expect.objectContaining(values));
+  });
+
+  it('should merge any value that is in the cookiecutter.json path already', async () => {
+    const tempdir = os.tmpdir();
+    const existingJson = {
+      _copy_without_render: ['./github/workflows/*'],
+    };
+    await fs.writeJSON(`${tempdir}/cookiecutter.json`, existingJson);
+
+    const values = {
+      component_id: 'hello',
+      description: 'im something cool',
+    };
+
+    await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
+
+    const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`);
+
+    expect(cookieCutterJson).toEqual({ ...existingJson, ...values });
+  });
+
+  it('should throw an error if the cookiecutter json is malformed and not missing', async () => {
+    const tempdir = os.tmpdir();
+
+    await fs.writeFile(`${tempdir}/cookiecutter.json`, "{'");
+
+    const values = {
+      component_id: 'hello',
+      description: 'im something cool',
+    };
+
+    await expect(
+      cookie.run({ directory: tempdir, values, dockerClient: mockDocker }),
+    ).rejects.toThrow(/Unexpected token ' in JSON at position 1/);
+  });
+
+  it('should run the correct docker container with the correct bindings for the volumes', async () => {
+    const tempdir = os.tmpdir();
+
+    const values = {
+      component_id: 'test',
+      description: 'description',
+    };
+
+    await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
+
+    expect(runDockerContainer).toHaveBeenCalledWith({
+      imageName: 'backstage/cookiecutter',
+      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      templateDir: tempdir,
+      resultDir: `${tempdir}/result`,
+      logStream: undefined,
+      dockerClient: mockDocker,
+    });
+  });
+  it('should return the result path to the end templated folder', async () => {
+    const tempdir = os.tmpdir();
+
+    const values = {
+      component_id: 'test',
+      description: 'description',
+    };
+
+    const path = await cookie.run({
+      directory: tempdir,
+      values,
+      dockerClient: mockDocker,
+    });
+
+    expect(path).toBe(`${tempdir}/result`);
+  });
+
+  it('should pass through the streamer to the run docker helper', async () => {
+    const stream = new PassThrough();
+
+    const tempdir = os.tmpdir();
+
+    const values = {
+      component_id: 'test',
+      description: 'description',
+    };
+
+    await cookie.run({
+      directory: tempdir,
+      values,
+      logStream: stream,
+      dockerClient: mockDocker,
+    });
+
+    expect(runDockerContainer).toHaveBeenCalledWith({
+      imageName: 'backstage/cookiecutter',
+      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      templateDir: tempdir,
+      resultDir: `${tempdir}/result`,
+      logStream: stream,
+      dockerClient: mockDocker,
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts

@@ -16,17 +16,52 @@ import { TemplaterBase, TemplaterRunOptions } from '.';
  * limitations under the License.
  */
 import fs from 'fs-extra';
+import { JsonValue } from '@backstage/config';
+import { runDockerContainer } from './helpers';
 
 export class CookieCutter implements TemplaterBase {
+  private async fetchTemplateCookieCutter(
+    directory: string,
+  ): Promise<Record<string, JsonValue>> {
+    try {
+      return await fs.readJSON(`${directory}/cookiecutter.json`);
+    } catch (ex) {
+      if (ex.code !== 'ENOENT') {
+        throw ex;
+      }
+
+      return {};
+    }
+  }
+
   public async run(options: TemplaterRunOptions): Promise<string> {
-    // first we need to make cookiecutter.json in the directory provided with the input values.
+    // First lets grab the default cookiecutter.json file
+    const cookieCutterJson = await this.fetchTemplateCookieCutter(
+      options.directory,
+    );
+
     const cookieInfo = {
-      _copy_without_render: ['.github/workflows/*'],
+      ...cookieCutterJson,
       ...options.values,
     };
 
     await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo);
-    return '';
-    // run cookie cutter with new json
+
+    const templateDir = options.directory;
+
+    // TODO(blam): This should be an entirely different directory on the host machine
+    // not in the template directory
+    const resultDir = `${templateDir}/result`;
+
+    await runDockerContainer({
+      imageName: 'backstage/cookiecutter',
+      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      templateDir,
+      resultDir,
+      logStream: options.logStream,
+      dockerClient: options.dockerClient,
+    });
+
+    return resultDir;
   }
 }

plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts

@@ -0,0 +1,135 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Stream, { PassThrough } from 'stream';
+import os from 'os';
+import fs from 'fs';
+import Docker from 'dockerode';
+import { runDockerContainer } from './helpers';
+
+describe('helpers', () => {
+  const mockDocker = new Docker() as jest.Mocked<Docker>;
+
+  beforeEach(() => {
+    jest
+      .spyOn(mockDocker, 'run')
+      .mockResolvedValue([{ Error: null, StatusCode: 0 }]);
+  });
+
+  describe('runDockerContainer', () => {
+    const imageName = 'blam/github:ben';
+    const args = ['bash', '-c', 'echo lol'];
+    const templateDir = os.tmpdir();
+    const resultDir = os.tmpdir();
+
+    it('should call the dockerClient run command with the correct arguments passed through', async () => {
+      await runDockerContainer({
+        imageName,
+        args,
+        templateDir,
+        resultDir,
+        dockerClient: mockDocker,
+      });
+
+      expect(mockDocker.run).toHaveBeenCalledWith(
+        imageName,
+        args,
+        expect.any(Stream),
+        expect.objectContaining({
+          HostConfig: {
+            Binds: expect.arrayContaining([
+              `${await fs.promises.realpath(templateDir)}:/template`,
+              `${await fs.promises.realpath(resultDir)}:/result`,
+            ]),
+          },
+          Volumes: {
+            '/template': {},
+            '/result': {},
+          },
+        }),
+      );
+    });
+
+    it('throws a correct error if the templating fails in docker', async () => {
+      mockDocker.run.mockResolvedValueOnce([
+        {
+          Error: new Error('Something went wrong with docker'),
+          StatusCode: 0,
+        },
+      ]);
+
+      await expect(
+        runDockerContainer({
+          imageName,
+          args,
+          templateDir,
+          resultDir,
+          dockerClient: mockDocker,
+        }),
+      ).rejects.toThrow(/Something went wrong with docker/);
+    });
+
+    it('throws a correct error when the response code of the container is non-zero', async () => {
+      mockDocker.run.mockResolvedValueOnce([
+        {
+          Error: null,
+          StatusCode: 123,
+        },
+      ]);
+
+      await expect(
+        runDockerContainer({
+          imageName,
+          args,
+          templateDir,
+          resultDir,
+          dockerClient: mockDocker,
+        }),
+      ).rejects.toThrow(
+        /Docker container returned a non-zero exit code \(123\)/,
+      );
+    });
+
+    it('should pass through the log stream to the docker client', async () => {
+      const logStream = new PassThrough();
+      await runDockerContainer({
+        imageName,
+        args,
+        templateDir,
+        resultDir,
+        logStream,
+        dockerClient: mockDocker,
+      });
+
+      expect(mockDocker.run).toHaveBeenCalledWith(
+        imageName,
+        args,
+        logStream,
+        expect.objectContaining({
+          HostConfig: {
+            Binds: expect.arrayContaining([
+              `${await fs.promises.realpath(templateDir)}:/template`,
+              `${await fs.promises.realpath(resultDir)}:/result`,
+            ]),
+          },
+          Volumes: {
+            '/template': {},
+            '/result': {},
+          },
+        }),
+      );
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Writable, PassThrough } from 'stream';
+import Docker from 'dockerode';
+import fs from 'fs';
+
+export type RunDockerContainerOptions = {
+  imageName: string;
+  args: string[];
+  logStream?: Writable;
+  resultDir: string;
+  templateDir: string;
+  dockerClient: Docker;
+};
+
+/**
+ *
+ * @param options the options object
+ * @param options.imageName the image to run
+ * @param options.args the arguments to pass the container
+ * @param options.logStream the log streamer to capture log messages
+ * @param options.resultDir the /result path inside the container
+ * @param options.templateDir the /template path inside the container
+ * @param options.dockerClient the dockerClient to use
+ */
+export const runDockerContainer = async ({
+  imageName,
+  args,
+  logStream = new PassThrough(),
+  resultDir,
+  templateDir,
+  dockerClient,
+}: RunDockerContainerOptions) => {
+  const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run(
+    imageName,
+    args,
+    logStream,
+    {
+      Volumes: { '/result': {}, '/template': {} },
+      HostConfig: {
+        Binds: [
+          // Need to use realpath here as Docker mounting does not like
+          // symlinks for binding volumes
+          `${await fs.promises.realpath(resultDir)}:/result`,
+          `${await fs.promises.realpath(templateDir)}:/template`,
+        ],
+      },
+    },
+  );
+
+  if (error) {
+    throw new Error(
+      `Docker failed to run with the following error message: ${error}`,
+    );
+  }
+
+  if (statusCode !== 0) {
+    throw new Error(
+      `Docker container returned a non-zero exit code (${statusCode})`,
+    );
+  }
+
+  return { error, statusCode };
+};

plugins/scaffolder-backend/src/scaffolder/templater/index.ts

@@ -14,6 +14,9 @@
  * limitations under the License.
  */
 
+import type { Writable } from 'stream';
+import Docker from 'dockerode';
+
 export interface RequiredTemplateValues {
   component_id: string;
 }
@@ -21,12 +24,14 @@ export interface RequiredTemplateValues {
 export interface TemplaterRunOptions {
   directory: string;
   values: RequiredTemplateValues & object;
+  logStream?: Writable;
+  dockerClient: Docker;
 }
 
-export abstract class TemplaterBase {
+export type TemplaterBase = {
   // runs the templating with the values and returns the directory to push the VCS
-  abstract async run(opts: TemplaterRunOptions): Promise<string>;
-}
+  run(opts: TemplaterRunOptions): Promise<string>;
+};
 
 export interface TemplaterConfig {
   templater?: TemplaterBase;

plugins/scaffolder-backend/src/service/router.ts

@@ -19,18 +19,20 @@ import Router from 'express-promise-router';
 import express from 'express';
 import { PreparerBuilder, TemplaterBase } from '../scaffolder';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import Docker from 'dockerode';
 
 export interface RouterOptions {
   preparers: PreparerBuilder;
   templater: TemplaterBase;
   logger: Logger;
+  dockerClient: Docker;
 }
 
 export async function createRouter(
   options: RouterOptions,
 ): Promise<express.Router> {
   const router = Router();
-  const { preparers, templater, logger: parentLogger } = options;
+  const { preparers, templater, logger: parentLogger, dockerClient } = options;
   const logger = parentLogger.child({ plugin: 'scaffolder' });
 
   router.post('/v1/jobs', async (_, res) => {
@@ -66,12 +68,16 @@ export async function createRouter(
     const preparer = preparers.get(mockEntity);
 
     // Run the preparer for the mock entity to produce a temporary directory with template in
-    const path = await preparer.prepare(mockEntity);
+    const skeletonPath = await preparer.prepare(mockEntity);
 
     // Run the templater on the mock directory with values from the post body
-    await templater.run({ directory: path, values: { component_id: 'test' } });
+    const templatedPath = await templater.run({
+      directory: skeletonPath,
+      values: { component_id: 'test' },
+      dockerClient,
+    });
 
-    console.warn(path);
+    console.warn(templatedPath);
   });
 
   const app = express();

plugins/techdocs/mkdocs/container/techdocs-core/src/core.py

@@ -44,42 +44,42 @@ class TechDocsCore(BasePlugin):
         config["plugins"]["search"] = search_plugin
 
         # Markdown Extensions
-        config['markdown_extensions'].append('admonition')
-        config['markdown_extensions'].append('abbr')
-        config['markdown_extensions'].append('attr_list')
-        config['markdown_extensions'].append('def_list')
-        config['markdown_extensions'].append('codehilite')
-        config['mdx_configs']['codehilite'] = {
-            'linenums': True,
-            'guess_lang': False,
-            'pygments_style': 'friendly',
+        config["markdown_extensions"].append("admonition")
+        config["markdown_extensions"].append("abbr")
+        config["markdown_extensions"].append("attr_list")
+        config["markdown_extensions"].append("def_list")
+        config["markdown_extensions"].append("codehilite")
+        config["mdx_configs"]["codehilite"] = {
+            "linenums": True,
+            "guess_lang": False,
+            "pygments_style": "friendly",
         }
-        config['markdown_extensions'].append('toc')
-        config['mdx_configs']['toc'] = {
-            'permalink': True,
+        config["markdown_extensions"].append("toc")
+        config["mdx_configs"]["toc"] = {
+            "permalink": True,
         }
-        config['markdown_extensions'].append('footnotes')
-        config['markdown_extensions'].append('markdown.extensions.tables')
-        config['markdown_extensions'].append('pymdownx.betterem')
-        config['mdx_configs']['pymdownx.betterem'] = {
-            'smart_enable': 'all',
+        config["markdown_extensions"].append("footnotes")
+        config["markdown_extensions"].append("markdown.extensions.tables")
+        config["markdown_extensions"].append("pymdownx.betterem")
+        config["mdx_configs"]["pymdownx.betterem"] = {
+            "smart_enable": "all",
         }
-        config['markdown_extensions'].append('pymdownx.caret')
-        config['markdown_extensions'].append('pymdownx.critic')
-        config['markdown_extensions'].append('pymdownx.details')
-        config['markdown_extensions'].append('pymdownx.emoji')
-        config['mdx_configs']['pymdownx.emoji'] = {
-            'emoji_generator': '!!python/name:pymdownx.emoji.to_svg',
+        config["markdown_extensions"].append("pymdownx.caret")
+        config["markdown_extensions"].append("pymdownx.critic")
+        config["markdown_extensions"].append("pymdownx.details")
+        config["markdown_extensions"].append("pymdownx.emoji")
+        config["mdx_configs"]["pymdownx.emoji"] = {
+            "emoji_generator": "!!python/name:pymdownx.emoji.to_svg",
         }
-        config['markdown_extensions'].append('pymdownx.inlinehilite')
-        config['markdown_extensions'].append('pymdownx.magiclink')
-        config['markdown_extensions'].append('pymdownx.mark')
-        config['markdown_extensions'].append('pymdownx.smartsymbols')
-        config['markdown_extensions'].append('pymdownx.superfences')
-        config['markdown_extensions'].append('pymdownx.tasklist')
-        config['mdx_configs']['pymdownx.tasklist'] = {
-            'custom_checkbox': True,
+        config["markdown_extensions"].append("pymdownx.inlinehilite")
+        config["markdown_extensions"].append("pymdownx.magiclink")
+        config["markdown_extensions"].append("pymdownx.mark")
+        config["markdown_extensions"].append("pymdownx.smartsymbols")
+        config["markdown_extensions"].append("pymdownx.superfences")
+        config["markdown_extensions"].append("pymdownx.tasklist")
+        config["mdx_configs"]["pymdownx.tasklist"] = {
+            "custom_checkbox": True,
         }
-        config['markdown_extensions'].append('pymdownx.tilde')
+        config["markdown_extensions"].append("pymdownx.tilde")
 
         return config

plugins/techdocs/package.json

@@ -22,6 +22,7 @@
   },
   "dependencies": {
     "@backstage/core": "^0.1.1-alpha.12",
+    "@backstage/test-utils": "^0.1.1-alpha.12",
     "@backstage/theme": "^0.1.1-alpha.12",
     "@material-ui/core": "^4.9.1",
     "@material-ui/icons": "^4.9.1",

plugins/techdocs/src/components/ExampleComponent/ExampleComponent.test.tsx

@@ -1,34 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React from 'react';
-import { render } from '@testing-library/react';
-import mockFetch from 'jest-fetch-mock';
-import ExampleComponent from './ExampleComponent';
-import { ThemeProvider } from '@material-ui/core';
-import { lightTheme } from '@backstage/theme';
-
-describe('ExampleComponent', () => {
-  it('should render', () => {
-    mockFetch.mockResponse(() => new Promise(() => {}));
-    const rendered = render(
-      <ThemeProvider theme={lightTheme}>
-        <ExampleComponent />
-      </ThemeProvider>,
-    );
-    expect(rendered.getByText('Welcome to techdocs!')).toBeInTheDocument();
-  });
-});

plugins/techdocs/src/components/ExampleComponent/ExampleComponent.tsx

@@ -1,57 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React, { FC } from 'react';
-import { Typography, Grid } from '@material-ui/core';
-import {
-  InfoCard,
-  Header,
-  Page,
-  pageTheme,
-  Content,
-  ContentHeader,
-  HeaderLabel,
-  SupportButton,
-} from '@backstage/core';
-import ExampleFetchComponent from '../ExampleFetchComponent';
-
-const ExampleComponent: FC<{}> = () => (
-  <Page theme={pageTheme.tool}>
-    <Header title="Welcome to techdocs!" subtitle="Optional subtitle">
-      <HeaderLabel label="Owner" value="Team X" />
-      <HeaderLabel label="Lifecycle" value="Alpha" />
-    </Header>
-    <Content>
-      <ContentHeader title="Plugin title">
-        <SupportButton>A description of your plugin goes here.</SupportButton>
-      </ContentHeader>
-      <Grid container spacing={3} direction="column">
-        <Grid item>
-          <InfoCard title="Information card">
-            <Typography variant="body1">
-              All content should be wrapped in a card like this.
-            </Typography>
-          </InfoCard>
-        </Grid>
-        <Grid item>
-          <ExampleFetchComponent />
-        </Grid>
-      </Grid>
-    </Content>
-  </Page>
-);
-
-export default ExampleComponent;

plugins/techdocs/src/components/ExampleFetchComponent/ExampleFetchComponent.tsx

@@ -1,108 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React, { FC } from 'react';
-import { makeStyles } from '@material-ui/core/styles';
-import { Table, TableColumn, Progress } from '@backstage/core';
-import Alert from '@material-ui/lab/Alert';
-import { useAsync } from 'react-use';
-
-const useStyles = makeStyles({
-  avatar: {
-    height: 32,
-    width: 32,
-    borderRadius: '50%',
-  },
-});
-
-type User = {
-  gender: string; // "male"
-  name: {
-    title: string; // "Mr",
-    first: string; // "Duane",
-    last: string; // "Reed"
-  };
-  location: object; // {street: {number: 5060, name: "Hickory Creek Dr"}, city: "Albany", state: "New South Wales",…}
-  email: string; // "duane.reed@example.com"
-  login: object; // {uuid: "4b785022-9a23-4ab9-8a23-cb3fb43969a9", username: "blackdog796", password: "patch",…}
-  dob: object; // {date: "1983-06-22T12:30:23.016Z", age: 37}
-  registered: object; // {date: "2006-06-13T18:48:28.037Z", age: 14}
-  phone: string; // "07-2154-5651"
-  cell: string; // "0405-592-879"
-  id: {
-    name: string; // "TFN",
-    value: string; // "796260432"
-  };
-  picture: { medium: string }; // {medium: "https://randomuser.me/api/portraits/men/95.jpg",…}
-  nat: string; // "AU"
-};
-
-type DenseTableProps = {
-  users: User[];
-};
-
-export const DenseTable: FC<DenseTableProps> = ({ users }) => {
-  const classes = useStyles();
-
-  const columns: TableColumn[] = [
-    { title: 'Avatar', field: 'avatar' },
-    { title: 'Name', field: 'name' },
-    { title: 'Email', field: 'email' },
-    { title: 'Nationality', field: 'nationality' },
-  ];
-
-  const data = users.map(user => {
-    return {
-      avatar: (
-        <img
-          src={user.picture.medium}
-          className={classes.avatar}
-          alt={user.name.first}
-        />
-      ),
-      name: `${user.name.first} ${user.name.last}`,
-      email: user.email,
-      nationality: user.nat,
-    };
-  });
-
-  return (
-    <Table
-      title="Example User List (fetching data from randomuser.me)"
-      options={{ search: false, paging: false }}
-      columns={columns}
-      data={data}
-    />
-  );
-};
-
-const ExampleFetchComponent: FC<{}> = () => {
-  const { value, loading, error } = useAsync(async (): Promise<User[]> => {
-    const response = await fetch('https://randomuser.me/api/?results=20');
-    const data = await response.json();
-    return data.results;
-  }, []);
-
-  if (loading) {
-    return <Progress />;
-  } else if (error) {
-    return <Alert severity="error">{error.message}</Alert>;
-  }
-
-  return <DenseTable users={value || []} />;
-};
-
-export default ExampleFetchComponent;

plugins/techdocs/src/plugin.ts

@@ -30,16 +30,16 @@
  */
 
 import { createPlugin, createRouteRef } from '@backstage/core';
-import ExampleComponent from './components/ExampleComponent';
+import { Reader } from './reader/components/Reader';
 
 export const rootRouteRef = createRouteRef({
-  path: '/techdocs',
-  title: 'techdocs',
+  path: '/docs',
+  title: 'Docs',
 });
 
 export const plugin = createPlugin({
   id: 'techdocs',
   register({ router }) {
-    router.addRoute(rootRouteRef, ExampleComponent);
+    router.addRoute(rootRouteRef, Reader);
   },
 });

plugins/techdocs/src/reader/components/Reader.tsx

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { useShadowDom } from '..';
+import { useAsync } from 'react-use';
+
+const useFetch = (url: string) => {
+  const state = useAsync(async () => {
+    const response = await fetch(url);
+    const raw = await response.text();
+    return raw;
+  }, [url]);
+
+  return state;
+};
+
+const addBaseUrl = (htmlString: string, baseUrl: string): string => {
+  const domParser = new DOMParser().parseFromString(htmlString, 'text/html');
+
+  const updateDom = <T extends Element>(
+    list: Array<T>,
+    attributeName: string,
+  ): void => {
+    Array.from(list).forEach((elem: T) => {
+      const newUrl = new URL(
+        elem.getAttribute(attributeName)!,
+        baseUrl,
+      ).toString();
+      elem.setAttribute(attributeName, newUrl);
+    });
+  };
+
+  updateDom<HTMLImageElement>(Array.from(domParser.images), 'src');
+  updateDom<HTMLAnchorElement | HTMLAreaElement>(
+    Array.from(domParser.links),
+    'href',
+  );
+  updateDom<HTMLLinkElement>(
+    Array.from(domParser.querySelectorAll('link')),
+    'href',
+  );
+
+  return domParser.body.parentElement?.outerHTML || htmlString;
+};
+
+export const Reader = () => {
+  const shadowDomRef = useShadowDom();
+  const state = useFetch(
+    'https://techdocs-mock-sites.storage.googleapis.com/mkdocs/index.html',
+  );
+
+  React.useEffect(() => {
+    const divElement = shadowDomRef.current;
+    if (divElement?.shadowRoot && state.value) {
+      divElement.shadowRoot.innerHTML = addBaseUrl(
+        state.value,
+        'https://techdocs-mock-sites.storage.googleapis.com/mkdocs/',
+      );
+    }
+  }, [shadowDomRef, state]);
+
+  return (
+    <>
+      <h3>Shadow DOM should be underneath</h3>
+      <div ref={shadowDomRef} />
+    </>
+  );
+};

plugins/techdocs/src/reader/hooks/index.ts

@@ -14,4 +14,4 @@
  * limitations under the License.
  */
 
-export { default } from './ExampleComponent';
+export { useShadowDom } from './shadowDom';

plugins/techdocs/src/reader/hooks/shadowDom.test.tsx

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { renderWithEffects } from '@backstage/test-utils';
+import { useShadowDom } from './shadowDom';
+
+const ComponentWithoutHook = () => {
+  return <div data-testid="shadow-dom" />;
+};
+
+const ComponentWithHook = () => {
+  const ref = useShadowDom();
+  return <div data-testid="shadow-dom" ref={ref} />;
+};
+
+describe('useShadowDom', () => {
+  it('does not create a Shadow DOM instance', async () => {
+    const rendered = await renderWithEffects(<ComponentWithoutHook />);
+
+    const divElement = rendered.getByTestId('shadow-dom');
+    expect(divElement.shadowRoot).not.toBeInstanceOf(ShadowRoot);
+  });
+
+  it('create a Shadow DOM instance', async () => {
+    const rendered = await renderWithEffects(<ComponentWithHook />);
+
+    const divElement = rendered.getByTestId('shadow-dom');
+    expect(divElement.shadowRoot).toBeInstanceOf(ShadowRoot);
+  });
+});

plugins/techdocs/src/reader/hooks/shadowDom.ts

@@ -14,15 +14,17 @@
  * limitations under the License.
  */
 
-import React from 'react';
-import { render } from '@testing-library/react';
-import mockFetch from 'jest-fetch-mock';
-import ExampleFetchComponent from './ExampleFetchComponent';
+import { useEffect, useRef } from 'react';
+import type { RefObject } from 'react';
 
-describe('ExampleFetchComponent', () => {
-  it('should render', async () => {
-    mockFetch.mockResponse(() => new Promise(() => {}));
-    const rendered = render(<ExampleFetchComponent />);
-    expect(await rendered.findByTestId('progress')).toBeInTheDocument();
-  });
-});
+type IShadowDOMRefObject = RefObject<HTMLDivElement>;
+export const useShadowDom: () => IShadowDOMRefObject = () => {
+  const ref: IShadowDOMRefObject = useRef(null);
+
+  useEffect(() => {
+    const divElement = ref.current;
+    divElement?.attachShadow({ mode: 'open' });
+  }, [ref]);
+
+  return ref;
+};

plugins/techdocs/src/reader/index.tsx

@@ -14,4 +14,4 @@
  * limitations under the License.
  */
 
-export { default } from './ExampleFetchComponent';
+export * from './hooks';