Support parsing mkdocs.yml files that are using custom yaml tags (#5860)

Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>

.changeset/techdocs-nice-forks-flow.md

@@ -0,0 +1,6 @@
+---
+'@backstage/techdocs-common': patch
+---
+
+Support parsing `mkdocs.yml` files that are using custom yaml tags like
+`!!python/name:materialx.emoji.twemoji`.

packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_extensions.yml

@@ -0,0 +1,7 @@
+site_name: Test site name
+site_description: Test site description
+
+markdown_extensions:
+  - pymdownx.emoji:
+      emoji_index: !!python/name:materialx.emoji.twemoji
+      emoji_generator: !!python/name:materialx.emoji.to_svg

packages/techdocs-common/src/stages/generate/helpers.test.ts

@@ -41,6 +41,9 @@ const mockEntity = {
 const mkdocsYml = fs.readFileSync(
   resolvePath(__filename, '../__fixtures__/mkdocs.yml'),
 );
+const mkdocsYmlWithExtensions = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_with_extensions.yml'),
+);
 const mkdocsYmlWithRepoUrl = fs.readFileSync(
   resolvePath(__filename, '../__fixtures__/mkdocs_with_repo_url.yml'),
 );
@@ -175,11 +178,12 @@ describe('helpers', () => {
     });
   });
 
-  describe('pathMkdocsPreBuild', () => {
+  describe('patchMkdocsYmlPreBuild', () => {
     beforeEach(() => {
       mockFs({
         '/mkdocs.yml': mkdocsYml,
         '/mkdocs_with_repo_url.yml': mkdocsYmlWithRepoUrl,
+        '/mkdocs_with_extensions.yml': mkdocsYmlWithExtensions,
       });
     });
 
@@ -206,6 +210,28 @@ describe('helpers', () => {
       );
     });
 
+    it('should add repo_url to mkdocs.yml that contains custom yaml tags', async () => {
+      const parsedLocationAnnotation: ParsedLocationAnnotation = {
+        type: 'github',
+        target: 'https://github.com/backstage/backstage',
+      };
+
+      await patchMkdocsYmlPreBuild(
+        '/mkdocs_with_extensions.yml',
+        mockLogger,
+        parsedLocationAnnotation,
+      );
+
+      const updatedMkdocsYml = await fs.readFile('/mkdocs_with_extensions.yml');
+
+      expect(updatedMkdocsYml.toString()).toContain(
+        'repo_url: https://github.com/backstage/backstage',
+      );
+      expect(updatedMkdocsYml.toString()).toContain(
+        "emoji_index: !!python/name:materialx.emoji.twemoji ''",
+      );
+    });
+
     it('should not override existing repo_url in mkdocs.yml', async () => {
       const parsedLocationAnnotation: ParsedLocationAnnotation = {
         type: 'github',
@@ -309,7 +335,7 @@ describe('helpers', () => {
     beforeEach(() => {
       mockFs({
         '/mkdocs.yml': mkdocsYml,
-        '/mkdocs_with_repo_url.yml': mkdocsYmlWithRepoUrl,
+        '/mkdocs_with_extensions.yml': mkdocsYmlWithExtensions,
         '/mkdocs_invalid_doc_dir.yml': mkdocsYmlWithInvalidDocDir,
       });
     });
@@ -327,5 +353,11 @@ describe('helpers', () => {
         validateMkdocsYaml('/mkdocs_invalid_doc_dir.yml'),
       ).rejects.toThrow();
     });
+
+    it('should validate files with custom yaml tags', async () => {
+      await expect(
+        validateMkdocsYaml('/mkdocs_with_extensions.yml'),
+      ).resolves.toBeUndefined();
+    });
   });
 });

packages/techdocs-common/src/stages/generate/helpers.ts

@@ -16,9 +16,9 @@
 
 import { Entity } from '@backstage/catalog-model';
 import { spawn } from 'child_process';
-import { isAbsolute, normalize } from 'path';
 import fs from 'fs-extra';
-import yaml from 'js-yaml';
+import yaml, { DEFAULT_SCHEMA, Type } from 'js-yaml';
+import { isAbsolute, normalize } from 'path';
 import { PassThrough, Writable } from 'stream';
 import { Logger } from 'winston';
 import { ParsedLocationAnnotation } from '../../helpers';
@@ -139,6 +139,21 @@ export const getRepoUrlFromLocationAnnotation = (
   return undefined;
 };
 
+class UnknownTag {
+  constructor(public readonly data: any, public readonly type?: string) {}
+}
+
+const MKDOCS_SCHEMA = DEFAULT_SCHEMA.extend([
+  new Type('', {
+    kind: 'scalar',
+    multi: true,
+    representName: o => (o as UnknownTag).type,
+    represent: o => (o as UnknownTag).data ?? '',
+    instanceOf: UnknownTag,
+    construct: (data: string, type?: string) => new UnknownTag(data, type),
+  }),
+]);
+
 /**
  * Validating mkdocs config file for incorrect/insecure values
  * Throws on invalid configs
@@ -155,7 +170,9 @@ export const validateMkdocsYaml = async (mkdocsYmlPath: string) => {
     );
   }
 
-  const mkdocsYml: any = yaml.load(mkdocsYmlFileString);
+  const mkdocsYml: any = yaml.load(mkdocsYmlFileString, {
+    schema: MKDOCS_SCHEMA,
+  });
   if (mkdocsYml.docs_dir && isAbsolute(normalize(mkdocsYml.docs_dir))) {
     throw new Error(
       "docs_dir configuration value in mkdocs can't be an absolute directory path for security reasons. Use relative paths instead which are resolved relative to your mkdocs.yml file location.",
@@ -196,7 +213,7 @@ export const patchMkdocsYmlPreBuild = async (
 
   let mkdocsYml: any;
   try {
-    mkdocsYml = yaml.load(mkdocsYmlFileString);
+    mkdocsYml = yaml.load(mkdocsYmlFileString, { schema: MKDOCS_SCHEMA });
 
     // mkdocsYml should be an object type after successful parsing.
     // But based on its type definition, it can also be a string or undefined, which we don't want.
@@ -222,7 +239,11 @@ export const patchMkdocsYmlPreBuild = async (
   }
 
   try {
-    await fs.writeFile(mkdocsYmlPath, yaml.dump(mkdocsYml), 'utf8');
+    await fs.writeFile(
+      mkdocsYmlPath,
+      yaml.dump(mkdocsYml, { schema: MKDOCS_SCHEMA }),
+      'utf8',
+    );
   } catch (error) {
     logger.warn(
       `Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`,