readTree method on UrlReader (#2925)

* Basic implementation of readTree on github and use it in techdocs

* Added a UrlPreparer and cleaned up some minor things

* Download entire archive and filter out the wanted files in readTree

* Added test for dir output of tree reader

* Fixed formatting

* Fixed formatting

* Some cleanups and fixes

* Fix typing issues

* Fix prettier

* Fix prettier

* Removed unused dependency

* Fixed comments on PR

* Prettier

* Moved @types/fs-extra to devDeps

* Fixed another PR comment

* Added a test for getDocFilesFromRepository

* Prettier

plugins/techdocs-backend/src/helpers.test.ts

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getDocFilesFromRepository } from './helpers';
+import { UrlReader, ReadTreeResponse } from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
+
+describe('getDocFilesFromRepository', () => {
+  it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in root', async () => {
+    class MockUrlReader implements UrlReader {
+      async read() {
+        return Buffer.from('mock');
+      }
+
+      async readTree(): Promise<ReadTreeResponse> {
+        return {
+          dir: async () => {
+            return '/tmp/testfolder';
+          },
+          files: () => {
+            return [];
+          },
+          archive: async () => {
+            return Buffer.from('');
+          },
+        };
+      }
+    }
+
+    const mockEntity: Entity = {
+      metadata: {
+        namespace: 'default',
+        annotations: {
+          'backstage.io/techdocs-ref':
+            'url:https://github.com/backstage/backstage/blob/master/mkdocs.yml',
+        },
+        name: 'mytestcomponent',
+        description: 'A component for testing',
+      },
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      spec: {
+        type: 'documentation',
+        lifecycle: 'experimental',
+        owner: 'testuser',
+      },
+    };
+
+    const output = await getDocFilesFromRepository(
+      new MockUrlReader(),
+      mockEntity,
+    );
+
+    expect(output).toBe('/tmp/testfolder/.');
+  });
+
+  it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in a subfolder', async () => {
+    class MockUrlReader implements UrlReader {
+      async read() {
+        return Buffer.from('mock');
+      }
+
+      async readTree(): Promise<ReadTreeResponse> {
+        return {
+          dir: async () => {
+            return '/tmp/testfolder';
+          },
+          files: () => {
+            return [];
+          },
+          archive: async () => {
+            return Buffer.from('');
+          },
+        };
+      }
+    }
+
+    const mockEntity: Entity = {
+      metadata: {
+        namespace: 'default',
+        annotations: {
+          'backstage.io/techdocs-ref':
+            'url:https://github.com/backstage/backstage/blob/master/subfolder/mkdocs.yml',
+        },
+        name: 'mytestcomponent',
+        description: 'A component for testing',
+      },
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      spec: {
+        type: 'documentation',
+        lifecycle: 'experimental',
+        owner: 'testuser',
+      },
+    };
+
+    const output = await getDocFilesFromRepository(
+      new MockUrlReader(),
+      mockEntity,
+    );
+
+    expect(output).toBe('/tmp/testfolder/subfolder');
+  });
+});

plugins/techdocs-backend/src/helpers.ts

@@ -22,7 +22,7 @@ import fs from 'fs-extra';
 import { getDefaultBranch } from './default-branch';
 import { getGitRepoType, getTokenForGitRepo } from './git-auth';
 import { Entity } from '@backstage/catalog-model';
-import { InputError } from '@backstage/backend-common';
+import { InputError, UrlReader } from '@backstage/backend-common';
 import { RemoteProtocol } from './techdocs/stages/prepare/types';
 import { Logger } from 'winston';
 
@@ -78,6 +78,7 @@ export const getLocationForEntity = (
     case 'github':
     case 'gitlab':
     case 'azure/api':
+    case 'url':
       return { type, target };
     case 'dir':
       if (path.isAbsolute(target)) return { type, target };
@@ -168,3 +169,34 @@ export const getLastCommitTimestamp = async (
 
   return commit.date().getTime();
 };
+
+export const getDocFilesFromRepository = async (
+  reader: UrlReader,
+  entity: Entity,
+): Promise<any> => {
+  const { target } = parseReferenceAnnotation(
+    'backstage.io/techdocs-ref',
+    entity,
+  );
+
+  const { ref, filepath: mkdocsPath } = parseGitUrl(target);
+
+  const docsRootPath = path.dirname(mkdocsPath);
+  const docsFolderPath = path.join(docsRootPath, 'docs');
+
+  if (reader.readTree) {
+    const readTreeResponse = await reader.readTree(
+      parseGitUrl(target).toString(),
+      ref,
+      [mkdocsPath, docsFolderPath],
+    );
+
+    const tmpDir = await readTreeResponse.dir();
+
+    return `${tmpDir}/${docsRootPath}`;
+  }
+
+  throw new Error(
+    `No readTree method available on the UrlReader for ${target}`,
+  );
+};

plugins/techdocs-backend/src/service/helpers.ts

@@ -103,7 +103,7 @@ export class DocsBuilder {
     const { type, target } = getLocationForEntity(this.entity);
 
     // Unless docs are stored locally
-    const nonAgeCheckTypes = ['dir', 'file'];
+    const nonAgeCheckTypes = ['dir', 'file', 'url'];
     if (!nonAgeCheckTypes.includes(type)) {
       const lastCommit = await getLastCommitTimestamp(target, this.logger);
       const storageTimeStamp = buildMetadataStorage.getTimestamp();

plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts

@@ -15,5 +15,6 @@
  */
 export { DirectoryPreparer } from './dir';
 export { CommonGitPreparer } from './commonGit';
+export { UrlPreparer } from './url';
 export { Preparers } from './preparers';
 export type { PreparerBuilder, PreparerBase } from './types';

plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts

@@ -30,4 +30,10 @@ export type PreparerBuilder = {
   get(entity: Entity): PreparerBase;
 };
 
-export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file' | 'azure/api';
+export type RemoteProtocol =
+  | 'dir'
+  | 'github'
+  | 'gitlab'
+  | 'file'
+  | 'azure/api'
+  | 'url';

plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Entity } from '@backstage/catalog-model';
+import { PreparerBase } from './types';
+import { getDocFilesFromRepository } from '../../../helpers';
+
+import { Logger } from 'winston';
+import { UrlReader } from '@backstage/backend-common';
+
+export class UrlPreparer implements PreparerBase {
+  private readonly logger: Logger;
+  private readonly reader: UrlReader;
+
+  constructor(reader: UrlReader, logger: Logger) {
+    this.logger = logger;
+    this.reader = reader;
+  }
+
+  async prepare(entity: Entity): Promise<string> {
+    try {
+      return getDocFilesFromRepository(this.reader, entity);
+    } catch (error) {
+      this.logger.debug(
+        `Unable to fetch files for building docs ${error.message}`,
+      );
+      throw error;
+    }
+  }
+}