Merge pull request #4287 from backstage/orkohunter/dir-preparer-uses-url-reader

.changeset/techdocs-chilly-steaks-brush.md

@@ -0,0 +1,5 @@
+---
+'@backstage/techdocs-common': patch
+---
+
+dir preparer will use URL Reader in its implementation.

app-config.yaml

@@ -208,7 +208,7 @@ catalog:
     # Backstage example components
     - type: file
       target: ../catalog-model/examples/all-components.yaml
-    # Example component for github-actions
+    # Example component for github-actions and TechDocs
     - type: file
       target: ../../plugins/github-actions/examples/sample.yaml
     # Example component for TechDocs

docs/features/techdocs/creating-and-publishing.md

@@ -66,9 +66,16 @@ Update your component's entity description by adding the following lines to its
 ```yaml
 metadata:
   annotations:
-    backstage.io/techdocs-ref: dir:./
+    backstage.io/techdocs-ref: url:https://github.com/org/repo
+    # Or
+    # backstage.io/techdocs-ref: url:https://github.com/org/repo/tree/branchName/subFolder
 ```
 
+The
+[`backstage.io/techdocs-ref` annotation](../software-catalog/well-known-annotations.md#backstageiotechdocs-ref)
+is used by TechDocs to download the documentation source files for generating an
+Entity's TechDocs site.
+
 Create a `/docs` folder in the root of the project with at least an `index.md`
 file. _(If you add more markdown files, make sure to update the nav in the
 mkdocs.yml file to get a proper navigation for your documentation.)_

packages/techdocs-common/src/stages/prepare/dir.test.ts

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { getVoidLogger } from '@backstage/backend-common';
+import { getVoidLogger, UrlReader } from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
 import { DirectoryPreparer } from './dir';
 import { checkoutGitRepository } from '../../helpers';
@@ -46,10 +46,18 @@ const createMockEntity = (annotations: {}) => {
 };
 
 const mockConfig = new ConfigReader({});
+const mockUrlReader: jest.Mocked<UrlReader> = {
+  read: jest.fn(),
+  readTree: jest.fn(),
+};
 
 describe('directory preparer', () => {
   it('should merge managed-by-location and techdocs-ref when techdocs-ref is relative', async () => {
-    const directoryPreparer = new DirectoryPreparer(mockConfig, logger);
+    const directoryPreparer = new DirectoryPreparer(
+      mockConfig,
+      logger,
+      mockUrlReader,
+    );
 
     const mockEntity = createMockEntity({
       'backstage.io/managed-by-location':
@@ -63,7 +71,11 @@ describe('directory preparer', () => {
   });
 
   it('should merge managed-by-location and techdocs-ref when techdocs-ref is absolute', async () => {
-    const directoryPreparer = new DirectoryPreparer(mockConfig, logger);
+    const directoryPreparer = new DirectoryPreparer(
+      mockConfig,
+      logger,
+      mockUrlReader,
+    );
 
     const mockEntity = createMockEntity({
       'backstage.io/managed-by-location':
@@ -77,7 +89,11 @@ describe('directory preparer', () => {
   });
 
   it('should merge managed-by-location and techdocs-ref when managed-by-location is a git repository', async () => {
-    const directoryPreparer = new DirectoryPreparer(mockConfig, logger);
+    const directoryPreparer = new DirectoryPreparer(
+      mockConfig,
+      logger,
+      mockUrlReader,
+    );
 
     const mockEntity = createMockEntity({
       'backstage.io/managed-by-location':

packages/techdocs-common/src/stages/prepare/dir.ts

@@ -18,17 +18,19 @@ import { Entity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import path from 'path';
 import { parseReferenceAnnotation, checkoutGitRepository } from '../../helpers';
-import { InputError } from '@backstage/backend-common';
+import { UrlReader, InputError } from '@backstage/backend-common';
 import parseGitUrl from 'git-url-parse';
 import { Logger } from 'winston';
 
 export class DirectoryPreparer implements PreparerBase {
-  private readonly config: Config;
-  private readonly logger: Logger;
-
-  constructor(config: Config, logger: Logger) {
+  constructor(
+    private readonly config: Config,
+    private readonly logger: Logger,
+    private readonly reader: UrlReader,
+  ) {
     this.config = config;
     this.logger = logger;
+    this.reader = reader;
   }
 
   private async resolveManagedByLocationToDir(entity: Entity) {
@@ -41,9 +43,12 @@ export class DirectoryPreparer implements PreparerBase {
       `Building docs for entity with type 'dir' and managed-by-location '${type}'`,
     );
     switch (type) {
+      case 'url': {
+        const response = await this.reader.readTree(target);
+        return await response.dir();
+      }
       case 'github':
       case 'gitlab':
-      case 'url':
       case 'azure/api': {
         const parsedGitLocation = parseGitUrl(target);
         const repoLocation = await checkoutGitRepository(
@@ -56,7 +61,6 @@ export class DirectoryPreparer implements PreparerBase {
           path.join(repoLocation, parsedGitLocation.filepath),
         );
       }
-
       case 'file':
         return path.dirname(target);
       default:

packages/techdocs-common/src/stages/prepare/preparers.ts

@@ -35,17 +35,22 @@ export class Preparers implements PreparerBuilder {
   ): Promise<PreparerBuilder> {
     const preparers = new Preparers();
 
-    const directoryPreparer = new DirectoryPreparer(config, logger);
+    const urlPreparer = new UrlPreparer(reader, logger);
+    preparers.register('url', urlPreparer);
+
+    /**
+     * Dir preparer is a syntactic sugar for users to define techdocs-ref annotation.
+     * When using dir preparer, the docs will be fetched using URL Reader.
+     */
+    const directoryPreparer = new DirectoryPreparer(config, logger, reader);
     preparers.register('dir', directoryPreparer);
 
+    // Common git preparers will be deprecated soon.
     const commonGitPreparer = new CommonGitPreparer(config, logger);
     preparers.register('github', commonGitPreparer);
     preparers.register('gitlab', commonGitPreparer);
     preparers.register('azure/api', commonGitPreparer);
 
-    const urlPreparer = new UrlPreparer(reader, logger);
-    preparers.register('url', urlPreparer);
-
     return preparers;
   }
 

plugins/techdocs-backend/src/service/standaloneServer.ts

@@ -17,6 +17,7 @@
 import {
   createServiceBuilder,
   SingleHostDiscovery,
+  UrlReader,
 } from '@backstage/backend-common';
 import { Server } from 'http';
 import { Logger } from 'winston';
@@ -49,10 +50,18 @@ export async function startStandaloneServer(
     },
   });
   const discovery = SingleHostDiscovery.fromConfig(config);
+  const mockUrlReader: jest.Mocked<UrlReader> = {
+    read: jest.fn(),
+    readTree: jest.fn(),
+  };
 
   logger.debug('Creating application...');
   const preparers = new Preparers();
-  const directoryPreparer = new DirectoryPreparer(config, logger);
+  const directoryPreparer = new DirectoryPreparer(
+    config,
+    logger,
+    mockUrlReader,
+  );
   preparers.register('dir', directoryPreparer);
 
   const generators = new Generators();