feat(techdocs-common): add Azure Storage

.changeset/chilly-dodos-drop.md

@@ -0,0 +1,6 @@
+---
+'@backstage/techdocs-common': patch
+'@backstage/plugin-techdocs-backend': patch
+---
+
+1. Added option to use Azure Storage as a choice to store the static generated files for TechDocs.

app-config.yaml

@@ -80,7 +80,7 @@ techdocs:
   generators:
     techdocs: 'docker' # Alternatives - 'local'
   publisher:
-    type: 'local' # Alternatives - 'googleGcs' or 'awsS3'. Read documentation for using alternatives.
+    type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureStorage'. Read documentation for using alternatives.
 
 sentry:
   organization: my-company

docs/features/techdocs/README.md

@@ -108,12 +108,12 @@ providers are used.
 | GitLab                       | Yes ✅         |
 | GitLab Enterprise            | Yes ✅         |
 
-| File Storage Provider             | Support Status                                                    |
-| --------------------------------- | ----------------------------------------------------------------- |
-| Local Filesystem of Backstage app | Yes ✅                                                            |
-| Google Cloud Storage (GCS)        | Yes ✅                                                            |
-| Amazon Web Services (AWS) S3      | Yes ✅                                                            |
-| Azure Storage                     | No ❌ [#3938](https://github.com/backstage/backstage/issues/3938) |
+| File Storage Provider             | Support Status |
+| --------------------------------- | -------------- |
+| Local Filesystem of Backstage app | Yes ✅         |
+| Google Cloud Storage (GCS)        | Yes ✅         |
+| Amazon Web Services (AWS) S3      | Yes ✅         |
+| Azure Storage                     | Yes ✅         |
 
 [Reach out to us](#feedback) if you want to request more platforms.
 

docs/features/techdocs/configuration.md

@@ -84,4 +84,17 @@ techdocs:
       # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html
       region:
         $env: AWS_REGION
+
+    # Required when techdocs.publisher.type is set to 'azureStorage'. Skip otherwise.
+
+    azureStorage:
+      # An API key is required to write to a storage container.
+      credentials:
+        account:
+          $env: TECHDOCS_AZURE_STORAGE_ACCOUNT
+        accountKey:
+          $env: TECHDOCS_AZURE_STORAGE_ACCOUNT_KEY
+
+      # Azure Storage Container Name
+      containerName: 'techdocs-storage'
 ```

docs/features/techdocs/using-cloud-storage.md

@@ -195,3 +195,64 @@ Your Backstage app is now ready to use AWS S3 for TechDocs, to store and read
 the static generated documentation files. When you start the backend of the app,
 you should be able to see
 `techdocs info Successfully connected to the AWS S3 bucket` in the logs.
+
+## Configuring Azure Storage Container with TechDocs
+
+Follow the
+[official Azure Storage documentation](https://docs.microsoft.com/pt-br/javascript/api/@azure/storage-blob/?view=azure-node-latest)
+for the latest instructions on the following steps involving Azure Storage.
+
+**1. Set `techdocs.publisher.type` config in your `app-config.yaml`**
+
+Set `techdocs.publisher.type` to `'azureStorage'`.
+
+```yaml
+techdocs:
+  publisher:
+    type: 'azureStorage'
+```
+
+**2. Service account credentials**
+
+To get credentials, access the Azure Portal and go to "Settings > Access Keys",
+and get your Storage account name and Primary Key.
+
+```yaml
+techdocs:
+  publisher:
+    type: 'azureStorage'
+    azureStorage:
+      credentials:
+        account: 'account'
+        accountKey: 'accountKey'
+```
+
+**3. Azure Storage Container**
+
+Create a dedicated container for TechDocs sites. techdocs-backend will publish
+documentation to this container. TechDocs will fetch files from here to serve
+documentation in Backstage.
+
+To create a new container, access "Blob Service > Containers > New Container".
+
+Set the name of the container to
+`techdocs.publisher.azureStorage.containerName`.
+
+```yaml
+techdocs:
+  publisher:
+    type: 'azureStorage'
+    azureStorage:
+      credentials:
+        account: 'account'
+        accountKey: 'accountKey'
+      containerName: 'name-of-techdocs-storage-container'
+```
+
+**4. That's it!**
+
+Your Backstage app is now ready to use Azure Storage for TechDocs, to store and
+read the static generated documentation files. When you start the backend of the
+app, you should be able to see
+`techdocs info Successfully connected to the Azure Storage container` in the
+logs.

packages/techdocs-common/__mocks__/@azure/storage-blob.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs';
+
+export class BlockBlobClient {
+  private readonly blobName;
+
+  constructor(blobName: string) {
+    this.blobName = blobName;
+  }
+
+  uploadFile(source: string) {
+    return new Promise((resolve, reject) => {
+      if (!fs.existsSync(source)) {
+        reject('');
+      } else {
+        resolve('');
+      }
+    });
+  }
+
+  exists() {
+    return new Promise((resolve, reject) => {
+      if (fs.existsSync(this.blobName)) {
+        resolve(true);
+      } else {
+        reject({ message: 'The object doest not exist !' });
+      }
+    });
+  }
+}
+
+export class ContainerClient {
+  private readonly containerName;
+
+  constructor(containerName: string) {
+    this.containerName = containerName;
+  }
+
+  getProperties() {
+    return new Promise(resolve => {
+      resolve('');
+    });
+  }
+
+  getBlockBlobClient(blobName: string) {
+    return new BlockBlobClient(blobName);
+  }
+}
+
+export class BlobServiceClient {
+  private readonly url;
+  private readonly credential;
+
+  constructor(url: string, credential?: StorageSharedKeyCredential) {
+    this.url = url;
+    this.credential = credential;
+  }
+
+  getContainerClient(containerName: string) {
+    return new ContainerClient(containerName);
+  }
+}
+
+export class StorageSharedKeyCredential {
+  private readonly accountName;
+  private readonly accountKey;
+
+  constructor(accountName: string, accountKey: string) {
+    this.accountName = accountName;
+    this.accountKey = accountKey;
+  }
+}

packages/techdocs-common/package.json

@@ -36,6 +36,7 @@
     "url": "https://github.com/backstage/backstage/issues"
   },
   "dependencies": {
+    "@azure/storage-blob": "^12.3.0",
     "@aws-sdk/client-s3": "^3.1.0",
     "@backstage/backend-common": "^0.5.1",
     "@backstage/catalog-model": "^0.7.0",

packages/techdocs-common/src/stages/publish/azureStorage.test.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import mockFs from 'mock-fs';
+import { ConfigReader } from '@backstage/config';
+import { getVoidLogger } from '@backstage/backend-common';
+import { AzureStoragePublish } from './azureStorage';
+import { PublisherBase } from './types';
+import type { Entity } from '@backstage/catalog-model';
+
+const createMockEntity = (annotations = {}) => {
+  return {
+    apiVersion: 'version',
+    kind: 'TestKind',
+    metadata: {
+      name: 'test-component-name',
+      namespace: 'test-namespace',
+      annotations: {
+        ...annotations,
+      },
+    },
+  };
+};
+
+const getEntityRootDir = (entity: Entity) => {
+  const {
+    kind,
+    metadata: { namespace, name },
+  } = entity;
+  const entityRootDir = `${namespace}/${kind}/${name}`;
+  return entityRootDir;
+};
+
+const logger = getVoidLogger();
+jest.spyOn(logger, 'info').mockReturnValue(logger);
+jest.spyOn(logger, 'error').mockReturnValue(logger);
+
+let publisher: PublisherBase;
+
+beforeEach(async () => {
+  const mockConfig = new ConfigReader({
+    techdocs: {
+      requestUrl: 'http://localhost:7000',
+      publisher: {
+        type: 'azureStorage',
+        azureStorage: {
+          credentials: {
+            account: 'account',
+            accountKey: 'accountKey',
+          },
+          containerName: 'containerName',
+        },
+      },
+    },
+  });
+
+  publisher = await AzureStoragePublish.fromConfig(mockConfig, logger);
+});
+
+describe('AzureStoragePublish', () => {
+  describe('publish', () => {
+    it('should publish a directory', async () => {
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'index.html': '',
+          '404.html': '',
+          assets: {
+            'main.css': '',
+          },
+        },
+      });
+
+      expect(
+        await publisher.publish({
+          entity,
+          directory: entityRootDir,
+        }),
+      ).toBeUndefined();
+      mockFs.restore();
+    });
+
+    it('should fail to publish a directory', async () => {
+      const wrongPathToGeneratedDirectory = 'wrong/path/to/generatedDirectory';
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'index.html': '',
+          '404.html': '',
+          assets: {
+            'main.css': '',
+          },
+        },
+      });
+
+      await publisher
+        .publish({
+          entity,
+          directory: wrongPathToGeneratedDirectory,
+        })
+        .catch(error =>
+          expect(error).toEqual(
+            new Error(
+              `Unable to upload file(s) to Azure Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
+            ),
+          ),
+        );
+      mockFs.restore();
+    });
+  });
+
+  describe('hasDocsBeenGenerated', () => {
+    it('should return true if docs has been generated', async () => {
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'index.html': 'file-content',
+        },
+      });
+
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+      mockFs.restore();
+    });
+
+    it('should return false if docs has not been generated', async () => {
+      const entity = createMockEntity();
+
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false);
+    });
+  });
+});

packages/techdocs-common/src/stages/publish/azureStorage.ts

@@ -0,0 +1,223 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import path from 'path';
+import express from 'express';
+import {
+  BlobServiceClient,
+  BlobUploadCommonResponse,
+  StorageSharedKeyCredential,
+} from '@azure/storage-blob';
+import { Logger } from 'winston';
+import { Entity, EntityName } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
+import { PublisherBase, PublishRequest } from './types';
+
+export class AzureStoragePublish implements PublisherBase {
+  static async fromConfig(
+    config: Config,
+    logger: Logger,
+  ): Promise<PublisherBase> {
+    let account = '';
+    let accountKey = '';
+    let containerName = '';
+    try {
+      account = config.getString(
+        'techdocs.publisher.azureStorage.credentials.account',
+      );
+      accountKey = config.getString(
+        'techdocs.publisher.azureStorage.credentials.accountKey',
+      );
+      containerName = config.getString(
+        'techdocs.publisher.azureStorage.containerName',
+      );
+    } catch (error) {
+      throw new Error(
+        "Since techdocs.publisher.type is set to 'azureStorage' in your app config, " +
+          'credentials and containerName are required in techdocs.publisher.azureStorage ' +
+          'required to authenticate with Azure Storage.',
+      );
+    }
+
+    const credential = new StorageSharedKeyCredential(account, accountKey);
+    const storageClient = new BlobServiceClient(
+      `https://${account}.blob.core.windows.net`,
+      credential,
+    );
+
+    await storageClient
+      .getContainerClient(containerName)
+      .getProperties()
+      .then(() => {
+        logger.info(
+          `Successfully connected to the Azure Storage container ${containerName}.`,
+        );
+      })
+      .catch(reason => {
+        logger.error(
+          `Could not retrieve metadata about the Azure Storage container ${containerName}. ` +
+            'Make sure the Azure project and the container exists and the access key located at the path ' +
+            "techdocs.publisher.azureStorage.credentials defined in app config has the role 'Storage Object Creator'. " +
+            'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
+        );
+        throw new Error(`from Azure Storage client library: ${reason.message}`);
+      });
+
+    return new AzureStoragePublish(storageClient, containerName, logger);
+  }
+
+  constructor(
+    private readonly storageClient: BlobServiceClient,
+    private readonly containerName: string,
+    private readonly logger: Logger,
+  ) {
+    this.storageClient = storageClient;
+    this.containerName = containerName;
+    this.logger = logger;
+  }
+
+  /**
+   * Upload all the files from the generated `directory` to the Azure Storage container.
+   * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
+   */
+  async publish({ entity, directory }: PublishRequest): Promise<void> {
+    try {
+      // Note: Azure Storage manages creation of parent directories if they do not exist.
+      // So collecting path of only the files is good enough.
+      const allFilesToUpload = await getFileTreeRecursively(directory);
+
+      const uploadPromises: Array<Promise<BlobUploadCommonResponse>> = [];
+      allFilesToUpload.forEach(filePath => {
+        // Remove the absolute path prefix of the source directory
+        // Path of all files to upload, relative to the root of the source directory
+        // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
+        const relativeFilePath = filePath.replace(`${directory}/`, '');
+        const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+        const destination = path.normalize(
+          `${entityRootDir}/${relativeFilePath}`,
+        ); // Azure Storage Container file relative path
+        // TODO: Upload in chunks of ~10 files instead of all files at once.
+        uploadPromises.push(
+          this.storageClient
+            .getContainerClient(this.containerName)
+            .getBlockBlobClient(destination)
+            .uploadFile(filePath),
+        );
+      });
+
+      await Promise.all(uploadPromises).then(() => {
+        this.logger.info(
+          `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
+        );
+      });
+      return;
+    } catch (e) {
+      const errorMessage = `Unable to upload file(s) to Azure Storage. Error ${e.message}`;
+      this.logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+  }
+
+  download(containerName: string, path: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const fileStreamChunks: Array<any> = [];
+      this.storageClient
+        .getContainerClient(containerName)
+        .getBlockBlobClient(path)
+        .download()
+        .then(res => {
+          const body = res.readableStreamBody;
+          if (!body) {
+            reject(new Error(`Unable to parse the response data`));
+            return;
+          }
+          body
+            .on('error', e => {
+              this.logger.error(e.message);
+              reject(e.message);
+            })
+            .on('data', chunk => {
+              fileStreamChunks.push(chunk);
+            })
+            .on('end', () => {
+              resolve(Buffer.concat(fileStreamChunks).toString());
+            });
+        });
+    });
+  }
+
+  async fetchTechDocsMetadata(entityName: EntityName): Promise<string> {
+    const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
+    try {
+      return this.download(
+        this.containerName,
+        `${entityRootDir}/techdocs_metadata.json`,
+      );
+    } catch (e) {
+      this.logger.error(e.message);
+      throw e;
+    }
+  }
+
+  /**
+   * Express route middleware to serve static files on a route in techdocs-backend.
+   */
+  docsRouter(): express.Handler {
+    return (req, res) => {
+      // Trim the leading forward slash
+      // filePath example - /default/Component/documented-component/index.html
+      const filePath = req.path.replace(/^\//, '');
+      // Files with different extensions (CSS, HTML) need to be served with different headers
+      const fileExtension = path.extname(filePath);
+      const responseHeaders = getHeadersForFileExtension(fileExtension);
+
+      try {
+        this.download(this.containerName, filePath).then(fileContent => {
+          // Inject response headers
+          for (const [headerKey, headerValue] of Object.entries(
+            responseHeaders,
+          )) {
+            res.setHeader(headerKey, headerValue);
+          }
+          res.send(fileContent);
+        });
+      } catch (e) {
+        this.logger.error(e.message);
+        res.status(404).send(e.message);
+      }
+    };
+  }
+
+  /**
+   * A helper function which checks if index.html of an Entity's docs site is available. This
+   * can be used to verify if there are any pre-generated docs available to serve.
+   */
+  async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
+    return new Promise(resolve => {
+      const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+      this.storageClient
+        .getContainerClient(this.containerName)
+        .getBlockBlobClient(`${entityRootDir}/index.html`)
+        .exists()
+        .then((response: boolean) => {
+          resolve(response);
+        })
+        .catch(() => {
+          resolve(false);
+        });
+    });
+  }
+}

packages/techdocs-common/src/stages/publish/publish.test.ts

@@ -22,6 +22,7 @@ import { Publisher } from './publish';
 import { LocalPublish } from './local';
 import { GoogleGCSPublish } from './googleStorage';
 import { AwsS3Publish } from './awsS3';
+import { AzureStoragePublish } from './azureStorage';
 
 const logger = getVoidLogger();
 const discovery: jest.Mocked<PluginEndpointDiscovery> = {
@@ -105,4 +106,28 @@ describe('Publisher', () => {
     });
     expect(publisher).toBeInstanceOf(AwsS3Publish);
   });
+
+  it('should create Azure Storage publisher from config', async () => {
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        requestUrl: 'http://localhost:7000',
+        publisher: {
+          type: 'azureStorage',
+          azureStorage: {
+            credentials: {
+              account: 'account',
+              accountKey: 'accountKey',
+            },
+            containerName: 'containerName',
+          },
+        },
+      },
+    });
+
+    const publisher = await Publisher.fromConfig(mockConfig, {
+      logger,
+      discovery,
+    });
+    expect(publisher).toBeInstanceOf(AzureStoragePublish);
+  });
 });

packages/techdocs-common/src/stages/publish/publish.ts

@@ -21,6 +21,7 @@ import { PublisherType, PublisherBase } from './types';
 import { LocalPublish } from './local';
 import { GoogleGCSPublish } from './googleStorage';
 import { AwsS3Publish } from './awsS3';
+import { AzureStoragePublish } from './azureStorage';
 
 type factoryOptions = {
   logger: Logger;
@@ -47,6 +48,9 @@ export class Publisher {
       case 'awsS3':
         logger.info('Creating AWS S3 Bucket publisher for TechDocs');
         return AwsS3Publish.fromConfig(config, logger);
+      case 'azureStorage':
+        logger.info('Creating Azure Storage Container publisher for TechDocs');
+        return AzureStoragePublish.fromConfig(config, logger);
       case 'local':
         logger.info('Creating Local publisher for TechDocs');
         return new LocalPublish(config, logger, discovery);

packages/techdocs-common/src/stages/publish/types.ts

@@ -19,7 +19,7 @@ import express from 'express';
 /**
  * Key for all the different types of TechDocs publishers that are supported.
  */
-export type PublisherType = 'local' | 'googleGcs' | 'awsS3';
+export type PublisherType = 'local' | 'googleGcs' | 'awsS3' | 'azureStorage';
 
 export type PublishRequest = {
   entity: Entity;

plugins/techdocs-backend/src/service/router.ts

@@ -148,6 +148,7 @@ export async function createRouter({
           }
           break;
         case 'awsS3':
+        case 'azureStorage':
         case 'googleGcs':
           // This block should be valid for all external storage implementations. So no need to duplicate in future,
           // add the publisher type in the list here.

plugins/techdocs/config.d.ts

@@ -114,6 +114,45 @@ export interface Config {
             region?: string;
           };
         }
+      | {
+          /**
+           * attr: 'type' - accepts a string value
+           * e.g. type: 'azureStorage'
+           * alternatives: 'azureStorage' etc.
+           * @see http://backstage.io/docs/features/techdocs/configuration
+           */
+          type: 'azureStorage';
+
+          /**
+           * azureStorage required when 'type' is set to azureStorage
+           */
+          azureStorage?: {
+            /**
+             * Credentials used to access a storage container
+             * @visibility secret
+             */
+            credentials: {
+              /**
+               * Account access name
+               * attr: 'account' - accepts a string value
+               * @visibility secret
+               */
+              account: string;
+              /**
+               * Account secret primary key
+               * attr: 'accountKey' - accepts a string value
+               * @visibility secret
+               */
+              accountKey: string;
+            };
+            /**
+             * Cloud Storage Container Name
+             * attr: 'containerName' - accepts a string value
+             * @visibility backend
+             */
+            containerName: string;
+          };
+        }
       | {
           /**
            * attr: 'type' - accepts a string value