Merge pull request #4017 from viavarejo/feature/techdocs-azure-storage

packages/techdocs-common/__mocks__/@azure/identity.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export class DefaultAzureCredential {
+  /**
+   * Creates an instance of the DefaultAzureCredential class.
+   */
+}

packages/techdocs-common/__mocks__/@azure/storage-blob.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs';
+
+export class BlockBlobClient {
+  private readonly blobName;
+
+  constructor(blobName: string) {
+    this.blobName = blobName;
+  }
+
+  uploadFile(source: string) {
+    return new Promise((resolve, reject) => {
+      if (!fs.existsSync(source)) {
+        reject('');
+      } else {
+        resolve('');
+      }
+    });
+  }
+
+  exists() {
+    return new Promise((resolve, reject) => {
+      if (fs.existsSync(this.blobName)) {
+        resolve(true);
+      } else {
+        reject({ message: 'The object doest not exist !' });
+      }
+    });
+  }
+}
+
+export class ContainerClient {
+  private readonly containerName;
+
+  constructor(containerName: string) {
+    this.containerName = containerName;
+  }
+
+  getProperties() {
+    return new Promise(resolve => {
+      resolve('');
+    });
+  }
+
+  getBlockBlobClient(blobName: string) {
+    return new BlockBlobClient(blobName);
+  }
+}
+
+export class BlobServiceClient {
+  private readonly url;
+  private readonly credential;
+
+  constructor(url: string, credential?: StorageSharedKeyCredential) {
+    this.url = url;
+    this.credential = credential;
+  }
+
+  getContainerClient(containerName: string) {
+    return new ContainerClient(containerName);
+  }
+}
+
+export class StorageSharedKeyCredential {
+  private readonly accountName;
+  private readonly accountKey;
+
+  constructor(accountName: string, accountKey: string) {
+    this.accountName = accountName;
+    this.accountKey = accountKey;
+  }
+}

packages/techdocs-common/package.json

@@ -37,6 +37,8 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1.0",
+    "@azure/identity": "^1.2.2",
+    "@azure/storage-blob": "^12.4.0",
     "@backstage/backend-common": "^0.5.1",
     "@backstage/catalog-model": "^0.7.0",
     "@backstage/config": "^0.1.2",

packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import mockFs from 'mock-fs';
+import { ConfigReader } from '@backstage/config';
+import { getVoidLogger } from '@backstage/backend-common';
+import { AzureBlobStoragePublish } from './azureBlobStorage';
+import { PublisherBase } from './types';
+import type { Entity } from '@backstage/catalog-model';
+
+const createMockEntity = (annotations = {}) => {
+  return {
+    apiVersion: 'version',
+    kind: 'TestKind',
+    metadata: {
+      name: 'test-component-name',
+      namespace: 'test-namespace',
+      annotations: {
+        ...annotations,
+      },
+    },
+  };
+};
+
+const getEntityRootDir = (entity: Entity) => {
+  const {
+    kind,
+    metadata: { namespace, name },
+  } = entity;
+  const entityRootDir = `${namespace}/${kind}/${name}`;
+  return entityRootDir;
+};
+
+const logger = getVoidLogger();
+jest.spyOn(logger, 'info').mockReturnValue(logger);
+jest.spyOn(logger, 'error').mockReturnValue(logger);
+
+let publisher: PublisherBase;
+
+beforeEach(async () => {
+  const mockConfig = new ConfigReader({
+    techdocs: {
+      requestUrl: 'http://localhost:7000',
+      publisher: {
+        type: 'azureBlobStorage',
+        azureBlobStorage: {
+          credentials: {
+            accountName: 'accountName',
+            accountKey: 'accountKey',
+          },
+          containerName: 'containerName',
+        },
+      },
+    },
+  });
+
+  publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
+});
+
+describe('AzureBlobStoragePublish', () => {
+  describe('publish', () => {
+    it('should publish a directory', async () => {
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'index.html': '',
+          '404.html': '',
+          assets: {
+            'main.css': '',
+          },
+        },
+      });
+
+      expect(
+        await publisher.publish({
+          entity,
+          directory: entityRootDir,
+        }),
+      ).toBeUndefined();
+      mockFs.restore();
+    });
+
+    it('should fail to publish a directory', async () => {
+      const wrongPathToGeneratedDirectory = 'wrong/path/to/generatedDirectory';
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'index.html': '',
+          '404.html': '',
+          assets: {
+            'main.css': '',
+          },
+        },
+      });
+
+      await publisher
+        .publish({
+          entity,
+          directory: wrongPathToGeneratedDirectory,
+        })
+        .catch(error =>
+          expect(error).toEqual(
+            new Error(
+              `Unable to upload file(s) to Azure Blob Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
+            ),
+          ),
+        );
+      mockFs.restore();
+    });
+  });
+
+  describe('hasDocsBeenGenerated', () => {
+    it('should return true if docs has been generated', async () => {
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'index.html': 'file-content',
+        },
+      });
+
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+      mockFs.restore();
+    });
+
+    it('should return false if docs has not been generated', async () => {
+      const entity = createMockEntity();
+
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false);
+    });
+  });
+});

packages/techdocs-common/src/stages/publish/azureBlobStorage.ts

@@ -0,0 +1,259 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import path from 'path';
+import express from 'express';
+import {
+  BlobServiceClient,
+  BlobUploadCommonResponse,
+  StorageSharedKeyCredential,
+} from '@azure/storage-blob';
+import { DefaultAzureCredential } from '@azure/identity';
+import { Logger } from 'winston';
+import { Entity, EntityName } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
+import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
+import limiterFactory from 'p-limit';
+import JSON5 from 'json5';
+
+// The number of batches that may be ongoing at the same time.
+const BATCH_CONCURRENCY = 3;
+
+export class AzureBlobStoragePublish implements PublisherBase {
+  static async fromConfig(
+    config: Config,
+    logger: Logger,
+  ): Promise<PublisherBase> {
+    let containerName = '';
+    try {
+      containerName = config.getString(
+        'techdocs.publisher.azureBlobStorage.containerName',
+      );
+    } catch (error) {
+      throw new Error(
+        "Since techdocs.publisher.type is set to 'azureBlobStorage' in your app config, " +
+          'techdocs.publisher.azureBlobStorage.containerName is required.',
+      );
+    }
+
+    let accountName = '';
+    try {
+      accountName = config.getString(
+        'techdocs.publisher.azureBlobStorage.credentials.accountName',
+      );
+    } catch (error) {
+      throw new Error(
+        "Since techdocs.publisher.type is set to 'azureBlobStorage' in your app config, " +
+          'techdocs.publisher.azureBlobStorage.credentials.accountName is required.',
+      );
+    }
+
+    // Credentials is an optional config. If missing, default Azure Blob Storage environment variables will be used.
+    // https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-app
+    const accountKey = config.getOptionalString(
+      'techdocs.publisher.azureBlobStorage.credentials.accountKey',
+    );
+
+    let credential;
+    if (accountKey) {
+      credential = new StorageSharedKeyCredential(accountName, accountKey);
+    } else {
+      credential = new DefaultAzureCredential();
+    }
+
+    const storageClient = new BlobServiceClient(
+      `https://${accountName}.blob.core.windows.net`,
+      credential,
+    );
+
+    await storageClient
+      .getContainerClient(containerName)
+      .getProperties()
+      .then(() => {
+        logger.info(
+          `Successfully connected to the Azure Blob Storage container ${containerName}.`,
+        );
+      })
+      .catch(reason => {
+        logger.error(
+          `Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` +
+            'Make sure that the Azure project and container exist and the access key is setup correctly ' +
+            'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' +
+            'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
+        );
+        throw new Error(
+          `from Azure Blob Storage client library: ${reason.message}`,
+        );
+      });
+
+    return new AzureBlobStoragePublish(storageClient, containerName, logger);
+  }
+
+  constructor(
+    private readonly storageClient: BlobServiceClient,
+    private readonly containerName: string,
+    private readonly logger: Logger,
+  ) {
+    this.storageClient = storageClient;
+    this.containerName = containerName;
+    this.logger = logger;
+  }
+
+  /**
+   * Upload all the files from the generated `directory` to the Azure Blob Storage container.
+   * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
+   */
+  async publish({ entity, directory }: PublishRequest): Promise<void> {
+    try {
+      // Note: Azure Blob Storage manages creation of parent directories if they do not exist.
+      // So collecting path of only the files is good enough.
+      const allFilesToUpload = await getFileTreeRecursively(directory);
+
+      const uploadPromises: Array<Promise<BlobUploadCommonResponse>> = [];
+
+      // Bound the number of concurrent batches. We want a bit of concurrency for
+      // performance reasons, but not so much that we starve the connection pool
+      // or start thrashing.
+      const limiter = limiterFactory(BATCH_CONCURRENCY);
+
+      const promises = allFilesToUpload.map(filePath => {
+        // Remove the absolute path prefix of the source directory
+        // Path of all files to upload, relative to the root of the source directory
+        // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
+        const relativeFilePath = filePath.replace(`${directory}/`, '');
+        const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+        const destination = path.normalize(
+          `${entityRootDir}/${relativeFilePath}`,
+        ); // Azure Blob Storage Container file relative path
+
+        return limiter(async () => {
+          await uploadPromises.push(
+            this.storageClient
+              .getContainerClient(this.containerName)
+              .getBlockBlobClient(destination)
+              .uploadFile(filePath),
+          );
+        });
+      });
+
+      await Promise.all(promises).then(() => {
+        this.logger.info(
+          `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
+        );
+      });
+      return;
+    } catch (e) {
+      const errorMessage = `Unable to upload file(s) to Azure Blob Storage. Error ${e.message}`;
+      this.logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+  }
+
+  private download(containerName: string, path: string): Promise<Buffer> {
+    return new Promise((resolve, reject) => {
+      const fileStreamChunks: Array<any> = [];
+      this.storageClient
+        .getContainerClient(containerName)
+        .getBlockBlobClient(path)
+        .download()
+        .then(res => {
+          const body = res.readableStreamBody;
+          if (!body) {
+            reject(new Error(`Unable to parse the response data`));
+            return;
+          }
+          body
+            .on('error', e => {
+              this.logger.error(e.message);
+              reject(e.message);
+            })
+            .on('data', chunk => {
+              fileStreamChunks.push(chunk);
+            })
+            .on('end', () => {
+              resolve(Buffer.concat(fileStreamChunks));
+            });
+        });
+    });
+  }
+
+  async fetchTechDocsMetadata(
+    entityName: EntityName,
+  ): Promise<TechDocsMetadata> {
+    const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
+    try {
+      return await new Promise<TechDocsMetadata>(resolve => {
+        const download = this.download(
+          this.containerName,
+          `${entityRootDir}/techdocs_metadata.json`,
+        );
+        resolve(JSON5.parse(download.toString()));
+      });
+    } catch (e) {
+      this.logger.error(e.message);
+      throw e;
+    }
+  }
+
+  /**
+   * Express route middleware to serve static files on a route in techdocs-backend.
+   */
+  docsRouter(): express.Handler {
+    return (req, res) => {
+      // Trim the leading forward slash
+      // filePath example - /default/Component/documented-component/index.html
+      const filePath = req.path.replace(/^\//, '');
+      // Files with different extensions (CSS, HTML) need to be served with different headers
+      const fileExtension = path.extname(filePath);
+      const responseHeaders = getHeadersForFileExtension(fileExtension);
+
+      try {
+        this.download(this.containerName, filePath).then(fileContent => {
+          // Inject response headers
+          for (const [headerKey, headerValue] of Object.entries(
+            responseHeaders,
+          )) {
+            res.setHeader(headerKey, headerValue);
+          }
+          res.send(fileContent);
+        });
+      } catch (e) {
+        this.logger.error(e.message);
+        res.status(404).send(e.message);
+      }
+    };
+  }
+
+  /**
+   * A helper function which checks if index.html of an Entity's docs site is available. This
+   * can be used to verify if there are any pre-generated docs available to serve.
+   */
+  async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
+    return new Promise(resolve => {
+      const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+      this.storageClient
+        .getContainerClient(this.containerName)
+        .getBlockBlobClient(`${entityRootDir}/index.html`)
+        .exists()
+        .then((response: boolean) => {
+          resolve(response);
+        })
+        .catch(() => {
+          resolve(false);
+        });
+    });
+  }
+}

packages/techdocs-common/src/stages/publish/publish.test.ts

@@ -22,6 +22,7 @@ import { Publisher } from './publish';
 import { LocalPublish } from './local';
 import { GoogleGCSPublish } from './googleStorage';
 import { AwsS3Publish } from './awsS3';
+import { AzureBlobStoragePublish } from './azureBlobStorage';
 
 const logger = getVoidLogger();
 const discovery: jest.Mocked<PluginEndpointDiscovery> = {
@@ -30,6 +31,10 @@ const discovery: jest.Mocked<PluginEndpointDiscovery> = {
 };
 
 describe('Publisher', () => {
+  beforeEach(() => {
+    jest.resetModules(); // clear the cache
+  });
+
   it('should create local publisher by default', async () => {
     const mockConfig = new ConfigReader({
       techdocs: {
@@ -105,4 +110,55 @@ describe('Publisher', () => {
     });
     expect(publisher).toBeInstanceOf(AwsS3Publish);
   });
+
+  it('should create Azure Blob Storage publisher from config', async () => {
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        requestUrl: 'http://localhost:7000',
+        publisher: {
+          type: 'azureBlobStorage',
+          azureBlobStorage: {
+            credentials: {
+              accountName: 'accountName',
+              accountKey: 'accountKey',
+            },
+            containerName: 'containerName',
+          },
+        },
+      },
+    });
+
+    const publisher = await Publisher.fromConfig(mockConfig, {
+      logger,
+      discovery,
+    });
+    expect(publisher).toBeInstanceOf(AzureBlobStoragePublish);
+  });
+
+  it('should create Azure Blob Storage publisher from environment variables', async () => {
+    process.env.AZURE_TENANT_ID = 'AZURE_TENANT_ID';
+    process.env.AZURE_CLIENT_ID = 'AZURE_CLIENT_ID';
+    process.env.AZURE_CLIENT_SECRET = 'AZURE_CLIENT_SECRET';
+
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        requestUrl: 'http://localhost:7000',
+        publisher: {
+          type: 'azureBlobStorage',
+          azureBlobStorage: {
+            credentials: {
+              accountName: 'accountName',
+            },
+            containerName: 'containerName',
+          },
+        },
+      },
+    });
+
+    const publisher = await Publisher.fromConfig(mockConfig, {
+      logger,
+      discovery,
+    });
+    expect(publisher).toBeInstanceOf(AzureBlobStoragePublish);
+  });
 });

packages/techdocs-common/src/stages/publish/publish.ts

@@ -21,6 +21,7 @@ import { PublisherType, PublisherBase } from './types';
 import { LocalPublish } from './local';
 import { GoogleGCSPublish } from './googleStorage';
 import { AwsS3Publish } from './awsS3';
+import { AzureBlobStoragePublish } from './azureBlobStorage';
 
 type factoryOptions = {
   logger: Logger;
@@ -47,6 +48,11 @@ export class Publisher {
       case 'awsS3':
         logger.info('Creating AWS S3 Bucket publisher for TechDocs');
         return AwsS3Publish.fromConfig(config, logger);
+      case 'azureBlobStorage':
+        logger.info(
+          'Creating Azure Blob Storage Container publisher for TechDocs',
+        );
+        return AzureBlobStoragePublish.fromConfig(config, logger);
       case 'local':
         logger.info('Creating Local publisher for TechDocs');
         return new LocalPublish(config, logger, discovery);

packages/techdocs-common/src/stages/publish/types.ts

@@ -19,7 +19,11 @@ import express from 'express';
 /**
  * Key for all the different types of TechDocs publishers that are supported.
  */
-export type PublisherType = 'local' | 'googleGcs' | 'awsS3';
+export type PublisherType =
+  | 'local'
+  | 'googleGcs'
+  | 'awsS3'
+  | 'azureBlobStorage';
 
 export type PublishRequest = {
   entity: Entity;