From c777df180afc3a6bede6d7d7febc0dea1902b396 Mon Sep 17 00:00:00 2001 From: vitorgrenzel Date: Wed, 13 Jan 2021 09:20:58 -0300 Subject: [PATCH 01/10] feat(techdocs-common): add Azure Storage --- .changeset/chilly-dodos-drop.md | 6 + app-config.yaml | 2 +- docs/features/techdocs/README.md | 12 +- docs/features/techdocs/configuration.md | 13 + docs/features/techdocs/using-cloud-storage.md | 61 +++++ .../__mocks__/@azure/storage-blob.ts | 86 +++++++ packages/techdocs-common/package.json | 1 + .../src/stages/publish/azureStorage.test.ts | 149 ++++++++++++ .../src/stages/publish/azureStorage.ts | 223 ++++++++++++++++++ .../src/stages/publish/publish.test.ts | 25 ++ .../src/stages/publish/publish.ts | 4 + .../src/stages/publish/types.ts | 2 +- .../techdocs-backend/src/service/router.ts | 1 + plugins/techdocs/config.d.ts | 39 +++ 14 files changed, 616 insertions(+), 8 deletions(-) create mode 100644 .changeset/chilly-dodos-drop.md create mode 100644 packages/techdocs-common/__mocks__/@azure/storage-blob.ts create mode 100644 packages/techdocs-common/src/stages/publish/azureStorage.test.ts create mode 100644 packages/techdocs-common/src/stages/publish/azureStorage.ts diff --git a/.changeset/chilly-dodos-drop.md b/.changeset/chilly-dodos-drop.md new file mode 100644 index 0000000000..b262c14e4d --- /dev/null +++ b/.changeset/chilly-dodos-drop.md @@ -0,0 +1,6 @@ +--- +'@backstage/techdocs-common': patch +'@backstage/plugin-techdocs-backend': patch +--- + +1. Added option to use Azure Storage as a choice to store the static generated files for TechDocs. diff --git a/app-config.yaml b/app-config.yaml index c150ef5902..09d2e75253 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -80,7 +80,7 @@ techdocs: generators: techdocs: 'docker' # Alternatives - 'local' publisher: - type: 'local' # Alternatives - 'googleGcs' or 'awsS3'. Read documentation for using alternatives. + type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureStorage'. Read documentation for using alternatives. sentry: organization: my-company diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index c43d6d978d..24b4410092 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -108,12 +108,12 @@ providers are used. | GitLab | Yes ✅ | | GitLab Enterprise | Yes ✅ | -| File Storage Provider | Support Status | -| --------------------------------- | ----------------------------------------------------------------- | -| Local Filesystem of Backstage app | Yes ✅ | -| Google Cloud Storage (GCS) | Yes ✅ | -| Amazon Web Services (AWS) S3 | Yes ✅ | -| Azure Storage | No ❌ [#3938](https://github.com/backstage/backstage/issues/3938) | +| File Storage Provider | Support Status | +| --------------------------------- | -------------- | +| Local Filesystem of Backstage app | Yes ✅ | +| Google Cloud Storage (GCS) | Yes ✅ | +| Amazon Web Services (AWS) S3 | Yes ✅ | +| Azure Storage | Yes ✅ | [Reach out to us](#feedback) if you want to request more platforms. diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 1580abe69e..8d5906b18f 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -84,4 +84,17 @@ techdocs: # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html region: $env: AWS_REGION + + # Required when techdocs.publisher.type is set to 'azureStorage'. Skip otherwise. + + azureStorage: + # An API key is required to write to a storage container. + credentials: + account: + $env: TECHDOCS_AZURE_STORAGE_ACCOUNT + accountKey: + $env: TECHDOCS_AZURE_STORAGE_ACCOUNT_KEY + + # Azure Storage Container Name + containerName: 'techdocs-storage' ``` diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 21206dede5..2d906ca1b9 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -195,3 +195,64 @@ Your Backstage app is now ready to use AWS S3 for TechDocs, to store and read the static generated documentation files. When you start the backend of the app, you should be able to see `techdocs info Successfully connected to the AWS S3 bucket` in the logs. + +## Configuring Azure Storage Container with TechDocs + +Follow the +[official Azure Storage documentation](https://docs.microsoft.com/pt-br/javascript/api/@azure/storage-blob/?view=azure-node-latest) +for the latest instructions on the following steps involving Azure Storage. + +**1. Set `techdocs.publisher.type` config in your `app-config.yaml`** + +Set `techdocs.publisher.type` to `'azureStorage'`. + +```yaml +techdocs: + publisher: + type: 'azureStorage' +``` + +**2. Service account credentials** + +To get credentials, access the Azure Portal and go to "Settings > Access Keys", +and get your Storage account name and Primary Key. + +```yaml +techdocs: + publisher: + type: 'azureStorage' + azureStorage: + credentials: + account: 'account' + accountKey: 'accountKey' +``` + +**3. Azure Storage Container** + +Create a dedicated container for TechDocs sites. techdocs-backend will publish +documentation to this container. TechDocs will fetch files from here to serve +documentation in Backstage. + +To create a new container, access "Blob Service > Containers > New Container". + +Set the name of the container to +`techdocs.publisher.azureStorage.containerName`. + +```yaml +techdocs: + publisher: + type: 'azureStorage' + azureStorage: + credentials: + account: 'account' + accountKey: 'accountKey' + containerName: 'name-of-techdocs-storage-container' +``` + +**4. That's it!** + +Your Backstage app is now ready to use Azure Storage for TechDocs, to store and +read the static generated documentation files. When you start the backend of the +app, you should be able to see +`techdocs info Successfully connected to the Azure Storage container` in the +logs. diff --git a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts new file mode 100644 index 0000000000..60a705f6a7 --- /dev/null +++ b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import fs from 'fs'; + +export class BlockBlobClient { + private readonly blobName; + + constructor(blobName: string) { + this.blobName = blobName; + } + + uploadFile(source: string) { + return new Promise((resolve, reject) => { + if (!fs.existsSync(source)) { + reject(''); + } else { + resolve(''); + } + }); + } + + exists() { + return new Promise((resolve, reject) => { + if (fs.existsSync(this.blobName)) { + resolve(true); + } else { + reject({ message: 'The object doest not exist !' }); + } + }); + } +} + +export class ContainerClient { + private readonly containerName; + + constructor(containerName: string) { + this.containerName = containerName; + } + + getProperties() { + return new Promise(resolve => { + resolve(''); + }); + } + + getBlockBlobClient(blobName: string) { + return new BlockBlobClient(blobName); + } +} + +export class BlobServiceClient { + private readonly url; + private readonly credential; + + constructor(url: string, credential?: StorageSharedKeyCredential) { + this.url = url; + this.credential = credential; + } + + getContainerClient(containerName: string) { + return new ContainerClient(containerName); + } +} + +export class StorageSharedKeyCredential { + private readonly accountName; + private readonly accountKey; + + constructor(accountName: string, accountKey: string) { + this.accountName = accountName; + this.accountKey = accountKey; + } +} diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 64c5c19425..09893bd38e 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -36,6 +36,7 @@ "url": "https://github.com/backstage/backstage/issues" }, "dependencies": { + "@azure/storage-blob": "^12.3.0", "@aws-sdk/client-s3": "^3.1.0", "@backstage/backend-common": "^0.5.1", "@backstage/catalog-model": "^0.7.0", diff --git a/packages/techdocs-common/src/stages/publish/azureStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureStorage.test.ts new file mode 100644 index 0000000000..05b5441bff --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/azureStorage.test.ts @@ -0,0 +1,149 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import mockFs from 'mock-fs'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { AzureStoragePublish } from './azureStorage'; +import { PublisherBase } from './types'; +import type { Entity } from '@backstage/catalog-model'; + +const createMockEntity = (annotations = {}) => { + return { + apiVersion: 'version', + kind: 'TestKind', + metadata: { + name: 'test-component-name', + namespace: 'test-namespace', + annotations: { + ...annotations, + }, + }, + }; +}; + +const getEntityRootDir = (entity: Entity) => { + const { + kind, + metadata: { namespace, name }, + } = entity; + const entityRootDir = `${namespace}/${kind}/${name}`; + return entityRootDir; +}; + +const logger = getVoidLogger(); +jest.spyOn(logger, 'info').mockReturnValue(logger); +jest.spyOn(logger, 'error').mockReturnValue(logger); + +let publisher: PublisherBase; + +beforeEach(async () => { + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'azureStorage', + azureStorage: { + credentials: { + account: 'account', + accountKey: 'accountKey', + }, + containerName: 'containerName', + }, + }, + }, + }); + + publisher = await AzureStoragePublish.fromConfig(mockConfig, logger); +}); + +describe('AzureStoragePublish', () => { + describe('publish', () => { + it('should publish a directory', async () => { + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'index.html': '', + '404.html': '', + assets: { + 'main.css': '', + }, + }, + }); + + expect( + await publisher.publish({ + entity, + directory: entityRootDir, + }), + ).toBeUndefined(); + mockFs.restore(); + }); + + it('should fail to publish a directory', async () => { + const wrongPathToGeneratedDirectory = 'wrong/path/to/generatedDirectory'; + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'index.html': '', + '404.html': '', + assets: { + 'main.css': '', + }, + }, + }); + + await publisher + .publish({ + entity, + directory: wrongPathToGeneratedDirectory, + }) + .catch(error => + expect(error).toEqual( + new Error( + `Unable to upload file(s) to Azure Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`, + ), + ), + ); + mockFs.restore(); + }); + }); + + describe('hasDocsBeenGenerated', () => { + it('should return true if docs has been generated', async () => { + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'index.html': 'file-content', + }, + }); + + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + mockFs.restore(); + }); + + it('should return false if docs has not been generated', async () => { + const entity = createMockEntity(); + + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false); + }); + }); +}); diff --git a/packages/techdocs-common/src/stages/publish/azureStorage.ts b/packages/techdocs-common/src/stages/publish/azureStorage.ts new file mode 100644 index 0000000000..bc9a6825d9 --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/azureStorage.ts @@ -0,0 +1,223 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import path from 'path'; +import express from 'express'; +import { + BlobServiceClient, + BlobUploadCommonResponse, + StorageSharedKeyCredential, +} from '@azure/storage-blob'; +import { Logger } from 'winston'; +import { Entity, EntityName } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; +import { PublisherBase, PublishRequest } from './types'; + +export class AzureStoragePublish implements PublisherBase { + static async fromConfig( + config: Config, + logger: Logger, + ): Promise { + let account = ''; + let accountKey = ''; + let containerName = ''; + try { + account = config.getString( + 'techdocs.publisher.azureStorage.credentials.account', + ); + accountKey = config.getString( + 'techdocs.publisher.azureStorage.credentials.accountKey', + ); + containerName = config.getString( + 'techdocs.publisher.azureStorage.containerName', + ); + } catch (error) { + throw new Error( + "Since techdocs.publisher.type is set to 'azureStorage' in your app config, " + + 'credentials and containerName are required in techdocs.publisher.azureStorage ' + + 'required to authenticate with Azure Storage.', + ); + } + + const credential = new StorageSharedKeyCredential(account, accountKey); + const storageClient = new BlobServiceClient( + `https://${account}.blob.core.windows.net`, + credential, + ); + + await storageClient + .getContainerClient(containerName) + .getProperties() + .then(() => { + logger.info( + `Successfully connected to the Azure Storage container ${containerName}.`, + ); + }) + .catch(reason => { + logger.error( + `Could not retrieve metadata about the Azure Storage container ${containerName}. ` + + 'Make sure the Azure project and the container exists and the access key located at the path ' + + "techdocs.publisher.azureStorage.credentials defined in app config has the role 'Storage Object Creator'. " + + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + ); + throw new Error(`from Azure Storage client library: ${reason.message}`); + }); + + return new AzureStoragePublish(storageClient, containerName, logger); + } + + constructor( + private readonly storageClient: BlobServiceClient, + private readonly containerName: string, + private readonly logger: Logger, + ) { + this.storageClient = storageClient; + this.containerName = containerName; + this.logger = logger; + } + + /** + * Upload all the files from the generated `directory` to the Azure Storage container. + * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html + */ + async publish({ entity, directory }: PublishRequest): Promise { + try { + // Note: Azure Storage manages creation of parent directories if they do not exist. + // So collecting path of only the files is good enough. + const allFilesToUpload = await getFileTreeRecursively(directory); + + const uploadPromises: Array> = []; + allFilesToUpload.forEach(filePath => { + // Remove the absolute path prefix of the source directory + // Path of all files to upload, relative to the root of the source directory + // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] + const relativeFilePath = filePath.replace(`${directory}/`, ''); + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const destination = path.normalize( + `${entityRootDir}/${relativeFilePath}`, + ); // Azure Storage Container file relative path + // TODO: Upload in chunks of ~10 files instead of all files at once. + uploadPromises.push( + this.storageClient + .getContainerClient(this.containerName) + .getBlockBlobClient(destination) + .uploadFile(filePath), + ); + }); + + await Promise.all(uploadPromises).then(() => { + this.logger.info( + `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, + ); + }); + return; + } catch (e) { + const errorMessage = `Unable to upload file(s) to Azure Storage. Error ${e.message}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } + } + + download(containerName: string, path: string): Promise { + return new Promise((resolve, reject) => { + const fileStreamChunks: Array = []; + this.storageClient + .getContainerClient(containerName) + .getBlockBlobClient(path) + .download() + .then(res => { + const body = res.readableStreamBody; + if (!body) { + reject(new Error(`Unable to parse the response data`)); + return; + } + body + .on('error', e => { + this.logger.error(e.message); + reject(e.message); + }) + .on('data', chunk => { + fileStreamChunks.push(chunk); + }) + .on('end', () => { + resolve(Buffer.concat(fileStreamChunks).toString()); + }); + }); + }); + } + + async fetchTechDocsMetadata(entityName: EntityName): Promise { + const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + try { + return this.download( + this.containerName, + `${entityRootDir}/techdocs_metadata.json`, + ); + } catch (e) { + this.logger.error(e.message); + throw e; + } + } + + /** + * Express route middleware to serve static files on a route in techdocs-backend. + */ + docsRouter(): express.Handler { + return (req, res) => { + // Trim the leading forward slash + // filePath example - /default/Component/documented-component/index.html + const filePath = req.path.replace(/^\//, ''); + // Files with different extensions (CSS, HTML) need to be served with different headers + const fileExtension = path.extname(filePath); + const responseHeaders = getHeadersForFileExtension(fileExtension); + + try { + this.download(this.containerName, filePath).then(fileContent => { + // Inject response headers + for (const [headerKey, headerValue] of Object.entries( + responseHeaders, + )) { + res.setHeader(headerKey, headerValue); + } + res.send(fileContent); + }); + } catch (e) { + this.logger.error(e.message); + res.status(404).send(e.message); + } + }; + } + + /** + * A helper function which checks if index.html of an Entity's docs site is available. This + * can be used to verify if there are any pre-generated docs available to serve. + */ + async hasDocsBeenGenerated(entity: Entity): Promise { + return new Promise(resolve => { + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + this.storageClient + .getContainerClient(this.containerName) + .getBlockBlobClient(`${entityRootDir}/index.html`) + .exists() + .then((response: boolean) => { + resolve(response); + }) + .catch(() => { + resolve(false); + }); + }); + } +} diff --git a/packages/techdocs-common/src/stages/publish/publish.test.ts b/packages/techdocs-common/src/stages/publish/publish.test.ts index 89f2c0ffdd..a97eadb404 100644 --- a/packages/techdocs-common/src/stages/publish/publish.test.ts +++ b/packages/techdocs-common/src/stages/publish/publish.test.ts @@ -22,6 +22,7 @@ import { Publisher } from './publish'; import { LocalPublish } from './local'; import { GoogleGCSPublish } from './googleStorage'; import { AwsS3Publish } from './awsS3'; +import { AzureStoragePublish } from './azureStorage'; const logger = getVoidLogger(); const discovery: jest.Mocked = { @@ -105,4 +106,28 @@ describe('Publisher', () => { }); expect(publisher).toBeInstanceOf(AwsS3Publish); }); + + it('should create Azure Storage publisher from config', async () => { + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'azureStorage', + azureStorage: { + credentials: { + account: 'account', + accountKey: 'accountKey', + }, + containerName: 'containerName', + }, + }, + }, + }); + + const publisher = await Publisher.fromConfig(mockConfig, { + logger, + discovery, + }); + expect(publisher).toBeInstanceOf(AzureStoragePublish); + }); }); diff --git a/packages/techdocs-common/src/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts index 82232c2fd1..fc72d2812e 100644 --- a/packages/techdocs-common/src/stages/publish/publish.ts +++ b/packages/techdocs-common/src/stages/publish/publish.ts @@ -21,6 +21,7 @@ import { PublisherType, PublisherBase } from './types'; import { LocalPublish } from './local'; import { GoogleGCSPublish } from './googleStorage'; import { AwsS3Publish } from './awsS3'; +import { AzureStoragePublish } from './azureStorage'; type factoryOptions = { logger: Logger; @@ -47,6 +48,9 @@ export class Publisher { case 'awsS3': logger.info('Creating AWS S3 Bucket publisher for TechDocs'); return AwsS3Publish.fromConfig(config, logger); + case 'azureStorage': + logger.info('Creating Azure Storage Container publisher for TechDocs'); + return AzureStoragePublish.fromConfig(config, logger); case 'local': logger.info('Creating Local publisher for TechDocs'); return new LocalPublish(config, logger, discovery); diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index db6a075d43..f67f65ee23 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -19,7 +19,7 @@ import express from 'express'; /** * Key for all the different types of TechDocs publishers that are supported. */ -export type PublisherType = 'local' | 'googleGcs' | 'awsS3'; +export type PublisherType = 'local' | 'googleGcs' | 'awsS3' | 'azureStorage'; export type PublishRequest = { entity: Entity; diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 5ad23cde84..cbb6efa3dd 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -148,6 +148,7 @@ export async function createRouter({ } break; case 'awsS3': + case 'azureStorage': case 'googleGcs': // This block should be valid for all external storage implementations. So no need to duplicate in future, // add the publisher type in the list here. diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index f9831f27cb..1dd7319f61 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -114,6 +114,45 @@ export interface Config { region?: string; }; } + | { + /** + * attr: 'type' - accepts a string value + * e.g. type: 'azureStorage' + * alternatives: 'azureStorage' etc. + * @see http://backstage.io/docs/features/techdocs/configuration + */ + type: 'azureStorage'; + + /** + * azureStorage required when 'type' is set to azureStorage + */ + azureStorage?: { + /** + * Credentials used to access a storage container + * @visibility secret + */ + credentials: { + /** + * Account access name + * attr: 'account' - accepts a string value + * @visibility secret + */ + account: string; + /** + * Account secret primary key + * attr: 'accountKey' - accepts a string value + * @visibility secret + */ + accountKey: string; + }; + /** + * Cloud Storage Container Name + * attr: 'containerName' - accepts a string value + * @visibility backend + */ + containerName: string; + }; + } | { /** * attr: 'type' - accepts a string value From 42494c7e91650459cd1fe5158b5ee9ec38bcd0ba Mon Sep 17 00:00:00 2001 From: vitorgrenzel Date: Wed, 20 Jan 2021 17:21:07 -0300 Subject: [PATCH 02/10] feat(docs): add Azure Blob Storage --- .changeset/chilly-dodos-drop.md | 2 +- docs/features/techdocs/README.md | 2 +- docs/features/techdocs/configuration.md | 20 +++++----- docs/features/techdocs/using-cloud-storage.md | 40 ++++++++++--------- 4 files changed, 34 insertions(+), 30 deletions(-) diff --git a/.changeset/chilly-dodos-drop.md b/.changeset/chilly-dodos-drop.md index b262c14e4d..2af5255e70 100644 --- a/.changeset/chilly-dodos-drop.md +++ b/.changeset/chilly-dodos-drop.md @@ -3,4 +3,4 @@ '@backstage/plugin-techdocs-backend': patch --- -1. Added option to use Azure Storage as a choice to store the static generated files for TechDocs. +1. Added option to use Azure Blob Storage as a choice to store the static generated files for TechDocs. diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index 24b4410092..f81f41668d 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -113,7 +113,7 @@ providers are used. | Local Filesystem of Backstage app | Yes ✅ | | Google Cloud Storage (GCS) | Yes ✅ | | Amazon Web Services (AWS) S3 | Yes ✅ | -| Azure Storage | Yes ✅ | +| Azure Blob Storage | Yes ✅ | [Reach out to us](#feedback) if you want to request more platforms. diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 8d5906b18f..efed37fc69 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -44,7 +44,7 @@ techdocs: # or you want to use External storage providers like Google Cloud Storage, AWS S3, etc. publisher: - # techdocs.publisher.type can be - 'local' or 'googleGcs' or 'awsS3' (azureStorage to be available in future). + # techdocs.publisher.type can be - 'local' or 'googleGcs' or 'awsS3' or 'azureBlobStorage'. # When set to 'local', techdocs-backend will create a 'static' directory at its root to store generated documentation files. # When set to 'googleGcs', techdocs-backend will use a Google Cloud Storage Bucket to store generated documentation files. # When set to 'awsS3', techdocs-backend will use an Amazon Web Service (AWS) S3 bucket to store generated documentation files. @@ -85,16 +85,18 @@ techdocs: region: $env: AWS_REGION - # Required when techdocs.publisher.type is set to 'azureStorage'. Skip otherwise. + # Required when techdocs.publisher.type is set to 'azureBlobStorage'. Skip otherwise. - azureStorage: - # An API key is required to write to a storage container. + azureBlobStorage: + # (Required) Azure Blob Storage Container Name + containerName: 'techdocs-storage' + + # (Optional) An API key is required to write to a storage container. + # If not set, environment variables will be used to authenticate. + #https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json credentials: account: - $env: TECHDOCS_AZURE_STORAGE_ACCOUNT + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT accountKey: - $env: TECHDOCS_AZURE_STORAGE_ACCOUNT_KEY - - # Azure Storage Container Name - containerName: 'techdocs-storage' + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY ``` diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 2d906ca1b9..9ae14abfe3 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -196,20 +196,20 @@ the static generated documentation files. When you start the backend of the app, you should be able to see `techdocs info Successfully connected to the AWS S3 bucket` in the logs. -## Configuring Azure Storage Container with TechDocs +## Configuring Azure Blob Storage Container with TechDocs Follow the -[official Azure Storage documentation](https://docs.microsoft.com/pt-br/javascript/api/@azure/storage-blob/?view=azure-node-latest) -for the latest instructions on the following steps involving Azure Storage. +[official Azure Blob Storage documentation](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json) +for the latest instructions on the following steps involving Azure Blob Storage. **1. Set `techdocs.publisher.type` config in your `app-config.yaml`** -Set `techdocs.publisher.type` to `'azureStorage'`. +Set `techdocs.publisher.type` to `'azureBlobStorage'`. ```yaml techdocs: publisher: - type: 'azureStorage' + type: 'azureBlobStorage' ``` **2. Service account credentials** @@ -220,14 +220,14 @@ and get your Storage account name and Primary Key. ```yaml techdocs: publisher: - type: 'azureStorage' - azureStorage: + type: 'azureBlobStorage' + azureBlobStorage: credentials: account: 'account' accountKey: 'accountKey' ``` -**3. Azure Storage Container** +**3. Azure Blob Storage Container** Create a dedicated container for TechDocs sites. techdocs-backend will publish documentation to this container. TechDocs will fetch files from here to serve @@ -236,23 +236,25 @@ documentation in Backstage. To create a new container, access "Blob Service > Containers > New Container". Set the name of the container to -`techdocs.publisher.azureStorage.containerName`. +`techdocs.publisher.azureBlobStorage.containerName`. ```yaml techdocs: publisher: - type: 'azureStorage' - azureStorage: - credentials: - account: 'account' - accountKey: 'accountKey' + type: 'azureBlobStorage' + azureBlobStorage: containerName: 'name-of-techdocs-storage-container' + credentials: + account: + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT + accountKey: + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY ``` **4. That's it!** -Your Backstage app is now ready to use Azure Storage for TechDocs, to store and -read the static generated documentation files. When you start the backend of the -app, you should be able to see -`techdocs info Successfully connected to the Azure Storage container` in the -logs. +Your Backstage app is now ready to use Azure Blob Storage for TechDocs, to store +and read the static generated documentation files. When you start the backend of +the app, you should be able to see +`techdocs info Successfully connected to the Azure Blob Storage container` in +the logs. From 59b8d5a0d94110b5d4c7ed9cc89076170e72a817 Mon Sep 17 00:00:00 2001 From: vitorgrenzel Date: Wed, 20 Jan 2021 17:21:40 -0300 Subject: [PATCH 03/10] feat(techdocs-common): add Azure Blob Storage --- app-config.yaml | 2 +- ...orage.test.ts => azureBlobStorage.test.ts} | 12 +- .../{azureStorage.ts => azureBlobStorage.ts} | 103 +++++++++++------- .../src/stages/publish/publish.test.ts | 10 +- .../src/stages/publish/publish.ts | 10 +- .../src/stages/publish/types.ts | 6 +- .../techdocs-backend/src/service/router.ts | 2 +- plugins/techdocs/config.d.ts | 18 +-- 8 files changed, 99 insertions(+), 64 deletions(-) rename packages/techdocs-common/src/stages/publish/{azureStorage.test.ts => azureBlobStorage.test.ts} (89%) rename packages/techdocs-common/src/stages/publish/{azureStorage.ts => azureBlobStorage.ts} (67%) diff --git a/app-config.yaml b/app-config.yaml index 09d2e75253..90cf73f58f 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -80,7 +80,7 @@ techdocs: generators: techdocs: 'docker' # Alternatives - 'local' publisher: - type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureStorage'. Read documentation for using alternatives. + type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureBlobStorage'. Read documentation for using alternatives. sentry: organization: my-company diff --git a/packages/techdocs-common/src/stages/publish/azureStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts similarity index 89% rename from packages/techdocs-common/src/stages/publish/azureStorage.test.ts rename to packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 05b5441bff..95e4a52f62 100644 --- a/packages/techdocs-common/src/stages/publish/azureStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -16,7 +16,7 @@ import mockFs from 'mock-fs'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; -import { AzureStoragePublish } from './azureStorage'; +import { AzureBlobStoragePublish } from './azureBlobStorage'; import { PublisherBase } from './types'; import type { Entity } from '@backstage/catalog-model'; @@ -54,8 +54,8 @@ beforeEach(async () => { techdocs: { requestUrl: 'http://localhost:7000', publisher: { - type: 'azureStorage', - azureStorage: { + type: 'azureBlobStorage', + azureBlobStorage: { credentials: { account: 'account', accountKey: 'accountKey', @@ -66,10 +66,10 @@ beforeEach(async () => { }, }); - publisher = await AzureStoragePublish.fromConfig(mockConfig, logger); + publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger); }); -describe('AzureStoragePublish', () => { +describe('AzureBlobStoragePublish', () => { describe('publish', () => { it('should publish a directory', async () => { const entity = createMockEntity(); @@ -117,7 +117,7 @@ describe('AzureStoragePublish', () => { .catch(error => expect(error).toEqual( new Error( - `Unable to upload file(s) to Azure Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`, + `Unable to upload file(s) to Azure Blob Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`, ), ), ); diff --git a/packages/techdocs-common/src/stages/publish/azureStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts similarity index 67% rename from packages/techdocs-common/src/stages/publish/azureStorage.ts rename to packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index bc9a6825d9..7ba4d35892 100644 --- a/packages/techdocs-common/src/stages/publish/azureStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -24,34 +24,45 @@ import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; -import { PublisherBase, PublishRequest } from './types'; +import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; +import limiterFactory from 'p-limit'; +import JSON5 from 'json5'; -export class AzureStoragePublish implements PublisherBase { +// The number of batches that may be ongoing at the same time. +const BATCH_CONCURRENCY = 3; + +export class AzureBlobStoragePublish implements PublisherBase { static async fromConfig( config: Config, logger: Logger, ): Promise { - let account = ''; - let accountKey = ''; let containerName = ''; try { - account = config.getString( - 'techdocs.publisher.azureStorage.credentials.account', - ); - accountKey = config.getString( - 'techdocs.publisher.azureStorage.credentials.accountKey', - ); containerName = config.getString( - 'techdocs.publisher.azureStorage.containerName', + 'techdocs.publisher.azureBlobStorage.containerName', ); } catch (error) { throw new Error( - "Since techdocs.publisher.type is set to 'azureStorage' in your app config, " + - 'credentials and containerName are required in techdocs.publisher.azureStorage ' + - 'required to authenticate with Azure Storage.', + "Since techdocs.publisher.type is set to 'awsS3' in your app config, " + + 'techdocs.publisher.awsS3.bucketName is required.', ); } + // Credentials is an optional config. If missing, default AWS environment variables + // or AWS shared credentials file at ~/.aws/credentials will be used to authenticate + // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html + // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html + let account = ''; + let accountKey = ''; + account = + config.getOptionalString( + 'techdocs.publisher.azureBlobStorage.credentials.account', + ) || ''; + accountKey = + config.getOptionalString( + 'techdocs.publisher.azureBlobStorage.credentials.accountKey', + ) || ''; + const credential = new StorageSharedKeyCredential(account, accountKey); const storageClient = new BlobServiceClient( `https://${account}.blob.core.windows.net`, @@ -63,20 +74,22 @@ export class AzureStoragePublish implements PublisherBase { .getProperties() .then(() => { logger.info( - `Successfully connected to the Azure Storage container ${containerName}.`, + `Successfully connected to the Azure Blob Storage container ${containerName}.`, ); }) .catch(reason => { logger.error( - `Could not retrieve metadata about the Azure Storage container ${containerName}. ` + + `Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` + 'Make sure the Azure project and the container exists and the access key located at the path ' + - "techdocs.publisher.azureStorage.credentials defined in app config has the role 'Storage Object Creator'. " + + "techdocs.publisher.azureBlobStorage.credentials defined in app config has the role 'Storage Object Creator'. " + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', ); - throw new Error(`from Azure Storage client library: ${reason.message}`); + throw new Error( + `from Azure Blob Storage client library: ${reason.message}`, + ); }); - return new AzureStoragePublish(storageClient, containerName, logger); + return new AzureBlobStoragePublish(storageClient, containerName, logger); } constructor( @@ -90,17 +103,23 @@ export class AzureStoragePublish implements PublisherBase { } /** - * Upload all the files from the generated `directory` to the Azure Storage container. + * Upload all the files from the generated `directory` to the Azure Blob Storage container. * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html */ async publish({ entity, directory }: PublishRequest): Promise { try { - // Note: Azure Storage manages creation of parent directories if they do not exist. + // Note: Azure Blob Storage manages creation of parent directories if they do not exist. // So collecting path of only the files is good enough. const allFilesToUpload = await getFileTreeRecursively(directory); const uploadPromises: Array> = []; - allFilesToUpload.forEach(filePath => { + + // Bound the number of concurrent batches. We want a bit of concurrency for + // performance reasons, but not so much that we starve the connection pool + // or start thrashing. + const limiter = limiterFactory(BATCH_CONCURRENCY); + + const promises = allFilesToUpload.map(filePath => { // Remove the absolute path prefix of the source directory // Path of all files to upload, relative to the root of the source directory // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] @@ -108,30 +127,33 @@ export class AzureStoragePublish implements PublisherBase { const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; const destination = path.normalize( `${entityRootDir}/${relativeFilePath}`, - ); // Azure Storage Container file relative path + ); // Azure Blob Storage Container file relative path + // TODO: Upload in chunks of ~10 files instead of all files at once. - uploadPromises.push( - this.storageClient - .getContainerClient(this.containerName) - .getBlockBlobClient(destination) - .uploadFile(filePath), - ); + return limiter(async () => { + await uploadPromises.push( + this.storageClient + .getContainerClient(this.containerName) + .getBlockBlobClient(destination) + .uploadFile(filePath), + ); + }); }); - await Promise.all(uploadPromises).then(() => { + await Promise.all(promises).then(() => { this.logger.info( `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, ); }); return; } catch (e) { - const errorMessage = `Unable to upload file(s) to Azure Storage. Error ${e.message}`; + const errorMessage = `Unable to upload file(s) to Azure Blob Storage. Error ${e.message}`; this.logger.error(errorMessage); throw new Error(errorMessage); } } - download(containerName: string, path: string): Promise { + private download(containerName: string, path: string): Promise { return new Promise((resolve, reject) => { const fileStreamChunks: Array = []; this.storageClient @@ -153,19 +175,24 @@ export class AzureStoragePublish implements PublisherBase { fileStreamChunks.push(chunk); }) .on('end', () => { - resolve(Buffer.concat(fileStreamChunks).toString()); + resolve(Buffer.concat(fileStreamChunks)); }); }); }); } - async fetchTechDocsMetadata(entityName: EntityName): Promise { + async fetchTechDocsMetadata( + entityName: EntityName, + ): Promise { const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; try { - return this.download( - this.containerName, - `${entityRootDir}/techdocs_metadata.json`, - ); + return await new Promise(resolve => { + const download = this.download( + this.containerName, + `${entityRootDir}/techdocs_metadata.json`, + ); + resolve(JSON5.parse(download.toString())); + }); } catch (e) { this.logger.error(e.message); throw e; diff --git a/packages/techdocs-common/src/stages/publish/publish.test.ts b/packages/techdocs-common/src/stages/publish/publish.test.ts index a97eadb404..a563e53a9b 100644 --- a/packages/techdocs-common/src/stages/publish/publish.test.ts +++ b/packages/techdocs-common/src/stages/publish/publish.test.ts @@ -22,7 +22,7 @@ import { Publisher } from './publish'; import { LocalPublish } from './local'; import { GoogleGCSPublish } from './googleStorage'; import { AwsS3Publish } from './awsS3'; -import { AzureStoragePublish } from './azureStorage'; +import { AzureBlobStoragePublish } from './azureBlobStorage'; const logger = getVoidLogger(); const discovery: jest.Mocked = { @@ -107,13 +107,13 @@ describe('Publisher', () => { expect(publisher).toBeInstanceOf(AwsS3Publish); }); - it('should create Azure Storage publisher from config', async () => { + it('should create Azure Blob Storage publisher from config', async () => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', publisher: { - type: 'azureStorage', - azureStorage: { + type: 'azureBlobStorage', + azureBlobStorage: { credentials: { account: 'account', accountKey: 'accountKey', @@ -128,6 +128,6 @@ describe('Publisher', () => { logger, discovery, }); - expect(publisher).toBeInstanceOf(AzureStoragePublish); + expect(publisher).toBeInstanceOf(AzureBlobStoragePublish); }); }); diff --git a/packages/techdocs-common/src/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts index fc72d2812e..a4e33a2d9c 100644 --- a/packages/techdocs-common/src/stages/publish/publish.ts +++ b/packages/techdocs-common/src/stages/publish/publish.ts @@ -21,7 +21,7 @@ import { PublisherType, PublisherBase } from './types'; import { LocalPublish } from './local'; import { GoogleGCSPublish } from './googleStorage'; import { AwsS3Publish } from './awsS3'; -import { AzureStoragePublish } from './azureStorage'; +import { AzureBlobStoragePublish } from './azureBlobStorage'; type factoryOptions = { logger: Logger; @@ -48,9 +48,11 @@ export class Publisher { case 'awsS3': logger.info('Creating AWS S3 Bucket publisher for TechDocs'); return AwsS3Publish.fromConfig(config, logger); - case 'azureStorage': - logger.info('Creating Azure Storage Container publisher for TechDocs'); - return AzureStoragePublish.fromConfig(config, logger); + case 'azureBlobStorage': + logger.info( + 'Creating Azure Blob Storage Container publisher for TechDocs', + ); + return AzureBlobStoragePublish.fromConfig(config, logger); case 'local': logger.info('Creating Local publisher for TechDocs'); return new LocalPublish(config, logger, discovery); diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index f67f65ee23..5e953deb81 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -19,7 +19,11 @@ import express from 'express'; /** * Key for all the different types of TechDocs publishers that are supported. */ -export type PublisherType = 'local' | 'googleGcs' | 'awsS3' | 'azureStorage'; +export type PublisherType = + | 'local' + | 'googleGcs' + | 'awsS3' + | 'azureBlobStorage'; export type PublishRequest = { entity: Entity; diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index cbb6efa3dd..dc5e6a4911 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -148,7 +148,7 @@ export async function createRouter({ } break; case 'awsS3': - case 'azureStorage': + case 'azureBlobStorage': case 'googleGcs': // This block should be valid for all external storage implementations. So no need to duplicate in future, // add the publisher type in the list here. diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index 1dd7319f61..f7903ea3fb 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -117,21 +117,23 @@ export interface Config { | { /** * attr: 'type' - accepts a string value - * e.g. type: 'azureStorage' - * alternatives: 'azureStorage' etc. + * e.g. type: 'azureBlobStorage' + * alternatives: 'azureBlobStorage' etc. * @see http://backstage.io/docs/features/techdocs/configuration */ - type: 'azureStorage'; + type: 'azureBlobStorage'; /** - * azureStorage required when 'type' is set to azureStorage + * azureBlobStorage required when 'type' is set to azureBlobStorage */ - azureStorage?: { + azureBlobStorage?: { /** - * Credentials used to access a storage container + * (Optional) Credentials used to access a storage container. + * If not set, environment variables will be used to authenticate. + * https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json * @visibility secret */ - credentials: { + credentials?: { /** * Account access name * attr: 'account' - accepts a string value @@ -146,7 +148,7 @@ export interface Config { accountKey: string; }; /** - * Cloud Storage Container Name + * (Required) Cloud Storage Container Name * attr: 'containerName' - accepts a string value * @visibility backend */ From 74d34c8ff9ab85dc8748c333c0348d113d2cab30 Mon Sep 17 00:00:00 2001 From: vitorgrenzel Date: Wed, 20 Jan 2021 18:00:12 -0300 Subject: [PATCH 04/10] feat(docs): update Azure Blob Storage --- docs/features/techdocs/using-cloud-storage.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 9ae14abfe3..402ae657ea 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -223,8 +223,10 @@ techdocs: type: 'azureBlobStorage' azureBlobStorage: credentials: - account: 'account' - accountKey: 'accountKey' + account: + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT + accountKey: + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY ``` **3. Azure Blob Storage Container** From 2d8d7697d60d3bc685787f80eaa04d6dc6759219 Mon Sep 17 00:00:00 2001 From: vitorgrenzel Date: Wed, 20 Jan 2021 18:20:17 -0300 Subject: [PATCH 05/10] feat(techdocs-common): update Azure Blob Storage --- packages/techdocs-common/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 09893bd38e..1834248c3a 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -36,8 +36,8 @@ "url": "https://github.com/backstage/backstage/issues" }, "dependencies": { - "@azure/storage-blob": "^12.3.0", "@aws-sdk/client-s3": "^3.1.0", + "@azure/storage-blob": "^12.4.0", "@backstage/backend-common": "^0.5.1", "@backstage/catalog-model": "^0.7.0", "@backstage/config": "^0.1.2", From 0be9694aa6997d78b091ddcbb3fe3a5763ff283c Mon Sep 17 00:00:00 2001 From: vitorgrenzel Date: Fri, 22 Jan 2021 16:09:06 -0300 Subject: [PATCH 06/10] feat(techdocs-common): update Azure Blob Storage --- docs/features/techdocs/configuration.md | 12 +-- docs/features/techdocs/using-cloud-storage.md | 76 ++++++++++++------- packages/techdocs-common/package.json | 1 + .../stages/publish/azureBlobStorage.test.ts | 2 +- .../src/stages/publish/azureBlobStorage.ts | 48 +++++++----- .../src/stages/publish/publish.test.ts | 2 +- plugins/techdocs/config.d.ts | 14 ++-- 7 files changed, 96 insertions(+), 59 deletions(-) diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index efed37fc69..f588be7c8c 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -91,12 +91,14 @@ techdocs: # (Required) Azure Blob Storage Container Name containerName: 'techdocs-storage' - # (Optional) An API key is required to write to a storage container. - # If not set, environment variables will be used to authenticate. - #https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json + # (Required) An account name is required to write to a storage blob container. + # https://docs.microsoft.com/pt-br/rest/api/storageservices/authorize-with-shared-key credentials: - account: - $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT + accountName: + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME + # (Optional) An account key is required to write to a storage container. + # If missing,AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET environment variable will be used. + # https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json accountKey: $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY ``` diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 402ae657ea..f384270379 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -212,33 +212,17 @@ techdocs: type: 'azureBlobStorage' ``` -**2. Service account credentials** +**2. Create an Azure Blob Storage Container** -To get credentials, access the Azure Portal and go to "Settings > Access Keys", -and get your Storage account name and Primary Key. +Create a dedicated container for TechDocs sites. +[Refer to the official documentation](https://docs.microsoft.com/pt-br/azure/storage/blobs/storage-quickstart-blobs-portal). -```yaml -techdocs: - publisher: - type: 'azureBlobStorage' - azureBlobStorage: - credentials: - account: - $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT - accountKey: - $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY -``` +TechDocs will publish documentation to this container and will fetch files from +here to serve documentation in Backstage. Note that the container names are +globally unique. -**3. Azure Blob Storage Container** - -Create a dedicated container for TechDocs sites. techdocs-backend will publish -documentation to this container. TechDocs will fetch files from here to serve -documentation in Backstage. - -To create a new container, access "Blob Service > Containers > New Container". - -Set the name of the container to -`techdocs.publisher.azureBlobStorage.containerName`. +Set the config `techdocs.publisher.azureBlobStorage.containerName` in your +`app-config.yaml` to the name of the container you just created. ```yaml techdocs: @@ -246,9 +230,49 @@ techdocs: type: 'azureBlobStorage' azureBlobStorage: containerName: 'name-of-techdocs-storage-container' +``` + +**3a. (Recommended) Authentication using environment variable** + +Set the config `techdocs.publisher.azureBlobStorage.credentials.accountName` in +your `app-config.yaml` to the your account name. + +The storage blob client will automatically use the environment variable +`AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET` to authenticate with +Azure Blob Storage. +https://docs.microsoft.com/pt-br/azure/storage/common/storage-auth-aad for more +details. + +```yaml +techdocs: + publisher: + type: 'azureBlobStorage' + azureBlobStorage: + containerName: 'name-of-techdocs-storage-bucket' credentials: - account: - $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT + accountName: + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME +``` + +**3b. Authentication using app-config.yaml** + +If you do not prefer (3a) and optionally like to use a service account, you can +follow these steps. + +To get credentials, access the Azure Portal and go to "Settings > Access Keys", +and get your Storage account name and Primary Key. +https://docs.microsoft.com/pt-br/rest/api/storageservices/authorize-with-shared-key +for more details. + +```yaml +techdocs: + publisher: + type: 'azureBlobStorage' + azureBlobStorage: + containerName: 'name-of-techdocs-storage-bucket' + credentials: + accountName: + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME accountKey: $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY ``` diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 1834248c3a..2a065600a1 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -37,6 +37,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.1.0", + "@azure/identity": "^1.2.2", "@azure/storage-blob": "^12.4.0", "@backstage/backend-common": "^0.5.1", "@backstage/catalog-model": "^0.7.0", diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 95e4a52f62..9d2ee1ba9b 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -57,7 +57,7 @@ beforeEach(async () => { type: 'azureBlobStorage', azureBlobStorage: { credentials: { - account: 'account', + accountName: 'accountName', accountKey: 'accountKey', }, containerName: 'containerName', diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index 7ba4d35892..abfa9d71bf 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -20,6 +20,7 @@ import { BlobUploadCommonResponse, StorageSharedKeyCredential, } from '@azure/storage-blob'; +import { DefaultAzureCredential } from '@azure/identity'; import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; @@ -43,29 +44,39 @@ export class AzureBlobStoragePublish implements PublisherBase { ); } catch (error) { throw new Error( - "Since techdocs.publisher.type is set to 'awsS3' in your app config, " + - 'techdocs.publisher.awsS3.bucketName is required.', + "Since techdocs.publisher.type is set to 'azureBlobStorage' in your app config, " + + 'techdocs.publisher.azureBlobStorage.containerName is required.', ); } - // Credentials is an optional config. If missing, default AWS environment variables - // or AWS shared credentials file at ~/.aws/credentials will be used to authenticate - // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html - // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html - let account = ''; - let accountKey = ''; - account = - config.getOptionalString( - 'techdocs.publisher.azureBlobStorage.credentials.account', - ) || ''; - accountKey = - config.getOptionalString( - 'techdocs.publisher.azureBlobStorage.credentials.accountKey', - ) || ''; + let accountName = ''; + try { + accountName = config.getString( + 'techdocs.publisher.azureBlobStorage.credentials.accountName', + ); + } catch (error) { + throw new Error( + "Since techdocs.publisher.type is set to 'azureBlobStorage' in your app config, " + + 'techdocs.publisher.azureBlobStorage.credentials.accountName is required.', + ); + } + + // Credentials is an optional config. If missing, default Azure Blob Storage environment variables + // https://docs.microsoft.com/pt-br/azure/storage/common/storage-auth-aad-app + const accountKey = config.getOptionalString( + 'techdocs.publisher.azureBlobStorage.credentials.accountKey', + ); + + let credential; + if (accountKey) { + console.log('accountKey =>', accountKey); + credential = new StorageSharedKeyCredential(accountName, accountKey); + } else { + credential = new DefaultAzureCredential(); + } - const credential = new StorageSharedKeyCredential(account, accountKey); const storageClient = new BlobServiceClient( - `https://${account}.blob.core.windows.net`, + `https://${accountName}.blob.core.windows.net`, credential, ); @@ -129,7 +140,6 @@ export class AzureBlobStoragePublish implements PublisherBase { `${entityRootDir}/${relativeFilePath}`, ); // Azure Blob Storage Container file relative path - // TODO: Upload in chunks of ~10 files instead of all files at once. return limiter(async () => { await uploadPromises.push( this.storageClient diff --git a/packages/techdocs-common/src/stages/publish/publish.test.ts b/packages/techdocs-common/src/stages/publish/publish.test.ts index a563e53a9b..4ae59a597e 100644 --- a/packages/techdocs-common/src/stages/publish/publish.test.ts +++ b/packages/techdocs-common/src/stages/publish/publish.test.ts @@ -115,7 +115,7 @@ describe('Publisher', () => { type: 'azureBlobStorage', azureBlobStorage: { credentials: { - account: 'account', + accountName: 'accountName', accountKey: 'accountKey', }, containerName: 'containerName', diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index f7903ea3fb..0af3ab4290 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -128,24 +128,24 @@ export interface Config { */ azureBlobStorage?: { /** - * (Optional) Credentials used to access a storage container. - * If not set, environment variables will be used to authenticate. - * https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json + * (Required) Credentials used to access a storage container. * @visibility secret */ - credentials?: { + credentials: { /** * Account access name * attr: 'account' - accepts a string value * @visibility secret */ - account: string; + accountName: string; /** - * Account secret primary key + * (Optional) Account secret primary key + * If not set, environment variables will be used to authenticate. + * https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json * attr: 'accountKey' - accepts a string value * @visibility secret */ - accountKey: string; + accountKey?: string; }; /** * (Required) Cloud Storage Container Name From 0024d6d8c0c44ce074cc047729c77fea3448fa47 Mon Sep 17 00:00:00 2001 From: vitorgrenzel Date: Mon, 25 Jan 2021 10:01:30 -0300 Subject: [PATCH 07/10] feat(techdocs-common): update Azure Blob Storage --- docs/features/techdocs/using-cloud-storage.md | 6 ++-- .../src/stages/publish/azureBlobStorage.ts | 9 +++--- .../src/stages/publish/publish.test.ts | 31 +++++++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index f384270379..393e478302 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -215,7 +215,7 @@ techdocs: **2. Create an Azure Blob Storage Container** Create a dedicated container for TechDocs sites. -[Refer to the official documentation](https://docs.microsoft.com/pt-br/azure/storage/blobs/storage-quickstart-blobs-portal). +[Refer to the official documentation](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-portal). TechDocs will publish documentation to this container and will fetch files from here to serve documentation in Backstage. Note that the container names are @@ -240,7 +240,9 @@ your `app-config.yaml` to the your account name. The storage blob client will automatically use the environment variable `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET` to authenticate with Azure Blob Storage. -https://docs.microsoft.com/pt-br/azure/storage/common/storage-auth-aad for more +[Steps to create the service where the variables can be retrieved from](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal). + +https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad for more details. ```yaml diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index abfa9d71bf..1b4e59a714 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -61,15 +61,14 @@ export class AzureBlobStoragePublish implements PublisherBase { ); } - // Credentials is an optional config. If missing, default Azure Blob Storage environment variables - // https://docs.microsoft.com/pt-br/azure/storage/common/storage-auth-aad-app + // Credentials is an optional config. If missing, default Azure Blob Storage environment variables will be used. + // https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-app const accountKey = config.getOptionalString( 'techdocs.publisher.azureBlobStorage.credentials.accountKey', ); let credential; if (accountKey) { - console.log('accountKey =>', accountKey); credential = new StorageSharedKeyCredential(accountName, accountKey); } else { credential = new DefaultAzureCredential(); @@ -91,8 +90,8 @@ export class AzureBlobStoragePublish implements PublisherBase { .catch(reason => { logger.error( `Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` + - 'Make sure the Azure project and the container exists and the access key located at the path ' + - "techdocs.publisher.azureBlobStorage.credentials defined in app config has the role 'Storage Object Creator'. " + + 'Make sure that the Azure project and container exist and the access key is setup correctly ' + + 'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', ); throw new Error( diff --git a/packages/techdocs-common/src/stages/publish/publish.test.ts b/packages/techdocs-common/src/stages/publish/publish.test.ts index 4ae59a597e..d1faf82b78 100644 --- a/packages/techdocs-common/src/stages/publish/publish.test.ts +++ b/packages/techdocs-common/src/stages/publish/publish.test.ts @@ -31,6 +31,10 @@ const discovery: jest.Mocked = { }; describe('Publisher', () => { + beforeEach(() => { + jest.resetModules(); // clear the cache + }); + it('should create local publisher by default', async () => { const mockConfig = new ConfigReader({ techdocs: { @@ -130,4 +134,31 @@ describe('Publisher', () => { }); expect(publisher).toBeInstanceOf(AzureBlobStoragePublish); }); + + it('should create Azure Blob Storage publisher from environment variables', async () => { + process.env.AZURE_TENANT_ID = 'AZURE_TENANT_ID'; + process.env.AZURE_CLIENT_ID = 'AZURE_CLIENT_ID'; + process.env.AZURE_CLIENT_SECRET = 'AZURE_CLIENT_SECRET'; + + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'azureBlobStorage', + azureBlobStorage: { + credentials: { + accountName: 'accountName', + }, + containerName: 'containerName', + }, + }, + }, + }); + + const publisher = await Publisher.fromConfig(mockConfig, { + logger, + discovery, + }); + expect(publisher).toBeInstanceOf(AzureBlobStoragePublish); + }); }); From 9197ac3a0baf3281d016e95c7fdb1c0f2ce8b32f Mon Sep 17 00:00:00 2001 From: vitorgrenzel Date: Mon, 25 Jan 2021 10:20:56 -0300 Subject: [PATCH 08/10] feat(techdocs-common): update tests Azure Blob Storage --- packages/techdocs-common/__mocks__/@azure/identity.ts | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/techdocs-common/__mocks__/@azure/identity.ts diff --git a/packages/techdocs-common/__mocks__/@azure/identity.ts b/packages/techdocs-common/__mocks__/@azure/identity.ts new file mode 100644 index 0000000000..0fb8a75ebd --- /dev/null +++ b/packages/techdocs-common/__mocks__/@azure/identity.ts @@ -0,0 +1 @@ +export class DefaultAzureCredential {} From a2feeb288d08ff7e1f8daf9e323d584e7ee57e9b Mon Sep 17 00:00:00 2001 From: vitorgrenzel Date: Mon, 25 Jan 2021 11:24:12 -0300 Subject: [PATCH 09/10] feat(techdocs-common): update tests Azure Blob Storage --- .../__mocks__/@azure/identity.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/techdocs-common/__mocks__/@azure/identity.ts b/packages/techdocs-common/__mocks__/@azure/identity.ts index 0fb8a75ebd..cc89a4d514 100644 --- a/packages/techdocs-common/__mocks__/@azure/identity.ts +++ b/packages/techdocs-common/__mocks__/@azure/identity.ts @@ -1 +1,20 @@ -export class DefaultAzureCredential {} +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export class DefaultAzureCredential { + /** + * Creates an instance of the DefaultAzureCredential class. + */ +} From ee271bf86ed398d9cb27f26a588adbf5a8f87732 Mon Sep 17 00:00:00 2001 From: vitorgrenzel Date: Mon, 25 Jan 2021 11:46:25 -0300 Subject: [PATCH 10/10] feat(docs): update Azure Blob Storage --- docs/features/techdocs/configuration.md | 2 +- docs/features/techdocs/using-cloud-storage.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index f588be7c8c..bb44050dcf 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -92,7 +92,7 @@ techdocs: containerName: 'techdocs-storage' # (Required) An account name is required to write to a storage blob container. - # https://docs.microsoft.com/pt-br/rest/api/storageservices/authorize-with-shared-key + # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key credentials: accountName: $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 393e478302..de2d505c87 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -263,7 +263,7 @@ follow these steps. To get credentials, access the Azure Portal and go to "Settings > Access Keys", and get your Storage account name and Primary Key. -https://docs.microsoft.com/pt-br/rest/api/storageservices/authorize-with-shared-key +https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key for more details. ```yaml