diff --git a/app-config.yaml b/app-config.yaml index 09d2e75253..90cf73f58f 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -80,7 +80,7 @@ techdocs: generators: techdocs: 'docker' # Alternatives - 'local' publisher: - type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureStorage'. Read documentation for using alternatives. + type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureBlobStorage'. Read documentation for using alternatives. sentry: organization: my-company diff --git a/packages/techdocs-common/src/stages/publish/azureStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts similarity index 89% rename from packages/techdocs-common/src/stages/publish/azureStorage.test.ts rename to packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 05b5441bff..95e4a52f62 100644 --- a/packages/techdocs-common/src/stages/publish/azureStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -16,7 +16,7 @@ import mockFs from 'mock-fs'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; -import { AzureStoragePublish } from './azureStorage'; +import { AzureBlobStoragePublish } from './azureBlobStorage'; import { PublisherBase } from './types'; import type { Entity } from '@backstage/catalog-model'; @@ -54,8 +54,8 @@ beforeEach(async () => { techdocs: { requestUrl: 'http://localhost:7000', publisher: { - type: 'azureStorage', - azureStorage: { + type: 'azureBlobStorage', + azureBlobStorage: { credentials: { account: 'account', accountKey: 'accountKey', @@ -66,10 +66,10 @@ beforeEach(async () => { }, }); - publisher = await AzureStoragePublish.fromConfig(mockConfig, logger); + publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger); }); -describe('AzureStoragePublish', () => { +describe('AzureBlobStoragePublish', () => { describe('publish', () => { it('should publish a directory', async () => { const entity = createMockEntity(); @@ -117,7 +117,7 @@ describe('AzureStoragePublish', () => { .catch(error => expect(error).toEqual( new Error( - `Unable to upload file(s) to Azure Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`, + `Unable to upload file(s) to Azure Blob Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`, ), ), ); diff --git a/packages/techdocs-common/src/stages/publish/azureStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts similarity index 67% rename from packages/techdocs-common/src/stages/publish/azureStorage.ts rename to packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index bc9a6825d9..7ba4d35892 100644 --- a/packages/techdocs-common/src/stages/publish/azureStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -24,34 +24,45 @@ import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; -import { PublisherBase, PublishRequest } from './types'; +import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; +import limiterFactory from 'p-limit'; +import JSON5 from 'json5'; -export class AzureStoragePublish implements PublisherBase { +// The number of batches that may be ongoing at the same time. +const BATCH_CONCURRENCY = 3; + +export class AzureBlobStoragePublish implements PublisherBase { static async fromConfig( config: Config, logger: Logger, ): Promise { - let account = ''; - let accountKey = ''; let containerName = ''; try { - account = config.getString( - 'techdocs.publisher.azureStorage.credentials.account', - ); - accountKey = config.getString( - 'techdocs.publisher.azureStorage.credentials.accountKey', - ); containerName = config.getString( - 'techdocs.publisher.azureStorage.containerName', + 'techdocs.publisher.azureBlobStorage.containerName', ); } catch (error) { throw new Error( - "Since techdocs.publisher.type is set to 'azureStorage' in your app config, " + - 'credentials and containerName are required in techdocs.publisher.azureStorage ' + - 'required to authenticate with Azure Storage.', + "Since techdocs.publisher.type is set to 'awsS3' in your app config, " + + 'techdocs.publisher.awsS3.bucketName is required.', ); } + // Credentials is an optional config. If missing, default AWS environment variables + // or AWS shared credentials file at ~/.aws/credentials will be used to authenticate + // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html + // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html + let account = ''; + let accountKey = ''; + account = + config.getOptionalString( + 'techdocs.publisher.azureBlobStorage.credentials.account', + ) || ''; + accountKey = + config.getOptionalString( + 'techdocs.publisher.azureBlobStorage.credentials.accountKey', + ) || ''; + const credential = new StorageSharedKeyCredential(account, accountKey); const storageClient = new BlobServiceClient( `https://${account}.blob.core.windows.net`, @@ -63,20 +74,22 @@ export class AzureStoragePublish implements PublisherBase { .getProperties() .then(() => { logger.info( - `Successfully connected to the Azure Storage container ${containerName}.`, + `Successfully connected to the Azure Blob Storage container ${containerName}.`, ); }) .catch(reason => { logger.error( - `Could not retrieve metadata about the Azure Storage container ${containerName}. ` + + `Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` + 'Make sure the Azure project and the container exists and the access key located at the path ' + - "techdocs.publisher.azureStorage.credentials defined in app config has the role 'Storage Object Creator'. " + + "techdocs.publisher.azureBlobStorage.credentials defined in app config has the role 'Storage Object Creator'. " + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', ); - throw new Error(`from Azure Storage client library: ${reason.message}`); + throw new Error( + `from Azure Blob Storage client library: ${reason.message}`, + ); }); - return new AzureStoragePublish(storageClient, containerName, logger); + return new AzureBlobStoragePublish(storageClient, containerName, logger); } constructor( @@ -90,17 +103,23 @@ export class AzureStoragePublish implements PublisherBase { } /** - * Upload all the files from the generated `directory` to the Azure Storage container. + * Upload all the files from the generated `directory` to the Azure Blob Storage container. * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html */ async publish({ entity, directory }: PublishRequest): Promise { try { - // Note: Azure Storage manages creation of parent directories if they do not exist. + // Note: Azure Blob Storage manages creation of parent directories if they do not exist. // So collecting path of only the files is good enough. const allFilesToUpload = await getFileTreeRecursively(directory); const uploadPromises: Array> = []; - allFilesToUpload.forEach(filePath => { + + // Bound the number of concurrent batches. We want a bit of concurrency for + // performance reasons, but not so much that we starve the connection pool + // or start thrashing. + const limiter = limiterFactory(BATCH_CONCURRENCY); + + const promises = allFilesToUpload.map(filePath => { // Remove the absolute path prefix of the source directory // Path of all files to upload, relative to the root of the source directory // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] @@ -108,30 +127,33 @@ export class AzureStoragePublish implements PublisherBase { const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; const destination = path.normalize( `${entityRootDir}/${relativeFilePath}`, - ); // Azure Storage Container file relative path + ); // Azure Blob Storage Container file relative path + // TODO: Upload in chunks of ~10 files instead of all files at once. - uploadPromises.push( - this.storageClient - .getContainerClient(this.containerName) - .getBlockBlobClient(destination) - .uploadFile(filePath), - ); + return limiter(async () => { + await uploadPromises.push( + this.storageClient + .getContainerClient(this.containerName) + .getBlockBlobClient(destination) + .uploadFile(filePath), + ); + }); }); - await Promise.all(uploadPromises).then(() => { + await Promise.all(promises).then(() => { this.logger.info( `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, ); }); return; } catch (e) { - const errorMessage = `Unable to upload file(s) to Azure Storage. Error ${e.message}`; + const errorMessage = `Unable to upload file(s) to Azure Blob Storage. Error ${e.message}`; this.logger.error(errorMessage); throw new Error(errorMessage); } } - download(containerName: string, path: string): Promise { + private download(containerName: string, path: string): Promise { return new Promise((resolve, reject) => { const fileStreamChunks: Array = []; this.storageClient @@ -153,19 +175,24 @@ export class AzureStoragePublish implements PublisherBase { fileStreamChunks.push(chunk); }) .on('end', () => { - resolve(Buffer.concat(fileStreamChunks).toString()); + resolve(Buffer.concat(fileStreamChunks)); }); }); }); } - async fetchTechDocsMetadata(entityName: EntityName): Promise { + async fetchTechDocsMetadata( + entityName: EntityName, + ): Promise { const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; try { - return this.download( - this.containerName, - `${entityRootDir}/techdocs_metadata.json`, - ); + return await new Promise(resolve => { + const download = this.download( + this.containerName, + `${entityRootDir}/techdocs_metadata.json`, + ); + resolve(JSON5.parse(download.toString())); + }); } catch (e) { this.logger.error(e.message); throw e; diff --git a/packages/techdocs-common/src/stages/publish/publish.test.ts b/packages/techdocs-common/src/stages/publish/publish.test.ts index a97eadb404..a563e53a9b 100644 --- a/packages/techdocs-common/src/stages/publish/publish.test.ts +++ b/packages/techdocs-common/src/stages/publish/publish.test.ts @@ -22,7 +22,7 @@ import { Publisher } from './publish'; import { LocalPublish } from './local'; import { GoogleGCSPublish } from './googleStorage'; import { AwsS3Publish } from './awsS3'; -import { AzureStoragePublish } from './azureStorage'; +import { AzureBlobStoragePublish } from './azureBlobStorage'; const logger = getVoidLogger(); const discovery: jest.Mocked = { @@ -107,13 +107,13 @@ describe('Publisher', () => { expect(publisher).toBeInstanceOf(AwsS3Publish); }); - it('should create Azure Storage publisher from config', async () => { + it('should create Azure Blob Storage publisher from config', async () => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', publisher: { - type: 'azureStorage', - azureStorage: { + type: 'azureBlobStorage', + azureBlobStorage: { credentials: { account: 'account', accountKey: 'accountKey', @@ -128,6 +128,6 @@ describe('Publisher', () => { logger, discovery, }); - expect(publisher).toBeInstanceOf(AzureStoragePublish); + expect(publisher).toBeInstanceOf(AzureBlobStoragePublish); }); }); diff --git a/packages/techdocs-common/src/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts index fc72d2812e..a4e33a2d9c 100644 --- a/packages/techdocs-common/src/stages/publish/publish.ts +++ b/packages/techdocs-common/src/stages/publish/publish.ts @@ -21,7 +21,7 @@ import { PublisherType, PublisherBase } from './types'; import { LocalPublish } from './local'; import { GoogleGCSPublish } from './googleStorage'; import { AwsS3Publish } from './awsS3'; -import { AzureStoragePublish } from './azureStorage'; +import { AzureBlobStoragePublish } from './azureBlobStorage'; type factoryOptions = { logger: Logger; @@ -48,9 +48,11 @@ export class Publisher { case 'awsS3': logger.info('Creating AWS S3 Bucket publisher for TechDocs'); return AwsS3Publish.fromConfig(config, logger); - case 'azureStorage': - logger.info('Creating Azure Storage Container publisher for TechDocs'); - return AzureStoragePublish.fromConfig(config, logger); + case 'azureBlobStorage': + logger.info( + 'Creating Azure Blob Storage Container publisher for TechDocs', + ); + return AzureBlobStoragePublish.fromConfig(config, logger); case 'local': logger.info('Creating Local publisher for TechDocs'); return new LocalPublish(config, logger, discovery); diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index f67f65ee23..5e953deb81 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -19,7 +19,11 @@ import express from 'express'; /** * Key for all the different types of TechDocs publishers that are supported. */ -export type PublisherType = 'local' | 'googleGcs' | 'awsS3' | 'azureStorage'; +export type PublisherType = + | 'local' + | 'googleGcs' + | 'awsS3' + | 'azureBlobStorage'; export type PublishRequest = { entity: Entity; diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index cbb6efa3dd..dc5e6a4911 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -148,7 +148,7 @@ export async function createRouter({ } break; case 'awsS3': - case 'azureStorage': + case 'azureBlobStorage': case 'googleGcs': // This block should be valid for all external storage implementations. So no need to duplicate in future, // add the publisher type in the list here. diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index 1dd7319f61..f7903ea3fb 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -117,21 +117,23 @@ export interface Config { | { /** * attr: 'type' - accepts a string value - * e.g. type: 'azureStorage' - * alternatives: 'azureStorage' etc. + * e.g. type: 'azureBlobStorage' + * alternatives: 'azureBlobStorage' etc. * @see http://backstage.io/docs/features/techdocs/configuration */ - type: 'azureStorage'; + type: 'azureBlobStorage'; /** - * azureStorage required when 'type' is set to azureStorage + * azureBlobStorage required when 'type' is set to azureBlobStorage */ - azureStorage?: { + azureBlobStorage?: { /** - * Credentials used to access a storage container + * (Optional) Credentials used to access a storage container. + * If not set, environment variables will be used to authenticate. + * https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json * @visibility secret */ - credentials: { + credentials?: { /** * Account access name * attr: 'account' - accepts a string value @@ -146,7 +148,7 @@ export interface Config { accountKey: string; }; /** - * Cloud Storage Container Name + * (Required) Cloud Storage Container Name * attr: 'containerName' - accepts a string value * @visibility backend */