paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 50 Packages Projects Releases Wiki Activity

feat(techdocs-common): add Azure Blob Storage

Browse Source
This commit is contained in:
vitorgrenzel
2021-01-20 17:21:40 -03:00
committed by Tiago A. Simões
parent 42494c7e91
commit 59b8d5a0d9
8 changed files with 99 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app-config.yaml
+1 -1
View File
@@ -80,7 +80,7 @@ techdocs:
generators:
techdocs: 'docker' # Alternatives - 'local'
publisher:
type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureStorage'. Read documentation for using alternatives.
type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureBlobStorage'. Read documentation for using alternatives.
sentry:
organization: my-company
packages/techdocs-common/src/stages/publish/azureStorage.test.ts → packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
+6 -6
View File
@@ -16,7 +16,7 @@
import mockFs from 'mock-fs';
import { ConfigReader } from '@backstage/config';
import { getVoidLogger } from '@backstage/backend-common';
import { AzureStoragePublish } from './azureStorage';
import { AzureBlobStoragePublish } from './azureBlobStorage';
import { PublisherBase } from './types';
import type { Entity } from '@backstage/catalog-model';
@@ -54,8 +54,8 @@ beforeEach(async () => {
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureStorage',
azureStorage: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
account: 'account',
accountKey: 'accountKey',
@@ -66,10 +66,10 @@ beforeEach(async () => {
},
});
publisher = await AzureStoragePublish.fromConfig(mockConfig, logger);
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
});
describe('AzureStoragePublish', () => {
describe('AzureBlobStoragePublish', () => {
describe('publish', () => {
it('should publish a directory', async () => {
const entity = createMockEntity();
@@ -117,7 +117,7 @@ describe('AzureStoragePublish', () => {
.catch(error =>
expect(error).toEqual(
new Error(
`Unable to upload file(s) to Azure Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
`Unable to upload file(s) to Azure Blob Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
),
),
);
packages/techdocs-common/src/stages/publish/azureStorage.ts → packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
+65 -38
View File
@@ -24,34 +24,45 @@ import { Logger } from 'winston';
import { Entity, EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
import { PublisherBase, PublishRequest } from './types';
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
import limiterFactory from 'p-limit';
import JSON5 from 'json5';
export class AzureStoragePublish implements PublisherBase {
// The number of batches that may be ongoing at the same time.
const BATCH_CONCURRENCY = 3;
export class AzureBlobStoragePublish implements PublisherBase {
static async fromConfig(
config: Config,
logger: Logger,
): Promise<PublisherBase> {
let account = '';
let accountKey = '';
let containerName = '';
try {
account = config.getString(
'techdocs.publisher.azureStorage.credentials.account',
);
accountKey = config.getString(
'techdocs.publisher.azureStorage.credentials.accountKey',
);
containerName = config.getString(
'techdocs.publisher.azureStorage.containerName',
'techdocs.publisher.azureBlobStorage.containerName',
);
} catch (error) {
throw new Error(
"Since techdocs.publisher.type is set to 'azureStorage' in your app config, " +
'credentials and containerName are required in techdocs.publisher.azureStorage ' +
'required to authenticate with Azure Storage.',
"Since techdocs.publisher.type is set to 'awsS3' in your app config, " +
'techdocs.publisher.awsS3.bucketName is required.',
);
}
// Credentials is an optional config. If missing, default AWS environment variables
// or AWS shared credentials file at ~/.aws/credentials will be used to authenticate
// https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html
// https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html
let account = '';
let accountKey = '';
account =
config.getOptionalString(
'techdocs.publisher.azureBlobStorage.credentials.account',
) || '';
accountKey =
config.getOptionalString(
'techdocs.publisher.azureBlobStorage.credentials.accountKey',
) || '';
const credential = new StorageSharedKeyCredential(account, accountKey);
const storageClient = new BlobServiceClient(
`https://${account}.blob.core.windows.net`,
@@ -63,20 +74,22 @@ export class AzureStoragePublish implements PublisherBase {
.getProperties()
.then(() => {
logger.info(
`Successfully connected to the Azure Storage container ${containerName}.`,
`Successfully connected to the Azure Blob Storage container ${containerName}.`,
);
})
.catch(reason => {
logger.error(
`Could not retrieve metadata about the Azure Storage container ${containerName}. ` +
`Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` +
'Make sure the Azure project and the container exists and the access key located at the path ' +
"techdocs.publisher.azureStorage.credentials defined in app config has the role 'Storage Object Creator'. " +
"techdocs.publisher.azureBlobStorage.credentials defined in app config has the role 'Storage Object Creator'. " +
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
throw new Error(`from Azure Storage client library: ${reason.message}`);
throw new Error(
`from Azure Blob Storage client library: ${reason.message}`,
);
});
return new AzureStoragePublish(storageClient, containerName, logger);
return new AzureBlobStoragePublish(storageClient, containerName, logger);
}
constructor(
@@ -90,17 +103,23 @@ export class AzureStoragePublish implements PublisherBase {
}
/**
* Upload all the files from the generated `directory` to the Azure Storage container.
* Upload all the files from the generated `directory` to the Azure Blob Storage container.
* Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
try {
// Note: Azure Storage manages creation of parent directories if they do not exist.
// Note: Azure Blob Storage manages creation of parent directories if they do not exist.
// So collecting path of only the files is good enough.
const allFilesToUpload = await getFileTreeRecursively(directory);
const uploadPromises: Array<Promise<BlobUploadCommonResponse>> = [];
allFilesToUpload.forEach(filePath => {
// Bound the number of concurrent batches. We want a bit of concurrency for
// performance reasons, but not so much that we starve the connection pool
// or start thrashing.
const limiter = limiterFactory(BATCH_CONCURRENCY);
const promises = allFilesToUpload.map(filePath => {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
@@ -108,30 +127,33 @@ export class AzureStoragePublish implements PublisherBase {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const destination = path.normalize(
`${entityRootDir}/${relativeFilePath}`,
); // Azure Storage Container file relative path
); // Azure Blob Storage Container file relative path
// TODO: Upload in chunks of ~10 files instead of all files at once.
uploadPromises.push(
this.storageClient
.getContainerClient(this.containerName)
.getBlockBlobClient(destination)
.uploadFile(filePath),
);
return limiter(async () => {
await uploadPromises.push(
this.storageClient
.getContainerClient(this.containerName)
.getBlockBlobClient(destination)
.uploadFile(filePath),
);
});
});
await Promise.all(uploadPromises).then(() => {
await Promise.all(promises).then(() => {
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
);
});
return;
} catch (e) {
const errorMessage = `Unable to upload file(s) to Azure Storage. Error ${e.message}`;
const errorMessage = `Unable to upload file(s) to Azure Blob Storage. Error ${e.message}`;
this.logger.error(errorMessage);
throw new Error(errorMessage);
}
}
download(containerName: string, path: string): Promise<string> {
private download(containerName: string, path: string): Promise<Buffer> {
return new Promise((resolve, reject) => {
const fileStreamChunks: Array<any> = [];
this.storageClient
@@ -153,19 +175,24 @@ export class AzureStoragePublish implements PublisherBase {
fileStreamChunks.push(chunk);
})
.on('end', () => {
resolve(Buffer.concat(fileStreamChunks).toString());
resolve(Buffer.concat(fileStreamChunks));
});
});
});
}
async fetchTechDocsMetadata(entityName: EntityName): Promise<string> {
async fetchTechDocsMetadata(
entityName: EntityName,
): Promise<TechDocsMetadata> {
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
try {
return this.download(
this.containerName,
`${entityRootDir}/techdocs_metadata.json`,
);
return await new Promise<TechDocsMetadata>(resolve => {
const download = this.download(
this.containerName,
`${entityRootDir}/techdocs_metadata.json`,
);
resolve(JSON5.parse(download.toString()));
});
} catch (e) {
this.logger.error(e.message);
throw e;
packages/techdocs-common/src/stages/publish/publish.test.ts
+5 -5
View File
@@ -22,7 +22,7 @@ import { Publisher } from './publish';
import { LocalPublish } from './local';
import { GoogleGCSPublish } from './googleStorage';
import { AwsS3Publish } from './awsS3';
import { AzureStoragePublish } from './azureStorage';
import { AzureBlobStoragePublish } from './azureBlobStorage';
const logger = getVoidLogger();
const discovery: jest.Mocked<PluginEndpointDiscovery> = {
@@ -107,13 +107,13 @@ describe('Publisher', () => {
expect(publisher).toBeInstanceOf(AwsS3Publish);
});
it('should create Azure Storage publisher from config', async () => {
it('should create Azure Blob Storage publisher from config', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureStorage',
azureStorage: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
account: 'account',
accountKey: 'accountKey',
@@ -128,6 +128,6 @@ describe('Publisher', () => {
logger,
discovery,
});
expect(publisher).toBeInstanceOf(AzureStoragePublish);
expect(publisher).toBeInstanceOf(AzureBlobStoragePublish);
});
});
packages/techdocs-common/src/stages/publish/publish.ts
+6 -4
View File
@@ -21,7 +21,7 @@ import { PublisherType, PublisherBase } from './types';
import { LocalPublish } from './local';
import { GoogleGCSPublish } from './googleStorage';
import { AwsS3Publish } from './awsS3';
import { AzureStoragePublish } from './azureStorage';
import { AzureBlobStoragePublish } from './azureBlobStorage';
type factoryOptions = {
logger: Logger;
@@ -48,9 +48,11 @@ export class Publisher {
case 'awsS3':
logger.info('Creating AWS S3 Bucket publisher for TechDocs');
return AwsS3Publish.fromConfig(config, logger);
case 'azureStorage':
logger.info('Creating Azure Storage Container publisher for TechDocs');
return AzureStoragePublish.fromConfig(config, logger);
case 'azureBlobStorage':
logger.info(
'Creating Azure Blob Storage Container publisher for TechDocs',
);
return AzureBlobStoragePublish.fromConfig(config, logger);
case 'local':
logger.info('Creating Local publisher for TechDocs');
return new LocalPublish(config, logger, discovery);
packages/techdocs-common/src/stages/publish/types.ts
+5 -1
View File
@@ -19,7 +19,11 @@ import express from 'express';
/**
* Key for all the different types of TechDocs publishers that are supported.
*/
export type PublisherType = 'local' | 'googleGcs' | 'awsS3' | 'azureStorage';
export type PublisherType =
| 'local'
| 'googleGcs'
| 'awsS3'
| 'azureBlobStorage';
export type PublishRequest = {
entity: Entity;
plugins/techdocs-backend/src/service/router.ts
+1 -1
View File
@@ -148,7 +148,7 @@ export async function createRouter({
}
break;
case 'awsS3':
case 'azureStorage':
case 'azureBlobStorage':
case 'googleGcs':
// This block should be valid for all external storage implementations. So no need to duplicate in future,
// add the publisher type in the list here.
plugins/techdocs/config.d.ts
Vendored
+10 -8
View File
@@ -117,21 +117,23 @@ export interface Config {
| {
/**
* attr: 'type' - accepts a string value
* e.g. type: 'azureStorage'
* alternatives: 'azureStorage' etc.
* e.g. type: 'azureBlobStorage'
* alternatives: 'azureBlobStorage' etc.
* @see http://backstage.io/docs/features/techdocs/configuration
*/
type: 'azureStorage';
type: 'azureBlobStorage';
/**
* azureStorage required when 'type' is set to azureStorage
* azureBlobStorage required when 'type' is set to azureBlobStorage
*/
azureStorage?: {
azureBlobStorage?: {
/**
* Credentials used to access a storage container
* (Optional) Credentials used to access a storage container.
* If not set, environment variables will be used to authenticate.
* https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json
* @visibility secret
*/
credentials: {
credentials?: {
/**
* Account access name
* attr: 'account' - accepts a string value
@@ -146,7 +148,7 @@ export interface Config {
accountKey: string;
};
/**
* Cloud Storage Container Name
* (Required) Cloud Storage Container Name
* attr: 'containerName' - accepts a string value
* @visibility backend
*/