Merge pull request #4017 from viavarejo/feature/techdocs-azure-storage
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
---
|
||||
'@backstage/techdocs-common': patch
|
||||
'@backstage/plugin-techdocs-backend': patch
|
||||
---
|
||||
|
||||
1. Added option to use Azure Blob Storage as a choice to store the static generated files for TechDocs.
|
||||
+1
-1
@@ -80,7 +80,7 @@ techdocs:
|
||||
generators:
|
||||
techdocs: 'docker' # Alternatives - 'local'
|
||||
publisher:
|
||||
type: 'local' # Alternatives - 'googleGcs' or 'awsS3'. Read documentation for using alternatives.
|
||||
type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureBlobStorage'. Read documentation for using alternatives.
|
||||
|
||||
sentry:
|
||||
organization: my-company
|
||||
|
||||
@@ -108,12 +108,12 @@ providers are used.
|
||||
| GitLab | Yes ✅ |
|
||||
| GitLab Enterprise | Yes ✅ |
|
||||
|
||||
| File Storage Provider | Support Status |
|
||||
| --------------------------------- | ----------------------------------------------------------------- |
|
||||
| Local Filesystem of Backstage app | Yes ✅ |
|
||||
| Google Cloud Storage (GCS) | Yes ✅ |
|
||||
| Amazon Web Services (AWS) S3 | Yes ✅ |
|
||||
| Azure Storage | No ❌ [#3938](https://github.com/backstage/backstage/issues/3938) |
|
||||
| File Storage Provider | Support Status |
|
||||
| --------------------------------- | -------------- |
|
||||
| Local Filesystem of Backstage app | Yes ✅ |
|
||||
| Google Cloud Storage (GCS) | Yes ✅ |
|
||||
| Amazon Web Services (AWS) S3 | Yes ✅ |
|
||||
| Azure Blob Storage | Yes ✅ |
|
||||
|
||||
[Reach out to us](#feedback) if you want to request more platforms.
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ techdocs:
|
||||
# or you want to use External storage providers like Google Cloud Storage, AWS S3, etc.
|
||||
|
||||
publisher:
|
||||
# techdocs.publisher.type can be - 'local' or 'googleGcs' or 'awsS3' (azureStorage to be available in future).
|
||||
# techdocs.publisher.type can be - 'local' or 'googleGcs' or 'awsS3' or 'azureBlobStorage'.
|
||||
# When set to 'local', techdocs-backend will create a 'static' directory at its root to store generated documentation files.
|
||||
# When set to 'googleGcs', techdocs-backend will use a Google Cloud Storage Bucket to store generated documentation files.
|
||||
# When set to 'awsS3', techdocs-backend will use an Amazon Web Service (AWS) S3 bucket to store generated documentation files.
|
||||
@@ -84,4 +84,21 @@ techdocs:
|
||||
# https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html
|
||||
region:
|
||||
$env: AWS_REGION
|
||||
|
||||
# Required when techdocs.publisher.type is set to 'azureBlobStorage'. Skip otherwise.
|
||||
|
||||
azureBlobStorage:
|
||||
# (Required) Azure Blob Storage Container Name
|
||||
containerName: 'techdocs-storage'
|
||||
|
||||
# (Required) An account name is required to write to a storage blob container.
|
||||
# https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key
|
||||
credentials:
|
||||
accountName:
|
||||
$env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME
|
||||
# (Optional) An account key is required to write to a storage container.
|
||||
# If missing,AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET environment variable will be used.
|
||||
# https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json
|
||||
accountKey:
|
||||
$env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY
|
||||
```
|
||||
|
||||
@@ -195,3 +195,94 @@ Your Backstage app is now ready to use AWS S3 for TechDocs, to store and read
|
||||
the static generated documentation files. When you start the backend of the app,
|
||||
you should be able to see
|
||||
`techdocs info Successfully connected to the AWS S3 bucket` in the logs.
|
||||
|
||||
## Configuring Azure Blob Storage Container with TechDocs
|
||||
|
||||
Follow the
|
||||
[official Azure Blob Storage documentation](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json)
|
||||
for the latest instructions on the following steps involving Azure Blob Storage.
|
||||
|
||||
**1. Set `techdocs.publisher.type` config in your `app-config.yaml`**
|
||||
|
||||
Set `techdocs.publisher.type` to `'azureBlobStorage'`.
|
||||
|
||||
```yaml
|
||||
techdocs:
|
||||
publisher:
|
||||
type: 'azureBlobStorage'
|
||||
```
|
||||
|
||||
**2. Create an Azure Blob Storage Container**
|
||||
|
||||
Create a dedicated container for TechDocs sites.
|
||||
[Refer to the official documentation](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-portal).
|
||||
|
||||
TechDocs will publish documentation to this container and will fetch files from
|
||||
here to serve documentation in Backstage. Note that the container names are
|
||||
globally unique.
|
||||
|
||||
Set the config `techdocs.publisher.azureBlobStorage.containerName` in your
|
||||
`app-config.yaml` to the name of the container you just created.
|
||||
|
||||
```yaml
|
||||
techdocs:
|
||||
publisher:
|
||||
type: 'azureBlobStorage'
|
||||
azureBlobStorage:
|
||||
containerName: 'name-of-techdocs-storage-container'
|
||||
```
|
||||
|
||||
**3a. (Recommended) Authentication using environment variable**
|
||||
|
||||
Set the config `techdocs.publisher.azureBlobStorage.credentials.accountName` in
|
||||
your `app-config.yaml` to the your account name.
|
||||
|
||||
The storage blob client will automatically use the environment variable
|
||||
`AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET` to authenticate with
|
||||
Azure Blob Storage.
|
||||
[Steps to create the service where the variables can be retrieved from](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal).
|
||||
|
||||
https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad for more
|
||||
details.
|
||||
|
||||
```yaml
|
||||
techdocs:
|
||||
publisher:
|
||||
type: 'azureBlobStorage'
|
||||
azureBlobStorage:
|
||||
containerName: 'name-of-techdocs-storage-bucket'
|
||||
credentials:
|
||||
accountName:
|
||||
$env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME
|
||||
```
|
||||
|
||||
**3b. Authentication using app-config.yaml**
|
||||
|
||||
If you do not prefer (3a) and optionally like to use a service account, you can
|
||||
follow these steps.
|
||||
|
||||
To get credentials, access the Azure Portal and go to "Settings > Access Keys",
|
||||
and get your Storage account name and Primary Key.
|
||||
https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key
|
||||
for more details.
|
||||
|
||||
```yaml
|
||||
techdocs:
|
||||
publisher:
|
||||
type: 'azureBlobStorage'
|
||||
azureBlobStorage:
|
||||
containerName: 'name-of-techdocs-storage-bucket'
|
||||
credentials:
|
||||
accountName:
|
||||
$env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME
|
||||
accountKey:
|
||||
$env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY
|
||||
```
|
||||
|
||||
**4. That's it!**
|
||||
|
||||
Your Backstage app is now ready to use Azure Blob Storage for TechDocs, to store
|
||||
and read the static generated documentation files. When you start the backend of
|
||||
the app, you should be able to see
|
||||
`techdocs info Successfully connected to the Azure Blob Storage container` in
|
||||
the logs.
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export class DefaultAzureCredential {
|
||||
/**
|
||||
* Creates an instance of the DefaultAzureCredential class.
|
||||
*/
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import fs from 'fs';
|
||||
|
||||
export class BlockBlobClient {
|
||||
private readonly blobName;
|
||||
|
||||
constructor(blobName: string) {
|
||||
this.blobName = blobName;
|
||||
}
|
||||
|
||||
uploadFile(source: string) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!fs.existsSync(source)) {
|
||||
reject('');
|
||||
} else {
|
||||
resolve('');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
exists() {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (fs.existsSync(this.blobName)) {
|
||||
resolve(true);
|
||||
} else {
|
||||
reject({ message: 'The object doest not exist !' });
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export class ContainerClient {
|
||||
private readonly containerName;
|
||||
|
||||
constructor(containerName: string) {
|
||||
this.containerName = containerName;
|
||||
}
|
||||
|
||||
getProperties() {
|
||||
return new Promise(resolve => {
|
||||
resolve('');
|
||||
});
|
||||
}
|
||||
|
||||
getBlockBlobClient(blobName: string) {
|
||||
return new BlockBlobClient(blobName);
|
||||
}
|
||||
}
|
||||
|
||||
export class BlobServiceClient {
|
||||
private readonly url;
|
||||
private readonly credential;
|
||||
|
||||
constructor(url: string, credential?: StorageSharedKeyCredential) {
|
||||
this.url = url;
|
||||
this.credential = credential;
|
||||
}
|
||||
|
||||
getContainerClient(containerName: string) {
|
||||
return new ContainerClient(containerName);
|
||||
}
|
||||
}
|
||||
|
||||
export class StorageSharedKeyCredential {
|
||||
private readonly accountName;
|
||||
private readonly accountKey;
|
||||
|
||||
constructor(accountName: string, accountKey: string) {
|
||||
this.accountName = accountName;
|
||||
this.accountKey = accountKey;
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,8 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.1.0",
|
||||
"@azure/identity": "^1.2.2",
|
||||
"@azure/storage-blob": "^12.4.0",
|
||||
"@backstage/backend-common": "^0.5.1",
|
||||
"@backstage/catalog-model": "^0.7.0",
|
||||
"@backstage/config": "^0.1.2",
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import mockFs from 'mock-fs';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { AzureBlobStoragePublish } from './azureBlobStorage';
|
||||
import { PublisherBase } from './types';
|
||||
import type { Entity } from '@backstage/catalog-model';
|
||||
|
||||
const createMockEntity = (annotations = {}) => {
|
||||
return {
|
||||
apiVersion: 'version',
|
||||
kind: 'TestKind',
|
||||
metadata: {
|
||||
name: 'test-component-name',
|
||||
namespace: 'test-namespace',
|
||||
annotations: {
|
||||
...annotations,
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const getEntityRootDir = (entity: Entity) => {
|
||||
const {
|
||||
kind,
|
||||
metadata: { namespace, name },
|
||||
} = entity;
|
||||
const entityRootDir = `${namespace}/${kind}/${name}`;
|
||||
return entityRootDir;
|
||||
};
|
||||
|
||||
const logger = getVoidLogger();
|
||||
jest.spyOn(logger, 'info').mockReturnValue(logger);
|
||||
jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
|
||||
let publisher: PublisherBase;
|
||||
|
||||
beforeEach(async () => {
|
||||
const mockConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
requestUrl: 'http://localhost:7000',
|
||||
publisher: {
|
||||
type: 'azureBlobStorage',
|
||||
azureBlobStorage: {
|
||||
credentials: {
|
||||
accountName: 'accountName',
|
||||
accountKey: 'accountKey',
|
||||
},
|
||||
containerName: 'containerName',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
|
||||
});
|
||||
|
||||
describe('AzureBlobStoragePublish', () => {
|
||||
describe('publish', () => {
|
||||
it('should publish a directory', async () => {
|
||||
const entity = createMockEntity();
|
||||
const entityRootDir = getEntityRootDir(entity);
|
||||
|
||||
mockFs({
|
||||
[entityRootDir]: {
|
||||
'index.html': '',
|
||||
'404.html': '',
|
||||
assets: {
|
||||
'main.css': '',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
await publisher.publish({
|
||||
entity,
|
||||
directory: entityRootDir,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
mockFs.restore();
|
||||
});
|
||||
|
||||
it('should fail to publish a directory', async () => {
|
||||
const wrongPathToGeneratedDirectory = 'wrong/path/to/generatedDirectory';
|
||||
const entity = createMockEntity();
|
||||
const entityRootDir = getEntityRootDir(entity);
|
||||
|
||||
mockFs({
|
||||
[entityRootDir]: {
|
||||
'index.html': '',
|
||||
'404.html': '',
|
||||
assets: {
|
||||
'main.css': '',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await publisher
|
||||
.publish({
|
||||
entity,
|
||||
directory: wrongPathToGeneratedDirectory,
|
||||
})
|
||||
.catch(error =>
|
||||
expect(error).toEqual(
|
||||
new Error(
|
||||
`Unable to upload file(s) to Azure Blob Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
|
||||
),
|
||||
),
|
||||
);
|
||||
mockFs.restore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('hasDocsBeenGenerated', () => {
|
||||
it('should return true if docs has been generated', async () => {
|
||||
const entity = createMockEntity();
|
||||
const entityRootDir = getEntityRootDir(entity);
|
||||
|
||||
mockFs({
|
||||
[entityRootDir]: {
|
||||
'index.html': 'file-content',
|
||||
},
|
||||
});
|
||||
|
||||
expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
|
||||
mockFs.restore();
|
||||
});
|
||||
|
||||
it('should return false if docs has not been generated', async () => {
|
||||
const entity = createMockEntity();
|
||||
|
||||
expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,259 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import path from 'path';
|
||||
import express from 'express';
|
||||
import {
|
||||
BlobServiceClient,
|
||||
BlobUploadCommonResponse,
|
||||
StorageSharedKeyCredential,
|
||||
} from '@azure/storage-blob';
|
||||
import { DefaultAzureCredential } from '@azure/identity';
|
||||
import { Logger } from 'winston';
|
||||
import { Entity, EntityName } from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
|
||||
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
|
||||
import limiterFactory from 'p-limit';
|
||||
import JSON5 from 'json5';
|
||||
|
||||
// The number of batches that may be ongoing at the same time.
|
||||
const BATCH_CONCURRENCY = 3;
|
||||
|
||||
export class AzureBlobStoragePublish implements PublisherBase {
|
||||
static async fromConfig(
|
||||
config: Config,
|
||||
logger: Logger,
|
||||
): Promise<PublisherBase> {
|
||||
let containerName = '';
|
||||
try {
|
||||
containerName = config.getString(
|
||||
'techdocs.publisher.azureBlobStorage.containerName',
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
"Since techdocs.publisher.type is set to 'azureBlobStorage' in your app config, " +
|
||||
'techdocs.publisher.azureBlobStorage.containerName is required.',
|
||||
);
|
||||
}
|
||||
|
||||
let accountName = '';
|
||||
try {
|
||||
accountName = config.getString(
|
||||
'techdocs.publisher.azureBlobStorage.credentials.accountName',
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
"Since techdocs.publisher.type is set to 'azureBlobStorage' in your app config, " +
|
||||
'techdocs.publisher.azureBlobStorage.credentials.accountName is required.',
|
||||
);
|
||||
}
|
||||
|
||||
// Credentials is an optional config. If missing, default Azure Blob Storage environment variables will be used.
|
||||
// https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-app
|
||||
const accountKey = config.getOptionalString(
|
||||
'techdocs.publisher.azureBlobStorage.credentials.accountKey',
|
||||
);
|
||||
|
||||
let credential;
|
||||
if (accountKey) {
|
||||
credential = new StorageSharedKeyCredential(accountName, accountKey);
|
||||
} else {
|
||||
credential = new DefaultAzureCredential();
|
||||
}
|
||||
|
||||
const storageClient = new BlobServiceClient(
|
||||
`https://${accountName}.blob.core.windows.net`,
|
||||
credential,
|
||||
);
|
||||
|
||||
await storageClient
|
||||
.getContainerClient(containerName)
|
||||
.getProperties()
|
||||
.then(() => {
|
||||
logger.info(
|
||||
`Successfully connected to the Azure Blob Storage container ${containerName}.`,
|
||||
);
|
||||
})
|
||||
.catch(reason => {
|
||||
logger.error(
|
||||
`Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` +
|
||||
'Make sure that the Azure project and container exist and the access key is setup correctly ' +
|
||||
'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' +
|
||||
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
throw new Error(
|
||||
`from Azure Blob Storage client library: ${reason.message}`,
|
||||
);
|
||||
});
|
||||
|
||||
return new AzureBlobStoragePublish(storageClient, containerName, logger);
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly storageClient: BlobServiceClient,
|
||||
private readonly containerName: string,
|
||||
private readonly logger: Logger,
|
||||
) {
|
||||
this.storageClient = storageClient;
|
||||
this.containerName = containerName;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload all the files from the generated `directory` to the Azure Blob Storage container.
|
||||
* Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
|
||||
*/
|
||||
async publish({ entity, directory }: PublishRequest): Promise<void> {
|
||||
try {
|
||||
// Note: Azure Blob Storage manages creation of parent directories if they do not exist.
|
||||
// So collecting path of only the files is good enough.
|
||||
const allFilesToUpload = await getFileTreeRecursively(directory);
|
||||
|
||||
const uploadPromises: Array<Promise<BlobUploadCommonResponse>> = [];
|
||||
|
||||
// Bound the number of concurrent batches. We want a bit of concurrency for
|
||||
// performance reasons, but not so much that we starve the connection pool
|
||||
// or start thrashing.
|
||||
const limiter = limiterFactory(BATCH_CONCURRENCY);
|
||||
|
||||
const promises = allFilesToUpload.map(filePath => {
|
||||
// Remove the absolute path prefix of the source directory
|
||||
// Path of all files to upload, relative to the root of the source directory
|
||||
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
|
||||
const relativeFilePath = filePath.replace(`${directory}/`, '');
|
||||
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
|
||||
const destination = path.normalize(
|
||||
`${entityRootDir}/${relativeFilePath}`,
|
||||
); // Azure Blob Storage Container file relative path
|
||||
|
||||
return limiter(async () => {
|
||||
await uploadPromises.push(
|
||||
this.storageClient
|
||||
.getContainerClient(this.containerName)
|
||||
.getBlockBlobClient(destination)
|
||||
.uploadFile(filePath),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
await Promise.all(promises).then(() => {
|
||||
this.logger.info(
|
||||
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
|
||||
);
|
||||
});
|
||||
return;
|
||||
} catch (e) {
|
||||
const errorMessage = `Unable to upload file(s) to Azure Blob Storage. Error ${e.message}`;
|
||||
this.logger.error(errorMessage);
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
private download(containerName: string, path: string): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const fileStreamChunks: Array<any> = [];
|
||||
this.storageClient
|
||||
.getContainerClient(containerName)
|
||||
.getBlockBlobClient(path)
|
||||
.download()
|
||||
.then(res => {
|
||||
const body = res.readableStreamBody;
|
||||
if (!body) {
|
||||
reject(new Error(`Unable to parse the response data`));
|
||||
return;
|
||||
}
|
||||
body
|
||||
.on('error', e => {
|
||||
this.logger.error(e.message);
|
||||
reject(e.message);
|
||||
})
|
||||
.on('data', chunk => {
|
||||
fileStreamChunks.push(chunk);
|
||||
})
|
||||
.on('end', () => {
|
||||
resolve(Buffer.concat(fileStreamChunks));
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async fetchTechDocsMetadata(
|
||||
entityName: EntityName,
|
||||
): Promise<TechDocsMetadata> {
|
||||
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
|
||||
try {
|
||||
return await new Promise<TechDocsMetadata>(resolve => {
|
||||
const download = this.download(
|
||||
this.containerName,
|
||||
`${entityRootDir}/techdocs_metadata.json`,
|
||||
);
|
||||
resolve(JSON5.parse(download.toString()));
|
||||
});
|
||||
} catch (e) {
|
||||
this.logger.error(e.message);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Express route middleware to serve static files on a route in techdocs-backend.
|
||||
*/
|
||||
docsRouter(): express.Handler {
|
||||
return (req, res) => {
|
||||
// Trim the leading forward slash
|
||||
// filePath example - /default/Component/documented-component/index.html
|
||||
const filePath = req.path.replace(/^\//, '');
|
||||
// Files with different extensions (CSS, HTML) need to be served with different headers
|
||||
const fileExtension = path.extname(filePath);
|
||||
const responseHeaders = getHeadersForFileExtension(fileExtension);
|
||||
|
||||
try {
|
||||
this.download(this.containerName, filePath).then(fileContent => {
|
||||
// Inject response headers
|
||||
for (const [headerKey, headerValue] of Object.entries(
|
||||
responseHeaders,
|
||||
)) {
|
||||
res.setHeader(headerKey, headerValue);
|
||||
}
|
||||
res.send(fileContent);
|
||||
});
|
||||
} catch (e) {
|
||||
this.logger.error(e.message);
|
||||
res.status(404).send(e.message);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A helper function which checks if index.html of an Entity's docs site is available. This
|
||||
* can be used to verify if there are any pre-generated docs available to serve.
|
||||
*/
|
||||
async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
|
||||
return new Promise(resolve => {
|
||||
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
|
||||
this.storageClient
|
||||
.getContainerClient(this.containerName)
|
||||
.getBlockBlobClient(`${entityRootDir}/index.html`)
|
||||
.exists()
|
||||
.then((response: boolean) => {
|
||||
resolve(response);
|
||||
})
|
||||
.catch(() => {
|
||||
resolve(false);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import { Publisher } from './publish';
|
||||
import { LocalPublish } from './local';
|
||||
import { GoogleGCSPublish } from './googleStorage';
|
||||
import { AwsS3Publish } from './awsS3';
|
||||
import { AzureBlobStoragePublish } from './azureBlobStorage';
|
||||
|
||||
const logger = getVoidLogger();
|
||||
const discovery: jest.Mocked<PluginEndpointDiscovery> = {
|
||||
@@ -30,6 +31,10 @@ const discovery: jest.Mocked<PluginEndpointDiscovery> = {
|
||||
};
|
||||
|
||||
describe('Publisher', () => {
|
||||
beforeEach(() => {
|
||||
jest.resetModules(); // clear the cache
|
||||
});
|
||||
|
||||
it('should create local publisher by default', async () => {
|
||||
const mockConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
@@ -105,4 +110,55 @@ describe('Publisher', () => {
|
||||
});
|
||||
expect(publisher).toBeInstanceOf(AwsS3Publish);
|
||||
});
|
||||
|
||||
it('should create Azure Blob Storage publisher from config', async () => {
|
||||
const mockConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
requestUrl: 'http://localhost:7000',
|
||||
publisher: {
|
||||
type: 'azureBlobStorage',
|
||||
azureBlobStorage: {
|
||||
credentials: {
|
||||
accountName: 'accountName',
|
||||
accountKey: 'accountKey',
|
||||
},
|
||||
containerName: 'containerName',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const publisher = await Publisher.fromConfig(mockConfig, {
|
||||
logger,
|
||||
discovery,
|
||||
});
|
||||
expect(publisher).toBeInstanceOf(AzureBlobStoragePublish);
|
||||
});
|
||||
|
||||
it('should create Azure Blob Storage publisher from environment variables', async () => {
|
||||
process.env.AZURE_TENANT_ID = 'AZURE_TENANT_ID';
|
||||
process.env.AZURE_CLIENT_ID = 'AZURE_CLIENT_ID';
|
||||
process.env.AZURE_CLIENT_SECRET = 'AZURE_CLIENT_SECRET';
|
||||
|
||||
const mockConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
requestUrl: 'http://localhost:7000',
|
||||
publisher: {
|
||||
type: 'azureBlobStorage',
|
||||
azureBlobStorage: {
|
||||
credentials: {
|
||||
accountName: 'accountName',
|
||||
},
|
||||
containerName: 'containerName',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const publisher = await Publisher.fromConfig(mockConfig, {
|
||||
logger,
|
||||
discovery,
|
||||
});
|
||||
expect(publisher).toBeInstanceOf(AzureBlobStoragePublish);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -21,6 +21,7 @@ import { PublisherType, PublisherBase } from './types';
|
||||
import { LocalPublish } from './local';
|
||||
import { GoogleGCSPublish } from './googleStorage';
|
||||
import { AwsS3Publish } from './awsS3';
|
||||
import { AzureBlobStoragePublish } from './azureBlobStorage';
|
||||
|
||||
type factoryOptions = {
|
||||
logger: Logger;
|
||||
@@ -47,6 +48,11 @@ export class Publisher {
|
||||
case 'awsS3':
|
||||
logger.info('Creating AWS S3 Bucket publisher for TechDocs');
|
||||
return AwsS3Publish.fromConfig(config, logger);
|
||||
case 'azureBlobStorage':
|
||||
logger.info(
|
||||
'Creating Azure Blob Storage Container publisher for TechDocs',
|
||||
);
|
||||
return AzureBlobStoragePublish.fromConfig(config, logger);
|
||||
case 'local':
|
||||
logger.info('Creating Local publisher for TechDocs');
|
||||
return new LocalPublish(config, logger, discovery);
|
||||
|
||||
@@ -19,7 +19,11 @@ import express from 'express';
|
||||
/**
|
||||
* Key for all the different types of TechDocs publishers that are supported.
|
||||
*/
|
||||
export type PublisherType = 'local' | 'googleGcs' | 'awsS3';
|
||||
export type PublisherType =
|
||||
| 'local'
|
||||
| 'googleGcs'
|
||||
| 'awsS3'
|
||||
| 'azureBlobStorage';
|
||||
|
||||
export type PublishRequest = {
|
||||
entity: Entity;
|
||||
|
||||
@@ -148,6 +148,7 @@ export async function createRouter({
|
||||
}
|
||||
break;
|
||||
case 'awsS3':
|
||||
case 'azureBlobStorage':
|
||||
case 'googleGcs':
|
||||
// This block should be valid for all external storage implementations. So no need to duplicate in future,
|
||||
// add the publisher type in the list here.
|
||||
|
||||
Vendored
+41
@@ -114,6 +114,47 @@ export interface Config {
|
||||
region?: string;
|
||||
};
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* attr: 'type' - accepts a string value
|
||||
* e.g. type: 'azureBlobStorage'
|
||||
* alternatives: 'azureBlobStorage' etc.
|
||||
* @see http://backstage.io/docs/features/techdocs/configuration
|
||||
*/
|
||||
type: 'azureBlobStorage';
|
||||
|
||||
/**
|
||||
* azureBlobStorage required when 'type' is set to azureBlobStorage
|
||||
*/
|
||||
azureBlobStorage?: {
|
||||
/**
|
||||
* (Required) Credentials used to access a storage container.
|
||||
* @visibility secret
|
||||
*/
|
||||
credentials: {
|
||||
/**
|
||||
* Account access name
|
||||
* attr: 'account' - accepts a string value
|
||||
* @visibility secret
|
||||
*/
|
||||
accountName: string;
|
||||
/**
|
||||
* (Optional) Account secret primary key
|
||||
* If not set, environment variables will be used to authenticate.
|
||||
* https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json
|
||||
* attr: 'accountKey' - accepts a string value
|
||||
* @visibility secret
|
||||
*/
|
||||
accountKey?: string;
|
||||
};
|
||||
/**
|
||||
* (Required) Cloud Storage Container Name
|
||||
* attr: 'containerName' - accepts a string value
|
||||
* @visibility backend
|
||||
*/
|
||||
containerName: string;
|
||||
};
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* attr: 'type' - accepts a string value
|
||||
|
||||
Reference in New Issue
Block a user