Create a new method to check the configuration of a techdocs publisher to not crash the application on errors
Signed-off-by: Dominik Henneke <dominik.henneke@sda-se.com>
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
---
|
||||
'@backstage/techdocs-common': patch
|
||||
---
|
||||
|
||||
Move the sanity checks of the publisher configurations to a dedicated `PublisherBase#validateConfiguration()` method instead of throwing an error when doing `Publisher.fromConfig(...)`.
|
||||
If you want to preserve this check in your application, use the following code:
|
||||
|
||||
```ts
|
||||
const publisher = await Publisher.fromConfig(config, {
|
||||
logger,
|
||||
discovery,
|
||||
});
|
||||
|
||||
const validation = await publisher.validateConfiguration();
|
||||
if (!validation.isValid) {
|
||||
throw new Error('Invalid TechDocs publisher configuration');
|
||||
}
|
||||
```
|
||||
@@ -81,10 +81,12 @@ class Bucket {
|
||||
this.bucketName = bucketName;
|
||||
}
|
||||
|
||||
getMetadata() {
|
||||
return new Promise(resolve => {
|
||||
resolve('');
|
||||
});
|
||||
async getMetadata() {
|
||||
if (this.bucketName === 'errorBucket') {
|
||||
throw Error('Bucket does not exist');
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
upload(source: string, { destination }) {
|
||||
|
||||
@@ -80,10 +80,16 @@ export class S3 {
|
||||
};
|
||||
}
|
||||
|
||||
headBucket() {
|
||||
return new Promise(resolve => {
|
||||
resolve('');
|
||||
});
|
||||
headBucket({ Bucket }) {
|
||||
return {
|
||||
promise: async () => {
|
||||
if (Bucket === 'errorBucket') {
|
||||
throw new Error('Bucket does not exist');
|
||||
}
|
||||
|
||||
return {};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
upload({ Key }: { Key: string }) {
|
||||
|
||||
@@ -13,10 +13,11 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { EventEmitter } from 'events';
|
||||
import fs from 'fs-extra';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { EventEmitter } from 'events';
|
||||
import { ClientError } from 'pkgcloud';
|
||||
|
||||
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
|
||||
|
||||
@@ -37,12 +38,11 @@ class PkgCloudStorageClient {
|
||||
getFile(
|
||||
containerName: string,
|
||||
file: string,
|
||||
callback: (err: any, file: string) => any,
|
||||
callback: (err: any, file: any) => any,
|
||||
) {
|
||||
checkFileExists(file).then(res => {
|
||||
if (!res) {
|
||||
callback('File does not exist', file);
|
||||
throw new Error('File does not exist');
|
||||
callback('File does not exist', undefined);
|
||||
} else {
|
||||
callback(undefined, 'success');
|
||||
}
|
||||
@@ -51,13 +51,12 @@ class PkgCloudStorageClient {
|
||||
|
||||
getContainer(
|
||||
containerName: string,
|
||||
callback: (err: string, container: string) => any,
|
||||
callback: (err: ClientError, container: any) => any,
|
||||
) {
|
||||
if (containerName !== 'mock') {
|
||||
callback('Container does not exist', containerName);
|
||||
throw new Error('Container does not exist');
|
||||
callback(new Error('Container does not exist'), undefined);
|
||||
} else {
|
||||
callback('Container does not exist', 'success');
|
||||
callback(undefined, 'success');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,16 +13,16 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import {
|
||||
Entity,
|
||||
EntityName,
|
||||
ENTITY_DEFAULT_NAMESPACE,
|
||||
EntityName,
|
||||
} from '@backstage/catalog-model';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import mockFs from 'mock-fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import * as winston from 'winston';
|
||||
import { AwsS3Publish } from './awsS3';
|
||||
import { PublisherBase, TechDocsMetadata } from './types';
|
||||
|
||||
@@ -59,9 +59,7 @@ const getEntityRootDir = (entity: Entity) => {
|
||||
return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
|
||||
};
|
||||
|
||||
const logger = winston.createLogger();
|
||||
jest.spyOn(logger, 'info').mockReturnValue(logger);
|
||||
jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
const logger = getVoidLogger();
|
||||
|
||||
let publisher: PublisherBase;
|
||||
|
||||
@@ -87,6 +85,39 @@ beforeEach(() => {
|
||||
});
|
||||
|
||||
describe('AwsS3Publish', () => {
|
||||
describe('validateConfiguration', () => {
|
||||
it('should validate correct config', async () => {
|
||||
expect(await publisher.validateConfiguration()).toEqual({
|
||||
isValid: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('should reject incorrect config', async () => {
|
||||
const mockConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
requestUrl: 'http://localhost:7000',
|
||||
publisher: {
|
||||
type: 'awsS3',
|
||||
awsS3: {
|
||||
credentials: {
|
||||
accessKeyId: 'accessKeyId',
|
||||
secretAccessKey: 'secretAccessKey',
|
||||
},
|
||||
// this bucket name will throw an error
|
||||
bucketName: 'errorBucket',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const errorPublisher = AwsS3Publish.fromConfig(mockConfig, logger);
|
||||
|
||||
expect(await errorPublisher.validateConfiguration()).toEqual({
|
||||
isValid: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('publish', () => {
|
||||
beforeEach(() => {
|
||||
const entity = createMockEntity();
|
||||
|
||||
@@ -17,16 +17,21 @@ import { Entity, EntityName } from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import aws, { Credentials } from 'aws-sdk';
|
||||
import { ManagedUpload } from 'aws-sdk/clients/s3';
|
||||
import { CredentialsOptions } from 'aws-sdk/lib/credentials';
|
||||
import express from 'express';
|
||||
import fs from 'fs-extra';
|
||||
import JSON5 from 'json5';
|
||||
import createLimiter from 'p-limit';
|
||||
import { CredentialsOptions } from 'aws-sdk/lib/credentials';
|
||||
import path from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import { Logger } from 'winston';
|
||||
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
|
||||
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
|
||||
import {
|
||||
ConfigurationValidationResponse,
|
||||
PublisherBase,
|
||||
PublishRequest,
|
||||
TechDocsMetadata,
|
||||
} from './types';
|
||||
|
||||
const streamToBuffer = (stream: Readable): Promise<Buffer> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -81,30 +86,6 @@ export class AwsS3Publish implements PublisherBase {
|
||||
...(endpoint && { endpoint }),
|
||||
});
|
||||
|
||||
// Check if the defined bucket exists. Being able to connect means the configuration is good
|
||||
// and the storage client will work.
|
||||
storageClient.headBucket(
|
||||
{
|
||||
Bucket: bucketName,
|
||||
},
|
||||
err => {
|
||||
if (err) {
|
||||
logger.error(
|
||||
`Could not retrieve metadata about the AWS S3 bucket ${bucketName}. ` +
|
||||
'Make sure the bucket exists. Also make sure that authentication is setup either by ' +
|
||||
'explicitly defining credentials and region in techdocs.publisher.awsS3 in app config or ' +
|
||||
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
logger.error(`from AWS client library: ${err.message}`);
|
||||
throw new Error();
|
||||
} else {
|
||||
logger.info(
|
||||
`Successfully connected to the AWS S3 bucket ${bucketName}.`,
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
return new AwsS3Publish(storageClient, bucketName, logger);
|
||||
}
|
||||
|
||||
@@ -149,6 +130,35 @@ export class AwsS3Publish implements PublisherBase {
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the defined bucket exists. Being able to connect means the configuration is good
|
||||
* and the storage client will work.
|
||||
*/
|
||||
async validateConfiguration(): Promise<ConfigurationValidationResponse> {
|
||||
try {
|
||||
await this.storageClient
|
||||
.headBucket({ Bucket: this.bucketName })
|
||||
.promise();
|
||||
|
||||
this.logger.info(
|
||||
`Successfully connected to the AWS S3 bucket ${this.bucketName}.`,
|
||||
);
|
||||
|
||||
return { isValid: true };
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Could not retrieve metadata about the AWS S3 bucket ${this.bucketName}. ` +
|
||||
'Make sure the bucket exists. Also make sure that authentication is setup either by ' +
|
||||
'explicitly defining credentials and region in techdocs.publisher.awsS3 in app config or ' +
|
||||
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
this.logger.error(`from AWS client library`, error);
|
||||
return {
|
||||
isValid: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload all the files from the generated `directory` to the S3 bucket.
|
||||
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import {
|
||||
Entity,
|
||||
EntityName,
|
||||
ENTITY_DEFAULT_NAMESPACE,
|
||||
EntityName,
|
||||
} from '@backstage/catalog-model';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import mockFs from 'mock-fs';
|
||||
@@ -59,12 +59,9 @@ const getEntityRootDir = (entity: Entity) => {
|
||||
return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
|
||||
};
|
||||
|
||||
function createLogger() {
|
||||
const logger = getVoidLogger();
|
||||
jest.spyOn(logger, 'info').mockReturnValue(logger);
|
||||
jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
return logger;
|
||||
}
|
||||
const logger = getVoidLogger();
|
||||
jest.spyOn(logger, 'info').mockReturnValue(logger);
|
||||
jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
|
||||
let publisher: PublisherBase;
|
||||
beforeEach(async () => {
|
||||
@@ -85,13 +82,51 @@ beforeEach(async () => {
|
||||
},
|
||||
});
|
||||
|
||||
publisher = await AzureBlobStoragePublish.fromConfig(
|
||||
mockConfig,
|
||||
createLogger(),
|
||||
);
|
||||
publisher = AzureBlobStoragePublish.fromConfig(mockConfig, logger);
|
||||
});
|
||||
|
||||
describe('publishing with valid credentials', () => {
|
||||
describe('validateConfiguration', () => {
|
||||
it('should validate correct config', async () => {
|
||||
expect(await publisher.validateConfiguration()).toEqual({
|
||||
isValid: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('should reject incorrect config', async () => {
|
||||
const mockConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
requestUrl: 'http://localhost:7000',
|
||||
publisher: {
|
||||
type: 'azureBlobStorage',
|
||||
azureBlobStorage: {
|
||||
credentials: {
|
||||
accountName: 'accountName',
|
||||
accountKey: 'accountKey',
|
||||
},
|
||||
containerName: 'bad_container',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const errorPublisher = await AzureBlobStoragePublish.fromConfig(
|
||||
mockConfig,
|
||||
logger,
|
||||
);
|
||||
|
||||
expect(await errorPublisher.validateConfiguration()).toEqual({
|
||||
isValid: false,
|
||||
});
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
`Could not retrieve metadata about the Azure Blob Storage container bad_container.`,
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('publish', () => {
|
||||
beforeEach(() => {
|
||||
const entity = createMockEntity();
|
||||
@@ -151,6 +186,60 @@ describe('publishing with valid credentials', () => {
|
||||
});
|
||||
mockFs.restore();
|
||||
});
|
||||
|
||||
it('reports an error when bad account credentials', async () => {
|
||||
const mockConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
requestUrl: 'http://localhost:7000',
|
||||
publisher: {
|
||||
type: 'azureBlobStorage',
|
||||
azureBlobStorage: {
|
||||
credentials: {
|
||||
accountName: 'failupload',
|
||||
accountKey: 'accountKey',
|
||||
},
|
||||
containerName: 'containerName',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
|
||||
|
||||
const entity = createMockEntity();
|
||||
const entityRootDir = getEntityRootDir(entity);
|
||||
|
||||
mockFs({
|
||||
[entityRootDir]: {
|
||||
'index.html': '',
|
||||
},
|
||||
});
|
||||
|
||||
let error;
|
||||
try {
|
||||
await publisher.publish({
|
||||
entity,
|
||||
directory: entityRootDir,
|
||||
});
|
||||
} catch (e) {
|
||||
error = e;
|
||||
}
|
||||
|
||||
expect(error.message).toContain(
|
||||
`Unable to upload file(s) to Azure Blob Storage.`,
|
||||
);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
`Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join(
|
||||
entityRootDir,
|
||||
'index.html',
|
||||
)} with status code 500`,
|
||||
),
|
||||
);
|
||||
|
||||
mockFs.restore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('hasDocsBeenGenerated', () => {
|
||||
@@ -243,156 +332,3 @@ describe('publishing with valid credentials', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('error reporting', () => {
|
||||
it('reports an error when unable to read container properties', async () => {
|
||||
const mockConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
requestUrl: 'http://localhost:7000',
|
||||
publisher: {
|
||||
type: 'azureBlobStorage',
|
||||
azureBlobStorage: {
|
||||
credentials: {
|
||||
accountName: 'accountName',
|
||||
},
|
||||
containerName: 'bad_container',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const logger = createLogger();
|
||||
|
||||
let error;
|
||||
try {
|
||||
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
|
||||
} catch (e) {
|
||||
error = e;
|
||||
}
|
||||
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
`Could not retrieve metadata about the Azure Blob Storage container bad_container.`,
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('reports an error when bad account credentials', async () => {
|
||||
const mockConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
requestUrl: 'http://localhost:7000',
|
||||
publisher: {
|
||||
type: 'azureBlobStorage',
|
||||
azureBlobStorage: {
|
||||
credentials: {
|
||||
accountName: 'failupload',
|
||||
accountKey: 'accountKey',
|
||||
},
|
||||
containerName: 'containerName',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const logger = createLogger();
|
||||
|
||||
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
|
||||
|
||||
const entity = createMockEntity();
|
||||
const entityRootDir = getEntityRootDir(entity);
|
||||
|
||||
mockFs({
|
||||
[entityRootDir]: {
|
||||
'index.html': '',
|
||||
},
|
||||
});
|
||||
|
||||
let error;
|
||||
try {
|
||||
await publisher.publish({
|
||||
entity,
|
||||
directory: entityRootDir,
|
||||
});
|
||||
} catch (e) {
|
||||
error = e;
|
||||
}
|
||||
|
||||
expect(error.message).toContain(
|
||||
`Unable to upload file(s) to Azure Blob Storage.`,
|
||||
);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
`Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join(
|
||||
entityRootDir,
|
||||
'index.html',
|
||||
)} with status code 500`,
|
||||
),
|
||||
);
|
||||
|
||||
mockFs.restore();
|
||||
});
|
||||
|
||||
describe('fetchTechDocsMetadata', () => {
|
||||
it('should return tech docs metadata', async () => {
|
||||
const entityNameMock = createMockEntityName();
|
||||
const entity = createMockEntity();
|
||||
const entityRootDir = getEntityRootDir(entity);
|
||||
|
||||
mockFs({
|
||||
[entityRootDir]: {
|
||||
'techdocs_metadata.json':
|
||||
'{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}',
|
||||
},
|
||||
});
|
||||
const expectedMetadata: TechDocsMetadata = {
|
||||
site_name: 'backstage',
|
||||
site_description: 'site_content',
|
||||
etag: 'etag',
|
||||
};
|
||||
expect(
|
||||
await publisher.fetchTechDocsMetadata(entityNameMock),
|
||||
).toStrictEqual(expectedMetadata);
|
||||
mockFs.restore();
|
||||
});
|
||||
|
||||
it('should return tech docs metadata when json encoded with single quotes', async () => {
|
||||
const entityNameMock = createMockEntityName();
|
||||
const entity = createMockEntity();
|
||||
const entityRootDir = getEntityRootDir(entity);
|
||||
|
||||
mockFs({
|
||||
[entityRootDir]: {
|
||||
'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`,
|
||||
},
|
||||
});
|
||||
|
||||
const expectedMetadata: TechDocsMetadata = {
|
||||
site_name: 'backstage',
|
||||
site_description: 'site_content',
|
||||
etag: 'etag',
|
||||
};
|
||||
expect(
|
||||
await publisher.fetchTechDocsMetadata(entityNameMock),
|
||||
).toStrictEqual(expectedMetadata);
|
||||
mockFs.restore();
|
||||
});
|
||||
|
||||
it('should return an error if the techdocs_metadata.json file is not present', async () => {
|
||||
const entityNameMock = createMockEntityName();
|
||||
|
||||
let error;
|
||||
try {
|
||||
await publisher.fetchTechDocsMetadata(entityNameMock);
|
||||
} catch (e) {
|
||||
error = e;
|
||||
}
|
||||
|
||||
expect(error.message).toEqual(
|
||||
expect.stringContaining('TechDocs metadata fetch'),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -26,16 +26,18 @@ import limiterFactory from 'p-limit';
|
||||
import { default as path, default as platformPath } from 'path';
|
||||
import { Logger } from 'winston';
|
||||
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
|
||||
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
|
||||
import {
|
||||
ConfigurationValidationResponse,
|
||||
PublisherBase,
|
||||
PublishRequest,
|
||||
TechDocsMetadata,
|
||||
} from './types';
|
||||
|
||||
// The number of batches that may be ongoing at the same time.
|
||||
const BATCH_CONCURRENCY = 3;
|
||||
|
||||
export class AzureBlobStoragePublish implements PublisherBase {
|
||||
static async fromConfig(
|
||||
config: Config,
|
||||
logger: Logger,
|
||||
): Promise<PublisherBase> {
|
||||
static fromConfig(config: Config, logger: Logger): PublisherBase {
|
||||
let containerName = '';
|
||||
try {
|
||||
containerName = config.getString(
|
||||
@@ -78,26 +80,6 @@ export class AzureBlobStoragePublish implements PublisherBase {
|
||||
credential,
|
||||
);
|
||||
|
||||
try {
|
||||
const response = await storageClient
|
||||
.getContainerClient(containerName)
|
||||
.getProperties();
|
||||
|
||||
if (response._response.status >= 400) {
|
||||
throw new Error(
|
||||
`Failed to retrieve metadata from ${response._response.request.url} with status code ${response._response.status}.`,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
logger.error(
|
||||
`Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` +
|
||||
'Make sure that the Azure project and container exist and the access key is setup correctly ' +
|
||||
'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' +
|
||||
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
throw new Error(`from Azure Blob Storage client library: ${e.message}`);
|
||||
}
|
||||
|
||||
return new AzureBlobStoragePublish(storageClient, containerName, logger);
|
||||
}
|
||||
|
||||
@@ -111,6 +93,37 @@ export class AzureBlobStoragePublish implements PublisherBase {
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
async validateConfiguration(): Promise<ConfigurationValidationResponse> {
|
||||
try {
|
||||
const response = await this.storageClient
|
||||
.getContainerClient(this.containerName)
|
||||
.getProperties();
|
||||
|
||||
if (response._response.status === 200) {
|
||||
return {
|
||||
isValid: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (response._response.status >= 400) {
|
||||
this.logger.error(
|
||||
`Failed to retrieve metadata from ${response._response.request.url} with status code ${response._response.status}.`,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
this.logger.error(`from Azure Blob Storage client library: ${e.message}`);
|
||||
}
|
||||
|
||||
this.logger.error(
|
||||
`Could not retrieve metadata about the Azure Blob Storage container ${this.containerName}. ` +
|
||||
'Make sure that the Azure project and container exist and the access key is setup correctly ' +
|
||||
'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' +
|
||||
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
|
||||
return { isValid: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload all the files from the generated `directory` to the Azure Blob Storage container.
|
||||
* Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import {
|
||||
Entity,
|
||||
EntityName,
|
||||
ENTITY_DEFAULT_NAMESPACE,
|
||||
EntityName,
|
||||
} from '@backstage/catalog-model';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import mockFs from 'mock-fs';
|
||||
@@ -83,6 +83,35 @@ beforeEach(async () => {
|
||||
});
|
||||
|
||||
describe('GoogleGCSPublish', () => {
|
||||
describe('validateConfiguration', () => {
|
||||
it('should validate correct config', async () => {
|
||||
expect(await publisher.validateConfiguration()).toEqual({
|
||||
isValid: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('should reject incorrect config', async () => {
|
||||
const mockConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
requestUrl: 'http://localhost:7000',
|
||||
publisher: {
|
||||
type: 'googleGcs',
|
||||
googleGcs: {
|
||||
credentials: '{}',
|
||||
bucketName: 'errorBucket',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const errorPublisher = GoogleGCSPublish.fromConfig(mockConfig, logger);
|
||||
|
||||
expect(await errorPublisher.validateConfiguration()).toEqual({
|
||||
isValid: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('publish', () => {
|
||||
beforeEach(() => {
|
||||
const entity = createMockEntity();
|
||||
|
||||
@@ -26,13 +26,15 @@ import createLimiter from 'p-limit';
|
||||
import path from 'path';
|
||||
import { Logger } from 'winston';
|
||||
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
|
||||
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
|
||||
import {
|
||||
ConfigurationValidationResponse,
|
||||
PublisherBase,
|
||||
PublishRequest,
|
||||
TechDocsMetadata,
|
||||
} from './types';
|
||||
|
||||
export class GoogleGCSPublish implements PublisherBase {
|
||||
static async fromConfig(
|
||||
config: Config,
|
||||
logger: Logger,
|
||||
): Promise<PublisherBase> {
|
||||
static fromConfig(config: Config, logger: Logger): PublisherBase {
|
||||
let bucketName = '';
|
||||
try {
|
||||
bucketName = config.getString('techdocs.publisher.googleGcs.bucketName');
|
||||
@@ -65,21 +67,6 @@ export class GoogleGCSPublish implements PublisherBase {
|
||||
}),
|
||||
});
|
||||
|
||||
// Check if the defined bucket exists. Being able to connect means the configuration is good
|
||||
// and the storage client will work.
|
||||
try {
|
||||
await storageClient.bucket(bucketName).getMetadata();
|
||||
logger.info(`Successfully connected to the GCS bucket ${bucketName}.`);
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`Could not retrieve metadata about the GCS bucket ${bucketName}. ` +
|
||||
'Make sure the bucket exists. Also make sure that authentication is setup either by explicitly defining ' +
|
||||
'techdocs.publisher.googleGcs.credentials in app config or by using environment variables. ' +
|
||||
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
throw new Error(err.message);
|
||||
}
|
||||
|
||||
return new GoogleGCSPublish(storageClient, bucketName, logger);
|
||||
}
|
||||
|
||||
@@ -93,6 +80,33 @@ export class GoogleGCSPublish implements PublisherBase {
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the defined bucket exists. Being able to connect means the configuration is good
|
||||
* and the storage client will work.
|
||||
*/
|
||||
async validateConfiguration(): Promise<ConfigurationValidationResponse> {
|
||||
try {
|
||||
await this.storageClient.bucket(this.bucketName).getMetadata();
|
||||
this.logger.info(
|
||||
`Successfully connected to the GCS bucket ${this.bucketName}.`,
|
||||
);
|
||||
|
||||
return {
|
||||
isValid: true,
|
||||
};
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Could not retrieve metadata about the GCS bucket ${this.bucketName}. ` +
|
||||
'Make sure the bucket exists. Also make sure that authentication is setup either by explicitly defining ' +
|
||||
'techdocs.publisher.googleGcs.credentials in app config or by using environment variables. ' +
|
||||
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
this.logger.error(`from GCS client library: ${err.message}`);
|
||||
|
||||
return { isValid: false };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload all the files from the generated `directory` to the GCS bucket.
|
||||
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
|
||||
|
||||
@@ -25,6 +25,7 @@ import os from 'os';
|
||||
import path from 'path';
|
||||
import { Logger } from 'winston';
|
||||
import {
|
||||
ConfigurationValidationResponse,
|
||||
PublisherBase,
|
||||
PublishRequest,
|
||||
PublishResponse,
|
||||
@@ -65,6 +66,12 @@ export class LocalPublish implements PublisherBase {
|
||||
this.discovery = discovery;
|
||||
}
|
||||
|
||||
async validateConfiguration(): Promise<ConfigurationValidationResponse> {
|
||||
return {
|
||||
isValid: true,
|
||||
};
|
||||
}
|
||||
|
||||
publish({ entity, directory }: PublishRequest): Promise<PublishResponse> {
|
||||
const entityNamespace = entity.metadata.namespace ?? 'default';
|
||||
|
||||
|
||||
@@ -13,16 +13,16 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import {
|
||||
Entity,
|
||||
EntityName,
|
||||
ENTITY_DEFAULT_NAMESPACE,
|
||||
EntityName,
|
||||
} from '@backstage/catalog-model';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import mockFs from 'mock-fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import * as winston from 'winston';
|
||||
import { OpenStackSwiftPublish } from './openStackSwift';
|
||||
import { PublisherBase, TechDocsMetadata } from './types';
|
||||
|
||||
@@ -59,9 +59,7 @@ const getEntityRootDir = (entity: Entity) => {
|
||||
return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
|
||||
};
|
||||
|
||||
const logger = winston.createLogger();
|
||||
jest.spyOn(logger, 'info').mockReturnValue(logger);
|
||||
jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
const logger = getVoidLogger();
|
||||
|
||||
let publisher: PublisherBase;
|
||||
|
||||
@@ -89,6 +87,43 @@ beforeEach(() => {
|
||||
});
|
||||
|
||||
describe('OpenStackSwiftPublish', () => {
|
||||
describe('validateConfiguration', () => {
|
||||
it('should validate correct config', async () => {
|
||||
expect(await publisher.validateConfiguration()).toEqual({
|
||||
isValid: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('should reject incorrect config', async () => {
|
||||
const mockConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
requestUrl: 'http://localhost:7000',
|
||||
publisher: {
|
||||
type: 'openStackSwift',
|
||||
openStackSwift: {
|
||||
credentials: {
|
||||
username: 'mockuser',
|
||||
password: 'verystrongpass',
|
||||
},
|
||||
authUrl: 'mockauthurl',
|
||||
region: 'mockregion',
|
||||
containerName: 'errorBucket',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const errorPublisher = OpenStackSwiftPublish.fromConfig(
|
||||
mockConfig,
|
||||
logger,
|
||||
);
|
||||
|
||||
expect(await errorPublisher.validateConfiguration()).toEqual({
|
||||
isValid: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('publish', () => {
|
||||
beforeEach(() => {
|
||||
const entity = createMockEntity();
|
||||
|
||||
@@ -15,16 +15,21 @@
|
||||
*/
|
||||
import { Entity, EntityName } from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import { storage } from 'pkgcloud';
|
||||
import express from 'express';
|
||||
import fs from 'fs-extra';
|
||||
import JSON5 from 'json5';
|
||||
import createLimiter from 'p-limit';
|
||||
import path from 'path';
|
||||
import { storage } from 'pkgcloud';
|
||||
import { Readable } from 'stream';
|
||||
import { Logger } from 'winston';
|
||||
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
|
||||
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
|
||||
import {
|
||||
ConfigurationValidationResponse,
|
||||
PublisherBase,
|
||||
PublishRequest,
|
||||
TechDocsMetadata,
|
||||
} from './types';
|
||||
|
||||
const streamToBuffer = (stream: Readable): Promise<Buffer> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -70,25 +75,6 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
region: openStackSwiftConfig.getString('region'),
|
||||
});
|
||||
|
||||
// Check if the defined container exists. Being able to connect means the configuration is good
|
||||
// and the storage client will work.
|
||||
storageClient.getContainer(containerName, (err, container) => {
|
||||
if (container) {
|
||||
logger.info(
|
||||
`Successfully connected to the OpenStack Swift container ${containerName}.`,
|
||||
);
|
||||
} else {
|
||||
logger.error(
|
||||
`Could not retrieve metadata about the OpenStack Swift container ${containerName}. ` +
|
||||
'Make sure the container exists. Also make sure that authentication is setup either by ' +
|
||||
'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
|
||||
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
|
||||
logger.error(`from OpenStack client library: ${err.message}`);
|
||||
}
|
||||
});
|
||||
|
||||
return new OpenStackSwiftPublish(storageClient, containerName, logger);
|
||||
}
|
||||
|
||||
@@ -102,6 +88,37 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check if the defined container exists. Being able to connect means the configuration is good
|
||||
* and the storage client will work.
|
||||
*/
|
||||
validateConfiguration(): Promise<ConfigurationValidationResponse> {
|
||||
return new Promise(resolve => {
|
||||
this.storageClient.getContainer(this.containerName, (err, container) => {
|
||||
if (container) {
|
||||
this.logger.info(
|
||||
`Successfully connected to the OpenStack Swift container ${this.containerName}.`,
|
||||
);
|
||||
resolve({
|
||||
isValid: true,
|
||||
});
|
||||
} else {
|
||||
this.logger.error(
|
||||
`Could not retrieve metadata about the OpenStack Swift container ${this.containerName}. ` +
|
||||
'Make sure the container exists. Also make sure that authentication is setup either by ' +
|
||||
'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
|
||||
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
|
||||
this.logger.error(`from OpenStack client library: ${err.message}`);
|
||||
resolve({
|
||||
isValid: false,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload all the files from the generated `directory` to the OpenStack Swift container.
|
||||
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
|
||||
|
||||
@@ -13,16 +13,16 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Logger } from 'winston';
|
||||
import { Config } from '@backstage/config';
|
||||
import { PluginEndpointDiscovery } from '@backstage/backend-common';
|
||||
|
||||
import { PublisherType, PublisherBase } from './types';
|
||||
import { LocalPublish } from './local';
|
||||
import { GoogleGCSPublish } from './googleStorage';
|
||||
import { PluginEndpointDiscovery } from '@backstage/backend-common';
|
||||
import { Config } from '@backstage/config';
|
||||
import { Logger } from 'winston';
|
||||
import { AwsS3Publish } from './awsS3';
|
||||
import { AzureBlobStoragePublish } from './azureBlobStorage';
|
||||
import { GoogleGCSPublish } from './googleStorage';
|
||||
import { LocalPublish } from './local';
|
||||
import { OpenStackSwiftPublish } from './openStackSwift';
|
||||
import { PublisherBase, PublisherType } from './types';
|
||||
|
||||
type factoryOptions = {
|
||||
logger: Logger;
|
||||
@@ -45,7 +45,7 @@ export class Publisher {
|
||||
switch (publisherType) {
|
||||
case 'googleGcs':
|
||||
logger.info('Creating Google Storage Bucket publisher for TechDocs');
|
||||
return await GoogleGCSPublish.fromConfig(config, logger);
|
||||
return GoogleGCSPublish.fromConfig(config, logger);
|
||||
case 'awsS3':
|
||||
logger.info('Creating AWS S3 Bucket publisher for TechDocs');
|
||||
return AwsS3Publish.fromConfig(config, logger);
|
||||
|
||||
@@ -37,6 +37,14 @@ export type PublishResponse = {
|
||||
remoteUrl?: string;
|
||||
} | void;
|
||||
|
||||
/**
|
||||
* Result for the validation check.
|
||||
*/
|
||||
export type ConfigurationValidationResponse = {
|
||||
/** Tells whether the configuration is valid. */
|
||||
isValid: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Type to hold metadata found in techdocs_metadata.json and associated with each site
|
||||
* @param etag ETag of the resource used to generate the site. Usually the latest commit sha of the source repository.
|
||||
@@ -53,6 +61,14 @@ export type TechDocsMetadata = {
|
||||
* It also provides APIs to communicate with the storage service.
|
||||
*/
|
||||
export interface PublisherBase {
|
||||
/**
|
||||
* Check if the configuration is valid. This check tries to perform certain checks to see if the
|
||||
* publisher is configured correctly and can be used to publish or read documentations.
|
||||
* The different implementations might e.g. use the provided service credentials to access the
|
||||
* target or check if a folder/bucket is available.
|
||||
*/
|
||||
validateConfiguration(): Promise<ConfigurationValidationResponse>;
|
||||
|
||||
/**
|
||||
* Store the generated static files onto a storage service (either local filesystem or external service).
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user