paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 31 Packages Projects Releases Wiki Activity

refactor(techdocs-common): add try/catch statements + fs-extra

Browse Source
This commit is contained in:
Remi
2021-01-03 03:40:21 +01:00
parent 3aa98a6753
commit 1517009d8a
3 changed files with 87 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/techdocs-common/__mocks__/aws-sdk.ts
+1 -4
View File
@@ -71,10 +71,7 @@ export class S3 {
return {
promise: () =>
new Promise((resolve, reject) => {
if (
fs.existsSync(Key) &&
fs.readFileSync(Key).toString() === 'mock-error'
) {
if (!fs.existsSync(Key)) {
reject('');
} else {
resolve('');
packages/techdocs-common/src/stages/publish/awsS3.test.ts
+28 -10
View File
@@ -25,6 +25,7 @@ const createMockEntity = (annotations = {}) => {
kind: 'TestKind',
metadata: {
name: 'test-component-name',
namespace: 'test-namespace',
annotations: {
...annotations,
},
@@ -61,8 +62,13 @@ beforeEach(() => {
describe('AwsS3Publish', () => {
describe('publish', () => {
it('should publish a directory', async () => {
const entity = createMockEntity();
const {
kind,
metadata: { namespace, name },
} = entity;
mockFs({
'/path/to/generatedDirectory': {
[`${namespace}/${kind}/${name}`]: {
'index.html': '',
'404.html': '',
assets: {
@@ -71,20 +77,27 @@ describe('AwsS3Publish', () => {
},
});
const entity = createMockEntity();
expect(
await publisher.publish({
entity,
directory: '/path/to/generatedDirectory',
directory: `${namespace}/${kind}/${name}`,
}),
).toBeUndefined();
mockFs.restore();
});
it('should fail to publish a directory', async () => {
const wrongPathToGeneratedDirectory = '/wrong/path/to/generatedDirectory';
const entity = createMockEntity();
const {
kind,
metadata: { namespace, name },
} = entity;
mockFs({
'/path/to/generatedDirectory': {
'index.html': 'mock-error',
[`${namespace}/${kind}/${name}`]: {
'index.html': '',
'404.html': '',
assets: {
'main.css': '',
@@ -92,14 +105,17 @@ describe('AwsS3Publish', () => {
},
});
const entity = createMockEntity();
await publisher
.publish({
entity,
directory: '/path/to/generatedDirectory',
directory: wrongPathToGeneratedDirectory,
})
.catch(error =>
expect(error).toBe(`Unable to upload file(s) to AWS S3. Error`),
expect(error).toEqual(
new Error(
`Unable to upload file(s) to AWS S3. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
),
),
);
mockFs.restore();
});
@@ -170,8 +186,10 @@ describe('AwsS3Publish', () => {
await publisher
.fetchTechDocsMetadata(entityNameMock)
.catch(error =>
expect(error).toBe(
`The file ${entityRootDir}/techdocs_metadata.json doest not exist !`,
expect(error).toEqual(
new Error(
`TechDocs metadata fetch failed, The file ${entityRootDir}/techdocs_metadata.json doest not exist !`,
),
),
);
});
packages/techdocs-common/src/stages/publish/awsS3.ts
+58 -64
View File
@@ -22,7 +22,7 @@ import { Config } from '@backstage/config';
import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
import { PublisherBase, PublishRequest } from './types';
import { ManagedUpload } from 'aws-sdk/clients/s3';
import fs from 'fs';
import fs from 'fs-extra';
export class AwsS3Publish implements PublisherBase {
static fromConfig(config: Config, logger: Logger): PublisherBase {
@@ -92,14 +92,15 @@ export class AwsS3Publish implements PublisherBase {
* Upload all the files from the generated `directory` to the S3 bucket.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
*/
publish({ entity, directory }: PublishRequest): Promise<void> {
return new Promise(async (resolve, reject) => {
async publish({ entity, directory }: PublishRequest): Promise<void> {
try {
// Note: S3 manages creation of parent directories if they do not exist.
// So collecting path of only the files is good enough.
const allFilesToUpload = await getFileTreeRecursively(directory);
const uploadPromises: Array<Promise<ManagedUpload.SendData>> = [];
allFilesToUpload.forEach(filePath => {
for (const filePath of allFilesToUpload) {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
@@ -107,61 +108,57 @@ export class AwsS3Publish implements PublisherBase {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const destination = `${entityRootDir}/${relativeFilePath}`; // S3 Bucket file relative path
fs.readFile(filePath, (err, data) => {
if (err) {
reject(err);
}
const fileContent = await fs.readFile(filePath, 'utf8');
const params = {
Bucket: this.bucketName,
Key: destination,
Body: data,
};
const params = {
Bucket: this.bucketName,
Key: destination,
Body: fileContent,
};
// TODO: Upload in chunks of ~10 files instead of all files at once.
uploadPromises.push(this.storageClient.upload(params).promise());
});
});
Promise.all(uploadPromises)
.then(() => {
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
);
resolve(undefined);
})
.catch((err: Error) => {
const errorMessage = `Unable to upload file(s) to AWS S3. Error ${err.message}`;
this.logger.error(errorMessage);
reject(errorMessage);
});
});
uploadPromises.push(this.storageClient.upload(params).promise());
}
await Promise.all(uploadPromises);
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
);
return;
} catch (e) {
const errorMessage = `Unable to upload file(s) to AWS S3. Error ${e.message}`;
this.logger.error(errorMessage);
throw new Error(errorMessage);
}
}
fetchTechDocsMetadata(entityName: EntityName): Promise<string> {
return new Promise((resolve, reject) => {
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
async fetchTechDocsMetadata(entityName: EntityName): Promise<string> {
try {
return await new Promise<string>((resolve, reject) => {
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
const fileStreamChunks: Array<any> = [];
this.storageClient
.getObject({
Bucket: this.bucketName,
Key: `${entityRootDir}/techdocs_metadata.json`,
})
.createReadStream()
.on('error', err => {
this.logger.error(err.message);
reject(err.message);
})
.on('data', chunk => {
fileStreamChunks.push(chunk);
})
.on('end', () => {
const techdocsMetadataJson = Buffer.concat(
fileStreamChunks,
).toString();
resolve(techdocsMetadataJson);
});
});
const fileStreamChunks: Array<any> = [];
this.storageClient
.getObject({
Bucket: this.bucketName,
Key: `${entityRootDir}/techdocs_metadata.json`,
})
.createReadStream()
.on('error', err => {
this.logger.error(err.message);
reject(new Error(err.message));
})
.on('data', chunk => {
fileStreamChunks.push(chunk);
})
.on('end', () => {
const techdocsMetadataJson = Buffer.concat(
fileStreamChunks,
).toString();
resolve(techdocsMetadataJson);
});
});
} catch (e) {
throw new Error(`TechDocs metadata fetch failed, ${e.message}`);
}
}
/**
@@ -207,20 +204,17 @@ export class AwsS3Publish implements PublisherBase {
* can be used to verify if there are any pre-generated docs available to serve.
*/
async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
return new Promise(resolve => {
try {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
this.storageClient
await this.storageClient
.headObject({
Bucket: this.bucketName,
Key: `${entityRootDir}/index.html`,
})
.promise()
.then(() => {
resolve(true);
})
.catch(() => {
resolve(false);
});
});
.promise();
return Promise.resolve(true);
} catch (e) {
return Promise.resolve(false);
}
}
}