paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 46 Packages Projects Releases Wiki Activity

Merge pull request #3953 from ayshiff/feature/techdocs-aws-sdkv3

Browse Source 
TechDocs: AWS S3 SDK version bump
This commit is contained in:
Fredrik Adelöw
2021-01-12 10:21:51 +01:00
committed by GitHub
parent 65d5c8e6fe 5826d0973e
commit 1c7771871e
6 changed files with 730 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/techdocs-common/__mocks__/aws-sdk.ts → packages/techdocs-common/__mocks__/@aws-sdk/client-s3.ts
+23 -48
View File
@@ -13,74 +13,49 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { S3 as S3Types } from 'aws-sdk';
import type { S3ClientConfig } from '@aws-sdk/client-s3';
import { EventEmitter } from 'events';
import fs from 'fs';
export class S3 {
private readonly options;
constructor(options: S3Types.ClientConfiguration) {
constructor(options: S3ClientConfig) {
this.options = options;
}
headObject({ Key }: { Key: string }) {
return {
promise: () => this.checkFileExists(Key),
};
}
getObject({ Key }: { Key: string }) {
return {
promise: () => this.checkFileExists(Key),
createReadStream: () => {
const emitter = new EventEmitter();
process.nextTick(() => {
if (fs.existsSync(Key)) {
emitter.emit('data', Buffer.from(fs.readFileSync(Key)));
} else {
emitter.emit(
'error',
new Error(`The file ${Key} doest not exist !`),
);
}
emitter.emit('end');
});
return emitter;
},
};
}
checkFileExists(Key: string) {
return new Promise((resolve, reject) => {
if (fs.existsSync(Key)) {
resolve('');
} else {
reject({ message: 'The object doest not exist !' });
reject({ message: `The file ${Key} doest not exist.` });
}
});
}
getObject({ Key }: { Key: string }) {
return new Promise((resolve, reject) => {
if (fs.existsSync(Key)) {
const emitter = new EventEmitter();
process.nextTick(() => {
emitter.emit('data', Buffer.from(fs.readFileSync(Key)));
emitter.emit('end');
});
resolve({
Body: emitter,
});
} else {
reject({ message: `The file ${Key} doest not exist.` });
}
});
}
headBucket() {
return new Promise(resolve => {
resolve('');
});
return '';
}
upload({ Key }: { Key: string }) {
return {
promise: () =>
new Promise((resolve, reject) => {
if (!fs.existsSync(Key)) {
reject('');
} else {
resolve('');
}
}),
};
putObject() {
return '';
}
}
export default {
S3,
};
packages/techdocs-common/package.json
+2 -1
View File
@@ -36,6 +36,7 @@
"url": "https://github.com/backstage/backstage/issues"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.1.0",
"@backstage/backend-common": "^0.4.2",
"@backstage/catalog-model": "^0.6.0",
"@backstage/config": "^0.1.2",
@@ -43,7 +44,6 @@
"@google-cloud/storage": "^5.6.0",
"@types/dockerode": "^3.2.1",
"@types/express": "^4.17.6",
"aws-sdk": "^2.817.0",
"cross-fetch": "^3.0.6",
"dockerode": "^3.2.1",
"express": "^4.17.1",
@@ -56,6 +56,7 @@
"winston": "^3.2.1"
},
"devDependencies": {
"@aws-sdk/types": "3.1.0",
"@backstage/cli": "^0.4.5",
"@types/fs-extra": "^9.0.5",
"@types/git-url-parse": "^9.0.0",
packages/techdocs-common/src/stages/publish/awsS3.test.ts
Regular → Executable
+8 -13
View File
@@ -102,12 +102,6 @@ describe('AwsS3Publish', () => {
});
it('should fail to publish a directory', async () => {
const wrongPathToGeneratedDirectory = path.join(
'wrong',
'path',
'to',
'generatedDirectory',
);
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
@@ -124,13 +118,11 @@ describe('AwsS3Publish', () => {
await publisher
.publish({
entity,
directory: wrongPathToGeneratedDirectory,
directory: '/wrong/path/to/generatedDirectory',
})
.catch(error =>
expect(error).toEqual(
new Error(
`Unable to upload file(s) to AWS S3. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
),
expect(error.message).toContain(
'Unable to upload file(s) to AWS S3. Error Failed to read template directory',
),
);
mockFs.restore();
@@ -180,14 +172,17 @@ describe('AwsS3Publish', () => {
it('should return an error if the techdocs_metadata.json file is not present', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
const {
metadata: { name, namespace },
kind,
} = entity;
await publisher
.fetchTechDocsMetadata(entityNameMock)
.catch(error =>
expect(error).toEqual(
new Error(
`TechDocs metadata fetch failed, The file ${entityRootDir}/techdocs_metadata.json doest not exist !`,
`TechDocs metadata fetch failed, The file ${namespace}/${kind}/${name}/techdocs_metadata.json doest not exist.`,
),
),
);
packages/techdocs-common/src/stages/publish/awsS3.ts
+47 -35
View File
@@ -15,14 +15,27 @@
*/
import path from 'path';
import express from 'express';
import aws from 'aws-sdk';
import { PutObjectCommandOutput, S3 } from '@aws-sdk/client-s3';
import { Logger } from 'winston';
import { Entity, EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
import { PublisherBase, PublishRequest } from './types';
import { ManagedUpload } from 'aws-sdk/clients/s3';
import fs from 'fs-extra';
import { Readable } from 'stream';
const streamToString = (stream: Readable): Promise<string> => {
return new Promise((resolve, reject) => {
try {
const chunks: any[] = [];
stream.on('data', chunk => chunks.push(chunk));
stream.on('error', reject);
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
} catch (e) {
throw new Error(`Unable to parse the response data, ${e.message}`);
}
});
};
export class AwsS3Publish implements PublisherBase {
static fromConfig(config: Config, logger: Logger): PublisherBase {
@@ -47,7 +60,7 @@ export class AwsS3Publish implements PublisherBase {
);
}
const storageClient = new aws.S3({
const storageClient = new S3({
credentials: { accessKeyId, secretAccessKey },
...(region && { region }),
});
@@ -79,7 +92,7 @@ export class AwsS3Publish implements PublisherBase {
}
constructor(
private readonly storageClient: aws.S3,
private readonly storageClient: S3,
private readonly bucketName: string,
private readonly logger: Logger,
) {
@@ -98,7 +111,7 @@ export class AwsS3Publish implements PublisherBase {
// So collecting path of only the files is good enough.
const allFilesToUpload = await getFileTreeRecursively(directory);
const uploadPromises: Array<Promise<ManagedUpload.SendData>> = [];
const uploadPromises: Array<Promise<PutObjectCommandOutput>> = [];
for (const filePath of allFilesToUpload) {
// Remove the absolute path prefix of the source directory
@@ -116,7 +129,7 @@ export class AwsS3Publish implements PublisherBase {
Body: fileContent,
};
uploadPromises.push(this.storageClient.upload(params).promise());
uploadPromises.push(this.storageClient.putObject(params));
}
await Promise.all(uploadPromises);
this.logger.info(
@@ -135,25 +148,27 @@ export class AwsS3Publish implements PublisherBase {
return await new Promise<string>((resolve, reject) => {
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
const fileStreamChunks: Array<any> = [];
this.storageClient
.getObject({
Bucket: this.bucketName,
Key: `${entityRootDir}/techdocs_metadata.json`,
})
.createReadStream()
.on('error', err => {
.then(async file => {
const techdocsMetadataJson = await streamToString(
file.Body as Readable,
);
if (!techdocsMetadataJson) {
throw new Error(
`Unable to parse the techdocs metadata file ${entityRootDir}/techdocs_metadata.json.`,
);
}
resolve(techdocsMetadataJson);
})
.catch(err => {
this.logger.error(err.message);
reject(new Error(err.message));
})
.on('data', chunk => {
fileStreamChunks.push(chunk);
})
.on('end', () => {
const techdocsMetadataJson = Buffer.concat(
fileStreamChunks,
).toString();
resolve(techdocsMetadataJson);
});
});
} catch (e) {
@@ -174,19 +189,14 @@ export class AwsS3Publish implements PublisherBase {
const fileExtension = path.extname(filePath);
const responseHeaders = getHeadersForFileExtension(fileExtension);
const fileStreamChunks: Array<any> = [];
this.storageClient
.getObject({ Bucket: this.bucketName, Key: filePath })
.createReadStream()
.on('error', err => {
this.logger.warn(err.message);
res.status(404).send(err.message);
})
.on('data', chunk => {
fileStreamChunks.push(chunk);
})
.on('end', () => {
const fileContent = Buffer.concat(fileStreamChunks);
.then(async object => {
const fileContent = await streamToString(object.Body as Readable);
if (!fileContent) {
throw new Error(`Unable to parse the file ${filePath}.`);
}
// Inject response headers
for (const [headerKey, headerValue] of Object.entries(
responseHeaders,
@@ -195,6 +205,10 @@ export class AwsS3Publish implements PublisherBase {
}
res.send(fileContent);
})
.catch(err => {
this.logger.warn(err.message);
res.status(404).send(err.message);
});
};
}
@@ -206,12 +220,10 @@ export class AwsS3Publish implements PublisherBase {
async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
try {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
await this.storageClient
.headObject({
Bucket: this.bucketName,
Key: `${entityRootDir}/index.html`,
})
.promise();
await this.storageClient.headObject({
Bucket: this.bucketName,
Key: `${entityRootDir}/index.html`,
});
return Promise.resolve(true);
} catch (e) {
return Promise.resolve(false);