paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

Merge pull request #4434 from lowjoel/aws-sdk-downgrade

Browse Source 
Downgrade the AWS SDK to v2
This commit is contained in:
Patrik Oldsberg
2021-02-11 11:02:11 +01:00
committed by GitHub
parent f89138f6c7 77ad0003a5
commit cd13b03bc6
9 changed files with 222 additions and 924 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/bright-shirts-glow.md
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/techdocs-common': patch
'@backstage/plugin-catalog-backend': patch
---
Revert AWS SDK version to v2
packages/techdocs-common/__mocks__/@aws-sdk/client-s3.ts → packages/techdocs-common/__mocks__/aws-sdk.ts
+48 -23
View File
@@ -13,49 +13,74 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { S3ClientConfig } from '@aws-sdk/client-s3';
import type { S3 as S3Types } from 'aws-sdk';
import { EventEmitter } from 'events';
import fs from 'fs';
export class S3 {
private readonly options;
constructor(options: S3ClientConfig) {
constructor(options: S3Types.ClientConfiguration) {
this.options = options;
}
headObject({ Key }: { Key: string }) {
return {
promise: () => this.checkFileExists(Key),
};
}
getObject({ Key }: { Key: string }) {
return {
promise: () => this.checkFileExists(Key),
createReadStream: () => {
const emitter = new EventEmitter();
process.nextTick(() => {
if (fs.existsSync(Key)) {
emitter.emit('data', Buffer.from(fs.readFileSync(Key)));
} else {
emitter.emit(
'error',
new Error(`The file ${Key} doest not exist !`),
);
}
emitter.emit('end');
});
return emitter;
},
};
}
checkFileExists(Key: string) {
return new Promise((resolve, reject) => {
if (fs.existsSync(Key)) {
resolve('');
} else {
reject({ message: `The file ${Key} doest not exist.` });
}
});
}
getObject({ Key }: { Key: string }) {
return new Promise((resolve, reject) => {
if (fs.existsSync(Key)) {
const emitter = new EventEmitter();
process.nextTick(() => {
emitter.emit('data', Buffer.from(fs.readFileSync(Key)));
emitter.emit('end');
});
resolve({
Body: emitter,
});
} else {
reject({ message: `The file ${Key} doest not exist.` });
reject({ message: 'The object doest not exist !' });
}
});
}
headBucket() {
return '';
return new Promise(resolve => {
resolve('');
});
}
putObject() {
return '';
upload({ Key }: { Key: string }) {
return {
promise: () =>
new Promise((resolve, reject) => {
if (!fs.existsSync(Key)) {
reject('');
} else {
resolve('');
}
}),
};
}
}
export default {
S3,
};
packages/techdocs-common/package.json
+1 -2
View File
@@ -36,7 +36,6 @@
"url": "https://github.com/backstage/backstage/issues"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.1.0",
"@azure/identity": "^1.2.2",
"@azure/storage-blob": "^12.4.0",
"@backstage/backend-common": "^0.5.2",
@@ -46,6 +45,7 @@
"@google-cloud/storage": "^5.6.0",
"@types/dockerode": "^3.2.1",
"@types/express": "^4.17.6",
"aws-sdk": "^2.840.0",
"cross-fetch": "^3.0.6",
"dockerode": "^3.2.1",
"express": "^4.17.1",
@@ -60,7 +60,6 @@
"winston": "^3.2.1"
},
"devDependencies": {
"@aws-sdk/types": "3.1.0",
"@backstage/cli": "^0.6.0",
"@types/fs-extra": "^9.0.5",
"@types/git-url-parse": "^9.0.0",
packages/techdocs-common/src/stages/publish/awsS3.test.ts
Executable → Regular
+13 -8
View File
@@ -102,6 +102,12 @@ describe('AwsS3Publish', () => {
});
it('should fail to publish a directory', async () => {
const wrongPathToGeneratedDirectory = path.join(
'wrong',
'path',
'to',
'generatedDirectory',
);
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
@@ -118,11 +124,13 @@ describe('AwsS3Publish', () => {
await publisher
.publish({
entity,
directory: '/wrong/path/to/generatedDirectory',
directory: wrongPathToGeneratedDirectory,
})
.catch(error =>
expect(error.message).toContain(
'Unable to upload file(s) to AWS S3. Error Failed to read template directory',
expect(error).toEqual(
new Error(
`Unable to upload file(s) to AWS S3. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
),
),
);
mockFs.restore();
@@ -198,17 +206,14 @@ describe('AwsS3Publish', () => {
it('should return an error if the techdocs_metadata.json file is not present', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const {
metadata: { name, namespace },
kind,
} = entity;
const entityRootDir = getEntityRootDir(entity);
await publisher
.fetchTechDocsMetadata(entityNameMock)
.catch(error =>
expect(error).toEqual(
new Error(
`TechDocs metadata fetch failed, The file ${namespace}/${kind}/${name}/techdocs_metadata.json doest not exist.`,
`TechDocs metadata fetch failed, The file ${entityRootDir}/techdocs_metadata.json doest not exist !`,
),
),
);
packages/techdocs-common/src/stages/publish/awsS3.ts
+47 -48
View File
@@ -15,7 +15,8 @@
*/
import path from 'path';
import express from 'express';
import { PutObjectCommandOutput, S3 } from '@aws-sdk/client-s3';
import aws from 'aws-sdk';
import { ManagedUpload } from 'aws-sdk/clients/s3';
import { Logger } from 'winston';
import { Entity, EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
@@ -71,7 +72,7 @@ export class AwsS3Publish implements PublisherBase {
// https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html
const region = config.getOptionalString('techdocs.publisher.awsS3.region');
const storageClient = new S3({
const storageClient = new aws.S3({
...(credentials &&
accessKeyId &&
secretAccessKey && {
@@ -113,7 +114,7 @@ export class AwsS3Publish implements PublisherBase {
}
constructor(
private readonly storageClient: S3,
private readonly storageClient: aws.S3,
private readonly bucketName: string,
private readonly logger: Logger,
) {
@@ -133,7 +134,7 @@ export class AwsS3Publish implements PublisherBase {
const allFilesToUpload = await getFileTreeRecursively(directory);
const limiter = createLimiter(10);
const uploadPromises: Array<Promise<PutObjectCommandOutput>> = [];
const uploadPromises: Array<Promise<ManagedUpload.SendData>> = [];
for (const filePath of allFilesToUpload) {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
@@ -151,7 +152,9 @@ export class AwsS3Publish implements PublisherBase {
};
// Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
const uploadFile = limiter(() => this.storageClient.putObject(params));
const uploadFile = limiter(() =>
this.storageClient.upload(params).promise(),
);
uploadPromises.push(uploadFile);
}
await Promise.all(uploadPromises);
@@ -170,34 +173,33 @@ export class AwsS3Publish implements PublisherBase {
entityName: EntityName,
): Promise<TechDocsMetadata> {
try {
return await new Promise<TechDocsMetadata>((resolve, reject) => {
return await new Promise<TechDocsMetadata>(async (resolve, reject) => {
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
this.storageClient
const stream = this.storageClient
.getObject({
Bucket: this.bucketName,
Key: `${entityRootDir}/techdocs_metadata.json`,
})
.then(async file => {
const techdocsMetadataJson = await streamToBuffer(
file.Body as Readable,
);
.createReadStream();
if (!techdocsMetadataJson) {
throw new Error(
`Unable to parse the techdocs metadata file ${entityRootDir}/techdocs_metadata.json.`,
);
}
const techdocsMetadata = JSON5.parse(
techdocsMetadataJson.toString('utf-8'),
try {
const techdocsMetadataJson = await streamToBuffer(stream);
if (!techdocsMetadataJson) {
throw new Error(
`Unable to parse the techdocs metadata file ${entityRootDir}/techdocs_metadata.json.`,
);
}
resolve(techdocsMetadata);
})
.catch(err => {
this.logger.error(err.message);
reject(new Error(err.message));
});
const techdocsMetadata = JSON5.parse(
techdocsMetadataJson.toString('utf-8'),
);
resolve(techdocsMetadata);
} catch (err) {
this.logger.error(err.message);
reject(new Error(err.message));
}
});
} catch (e) {
throw new Error(`TechDocs metadata fetch failed, ${e.message}`);
@@ -208,7 +210,7 @@ export class AwsS3Publish implements PublisherBase {
* Express route middleware to serve static files on a route in techdocs-backend.
*/
docsRouter(): express.Handler {
return (req, res) => {
return async (req, res) => {
// Trim the leading forward slash
// filePath example - /default/Component/documented-component/index.html
const filePath = req.path.replace(/^\//, '');
@@ -217,27 +219,22 @@ export class AwsS3Publish implements PublisherBase {
const fileExtension = path.extname(filePath);
const responseHeaders = getHeadersForFileExtension(fileExtension);
this.storageClient
const stream = this.storageClient
.getObject({ Bucket: this.bucketName, Key: filePath })
.then(async object => {
const fileContent = await streamToBuffer(object.Body as Readable);
if (!fileContent) {
throw new Error(`Unable to parse the file ${filePath}.`);
}
.createReadStream();
try {
// Inject response headers
for (const [headerKey, headerValue] of Object.entries(
responseHeaders,
)) {
res.setHeader(headerKey, headerValue);
}
// Inject response headers
for (const [headerKey, headerValue] of Object.entries(
responseHeaders,
)) {
res.setHeader(headerKey, headerValue);
}
res.send(fileContent);
})
.catch(err => {
this.logger.warn(err.message);
res.status(404).send(err.message);
});
res.send(await streamToBuffer(stream));
} catch (err) {
this.logger.warn(err.message);
res.status(404).send(err.message);
}
};
}
@@ -248,10 +245,12 @@ export class AwsS3Publish implements PublisherBase {
async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
try {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
await this.storageClient.headObject({
Bucket: this.bucketName,
Key: `${entityRootDir}/index.html`,
});
await this.storageClient
.headObject({
Bucket: this.bucketName,
Key: `${entityRootDir}/index.html`,
})
.promise();
return Promise.resolve(true);
} catch (e) {
return Promise.resolve(false);
plugins/catalog-backend/package.json
+1 -1
View File
@@ -29,7 +29,6 @@
"clean": "backstage-cli clean"
},
"dependencies": {
"@aws-sdk/client-organizations": "^3.2.0",
"@azure/msal-node": "^1.0.0-beta.3",
"@backstage/backend-common": "^0.5.2",
"@backstage/catalog-model": "^0.7.1",
@@ -38,6 +37,7 @@
"@octokit/graphql": "^4.5.8",
"@types/express": "^4.17.6",
"@types/ldapjs": "^1.0.9",
"aws-sdk": "^2.840.0",
"codeowners-utils": "^1.0.2",
"core-js": "^3.6.5",
"cross-fetch": "^3.0.6",
plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts
+37 -29
View File
@@ -27,18 +27,22 @@ describe('AwsOrganizationCloudAccountProcessor', () => {
afterEach(() => jest.resetAllMocks());
it('generates component entities for accounts', async () => {
listAccounts.mockImplementation(() =>
Promise.resolve({
Accounts: [
{
Arn:
'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
Name: 'testaccount',
},
],
NextToken: undefined,
}),
);
listAccounts.mockImplementation(() => {
return {
async promise() {
return {
Accounts: [
{
Arn:
'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
Name: 'testaccount',
},
],
NextToken: undefined,
};
},
};
});
await processor.readLocation(location, false, emit);
expect(emit).toBeCalledWith({
type: 'entity',
@@ -69,23 +73,27 @@ describe('AwsOrganizationCloudAccountProcessor', () => {
type: 'aws-cloud-accounts',
target: 'o-1vl18kc5a3',
};
listAccounts.mockImplementation(() =>
Promise.resolve({
Accounts: [
{
Arn:
'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
Name: 'testaccount',
},
{
Arn:
'arn:aws:organizations::192594491037:account/o-zzzzzzzzz/957140518395',
Name: 'testaccount2',
},
],
NextToken: undefined,
}),
);
listAccounts.mockImplementation(() => {
return {
async promise() {
return {
Accounts: [
{
Arn:
'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
Name: 'testaccount',
},
{
Arn:
'arn:aws:organizations::192594491037:account/o-zzzzzzzzz/957140518395',
Name: 'testaccount2',
},
],
NextToken: undefined,
};
},
};
});
await processor.readLocation(locationTest, false, emit);
expect(emit).toBeCalledTimes(1);
expect(emit).toBeCalledWith({
plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts
+6 -9
View File
@@ -14,11 +14,8 @@
* limitations under the License.
*/
import { LocationSpec, ResourceEntityV1alpha1 } from '@backstage/catalog-model';
import {
Account,
Organizations,
ListAccountsCommandOutput,
} from '@aws-sdk/client-organizations';
import AWS, { Organizations } from 'aws-sdk';
import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations';
import * as results from './results';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
@@ -38,7 +35,7 @@ const ORGANIZATION_ANNOTATION: string = 'amazonaws.com/organization-id';
export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
organizations: Organizations;
constructor() {
this.organizations = new Organizations({
this.organizations = new AWS.Organizations({
region: AWS_ORGANIZATION_REGION,
}); // Only available in us-east-1
}
@@ -67,9 +64,9 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
let nextToken = undefined;
while (isInitialAttempt || nextToken) {
isInitialAttempt = false;
const orgAccounts: ListAccountsCommandOutput = await this.organizations.listAccounts(
{ NextToken: nextToken },
);
const orgAccounts: ListAccountsResponse = await this.organizations
.listAccounts({ NextToken: nextToken })
.promise();
if (orgAccounts.Accounts) {
awsAccounts = awsAccounts.concat(orgAccounts.Accounts);
}
yarn.lock
+63 -804
View File
File diff suppressed because it is too large Load Diff