diff --git a/.changeset/techdocs-dirty-rivers-hope.md b/.changeset/techdocs-dirty-rivers-hope.md new file mode 100644 index 0000000000..3f49afa529 --- /dev/null +++ b/.changeset/techdocs-dirty-rivers-hope.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Add rate limiter for concurrent execution of file uploads in AWS and Google publishers diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 0a1a673f3a..93825511cb 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -53,6 +53,7 @@ "json5": "^2.1.3", "mime-types": "^2.1.27", "mock-fs": "^4.13.0", + "p-limit": "^3.1.0", "recursive-readdir": "^2.2.2", "winston": "^3.2.1" }, diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index b9e9fe12e6..e7014df967 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -24,6 +24,7 @@ import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; import fs from 'fs-extra'; import { Readable } from 'stream'; import JSON5 from 'json5'; +import limiterFactory from 'p-limit'; const streamToBuffer = (stream: Readable): Promise => { return new Promise((resolve, reject) => { @@ -131,8 +132,8 @@ export class AwsS3Publish implements PublisherBase { // So collecting path of only the files is good enough. const allFilesToUpload = await getFileTreeRecursively(directory); + const limiter = limiterFactory(10); const uploadPromises: Array> = []; - for (const filePath of allFilesToUpload) { // Remove the absolute path prefix of the source directory // Path of all files to upload, relative to the root of the source directory @@ -149,7 +150,11 @@ export class AwsS3Publish implements PublisherBase { Body: fileContent, }; - uploadPromises.push(this.storageClient.putObject(params)); + // Rate limit the concurrent execution of file uploads to batches of 10 + const uploadFile = limiter(async () => + this.storageClient.putObject(params), + ); + uploadPromises.push(uploadFile); } await Promise.all(uploadPromises); this.logger.info( diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 82a9620e8f..d8fe0b7d26 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -26,6 +26,7 @@ import { Config } from '@backstage/config'; import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; import JSON5 from 'json5'; +import limitFactory from 'p-limit'; export class GoogleGCSPublish implements PublisherBase { static async fromConfig( @@ -102,6 +103,7 @@ export class GoogleGCSPublish implements PublisherBase { // So collecting path of only the files is good enough. const allFilesToUpload = await getFileTreeRecursively(directory); + const limiter = limitFactory(10); const uploadPromises: Array> = []; allFilesToUpload.forEach(filePath => { // Remove the absolute path prefix of the source directory @@ -110,12 +112,14 @@ export class GoogleGCSPublish implements PublisherBase { const relativeFilePath = filePath.replace(`${directory}/`, ''); const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; const destination = `${entityRootDir}/${relativeFilePath}`; // GCS Bucket file relative path - // TODO: Upload in chunks of ~10 files instead of all files at once. - uploadPromises.push( - this.storageClient.bucket(this.bucketName).upload(filePath, { - destination, - }), + + // Rate limit the concurrent execution of file uploads to batches of 10 + const uploadFile = limiter(async () => + this.storageClient + .bucket(this.bucketName) + .upload(filePath, { destination }), ); + uploadPromises.push(uploadFile); }); Promise.all(uploadPromises) diff --git a/yarn.lock b/yarn.lock index f6762a63f0..cf807acf11 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19780,7 +19780,7 @@ p-limit@^2.0.0, p-limit@^2.2.0: dependencies: p-try "^2.0.0" -p-limit@^3.0.1, p-limit@^3.0.2: +p-limit@^3.0.1, p-limit@^3.0.2, p-limit@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==