From 8d950d0ae7bbbce10a413f23262888b6195997ff Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 15 Jan 2021 18:02:37 +0100 Subject: [PATCH 1/5] Stream files directly from GCS instead of buffering and sending. --- .../src/stages/publish/googleStorage.ts | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index e6b2c26943..f92b905f73 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -169,30 +169,23 @@ export class GoogleGCSPublish implements PublisherBase { // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); const responseHeaders = getHeadersForFileExtension(fileExtension); + res.writeHead(200, { + ...{ + 'Transfer-Encoding': 'chunked', + }, + ...responseHeaders, + }); - const fileStreamChunks: Array = []; + // Pipe file chunks directly from storage to client. this.storageClient .bucket(this.bucketName) .file(filePath) .createReadStream() .on('error', err => { this.logger.warn(err.message); - res.status(404).send(err.message); + res.destroy(err); }) - .on('data', chunk => { - fileStreamChunks.push(chunk); - }) - .on('end', () => { - const fileContent = Buffer.concat(fileStreamChunks); - // Inject response headers - for (const [headerKey, headerValue] of Object.entries( - responseHeaders, - )) { - res.setHeader(headerKey, headerValue); - } - - res.send(fileContent); - }); + .pipe(res); }; } From ce9d5ac220d9a22dab46d0d97e278fe3a0c5f0a7 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 15 Jan 2021 19:17:01 +0100 Subject: [PATCH 2/5] Remove unnecessary chunk transfer encoding header. --- .../techdocs-common/src/stages/publish/googleStorage.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index f92b905f73..df9d1120bf 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -169,12 +169,7 @@ export class GoogleGCSPublish implements PublisherBase { // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); const responseHeaders = getHeadersForFileExtension(fileExtension); - res.writeHead(200, { - ...{ - 'Transfer-Encoding': 'chunked', - }, - ...responseHeaders, - }); + res.writeHead(200, responseHeaders); // Pipe file chunks directly from storage to client. this.storageClient From da6a29b6d9b71cad303e00069f7f6ceaeaece8d1 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 18 Jan 2021 18:48:34 +0100 Subject: [PATCH 3/5] Ensure error handling happens the same way. --- .../src/stages/publish/googleStorage.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index df9d1120bf..9c28de4130 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -169,16 +169,23 @@ export class GoogleGCSPublish implements PublisherBase { // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); const responseHeaders = getHeadersForFileExtension(fileExtension); - res.writeHead(200, responseHeaders); // Pipe file chunks directly from storage to client. this.storageClient .bucket(this.bucketName) .file(filePath) .createReadStream() + .on('pipe', () => { + res.writeHead(200, responseHeaders); + }) .on('error', err => { this.logger.warn(err.message); - res.destroy(err); + // Send a 404 with a meaningful message if possible. + if (!res.headersSent) { + res.status(404).send(err.message); + } else { + res.destroy(); + } }) .pipe(res); }; From 53c9c51f2161c847cb6459ef4dd10c0bdd865758 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 19 Jan 2021 10:17:12 +0100 Subject: [PATCH 4/5] Create techdocs-glasses-wonder.md --- .changeset/techdocs-glasses-wonder.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/techdocs-glasses-wonder.md diff --git a/.changeset/techdocs-glasses-wonder.md b/.changeset/techdocs-glasses-wonder.md new file mode 100644 index 0000000000..f3c9adb29f --- /dev/null +++ b/.changeset/techdocs-glasses-wonder.md @@ -0,0 +1,5 @@ +--- +"@backstage/techdocs-common": patch +--- + +TechDocs backend now streams files through from Google Cloud Storage to the browser, improving memory usage. From 18c97a8757d86326d35c0d8b13e43f4b771d7c99 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 19 Jan 2021 12:01:10 +0100 Subject: [PATCH 5/5] Prettier? --- .changeset/techdocs-glasses-wonder.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/techdocs-glasses-wonder.md b/.changeset/techdocs-glasses-wonder.md index f3c9adb29f..7610be2665 100644 --- a/.changeset/techdocs-glasses-wonder.md +++ b/.changeset/techdocs-glasses-wonder.md @@ -1,5 +1,5 @@ --- -"@backstage/techdocs-common": patch +'@backstage/techdocs-common': patch --- TechDocs backend now streams files through from Google Cloud Storage to the browser, improving memory usage.