diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 1bec23c17e..141921a0cb 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -48,14 +48,13 @@ "fs-extra": "^9.0.1", "git-url-parse": "^11.4.0", "js-yaml": "^3.14.0", - "klaw": "^3.0.0", "mock-fs": "^4.13.0", "nodegit": "^0.27.0", + "recursive-readdir": "^2.2.2", "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.4.0", - "@types/klaw": "^3.0.1" + "@backstage/cli": "^0.4.0" }, "jest": { "roots": [ diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 342b0a2687..21742a272e 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -13,14 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import path from 'path'; import express from 'express'; -import walk from 'klaw'; import { Storage, UploadResponse } from '@google-cloud/storage'; import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { getHeadersForFileExtension, supportedFileType } from './helpers'; +import { + getHeadersForFileExtension, + supportedFileType, + getFileTreeRecursively, +} from './helpers'; import { PublisherBase, PublishRequest } from './types'; export class GoogleGCSPublish implements PublisherBase { @@ -94,53 +96,38 @@ export class GoogleGCSPublish implements PublisherBase { * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ publish({ entity, directory }: PublishRequest): Promise { - return new Promise((resolve, reject) => { - // Path of all files to upload, relative to the root of the source directory - // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] - const allFilesToUpload: Array = []; + return new Promise(async (resolve, reject) => { + // Note: GCS manages creation of parent directories if they do not exist. + // So collecting path of only the files is good enough. + const allFilesToUpload = await getFileTreeRecursively(directory); - // Iterate on all the files in the directory and its sub-directories - walk(directory) - .on('data', (item: walk.Item) => { - // GCS manages creation of parent directories if they do not exist. - // So collecting path of only the files is good enough. - if (item.stats.isFile()) { - // Remove the absolute path prefix of the source directory - const relativeFilePath = item.path.replace(`${directory}/`, ''); - allFilesToUpload.push(relativeFilePath); - } - }) - .on('error', (err: Error, item: walk.Item) => { - const errorMessage = `Unable to read file at ${item.path}. Error ${err.message}`; - this.logger.error(errorMessage); - reject(errorMessage); - }) - .on('end', () => { - // 'end' event happens when all the files have been read. - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - allFilesToUpload.forEach(filePath => { - const source = path.join(directory, filePath); // Local file absolute path - const destination = `${entityRootDir}/${filePath}`; // GCS Bucket file relative path - this.storageClient - .bucket(this.bucketName) - .upload(source, { destination }) - .then( - (uploadResp: UploadResponse) => ({ - fileName: destination, - status: uploadResp[0], - }), - (err: Error) => { - const errorMessage = `Unable to upload file ${destination} to GCS. Error ${err.message}`; - this.logger.error(errorMessage); - reject(errorMessage); - }, - ); - }); + const uploadPromises: Array> = []; + allFilesToUpload.forEach(filePath => { + // Remove the absolute path prefix of the source directory + // Path of all files to upload, relative to the root of the source directory + // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] + const relativeFilePath = filePath.replace(`${directory}/`, ''); + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const destination = `${entityRootDir}/${relativeFilePath}`; // GCS Bucket file relative path + // TODO: Upload in chunks of ~10 files instead of all files at once. + uploadPromises.push( + this.storageClient.bucket(this.bucketName).upload(filePath, { + destination, + }), + ); + }); + Promise.all(uploadPromises) + .then(() => { this.logger.info( - `Successfully uploaded all the generated files for Entity ${entityRootDir}. Total number of files: ${allFilesToUpload.length}`, + `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, ); resolve(undefined); + }) + .catch((err: Error) => { + const errorMessage = `Unable to upload file(s) to Google Cloud Storage. Error ${err.message}`; + this.logger.error(errorMessage); + reject(errorMessage); }); }); } diff --git a/packages/techdocs-common/src/stages/publish/helpers.test.ts b/packages/techdocs-common/src/stages/publish/helpers.test.ts new file mode 100644 index 0000000000..2f8fbadae0 --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/helpers.test.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import mockFs from 'mock-fs'; +import { getFileTreeRecursively } from './helpers'; + +describe('getFileTreeRecursively', () => { + beforeEach(() => { + mockFs({ + '/rootDir': { + file1: '', + subDirA: { + file2: '', + emptyDir1: mockFs.directory(), + }, + emptyDir2: mockFs.directory(), + }, + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('returns complete file tree of a path', async () => { + const fileList = await getFileTreeRecursively('/rootDir'); + expect(fileList.length).toBe(2); + expect(fileList).toContain('/rootDir/file1'); + expect(fileList).toContain('/rootDir/subDirA/file2'); + }); +}); diff --git a/packages/techdocs-common/src/stages/publish/helpers.ts b/packages/techdocs-common/src/stages/publish/helpers.ts index cca5d85150..2939bbfc1c 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import recursiveReadDir from 'recursive-readdir'; + export type supportedFileType = 'html' | 'css'; export type responseHeadersType = { @@ -44,3 +46,37 @@ export const getHeadersForFileExtension = ( return headersCommon; } }; + +/** + * Recursively traverse all the sub-directories of a path and return + * a list of absolute paths of all the files. e.g. tree command in Unix + * + * @example + * + * /User/username/my_dir + * dirA + * | subDirA + * | | file1 + * EmptyDir + * dirB + * | file2 + * file3 + * + * getFileListRecursively('/Users/username/myDir') + * // returns + * [ + * '/User/username/my_dir/dirA/subDirA/file1', + * '/User/username/my_dir/dirB/file2', + * '/User/username/my_dir/file3' + * ] + * @param rootDirPath Absolute path to the root directory. + */ +export const getFileTreeRecursively = async ( + rootDirPath: string, +): Promise => { + // Iterate on all the files in the directory and its sub-directories + const fileList = await recursiveReadDir(rootDirPath).catch(error => { + throw new Error(`Failed to read template directory: ${error.message}`); + }); + return fileList; +}; diff --git a/yarn.lock b/yarn.lock index 399a84dd17..e2cfe16504 100644 --- a/yarn.lock +++ b/yarn.lock @@ -844,14 +844,7 @@ dependencies: "@babel/helper-plugin-utils" "^7.8.0" -"@babel/plugin-syntax-top-level-await@^7.12.1": - version "7.12.1" - resolved "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.12.1.tgz#dd6c0b357ac1bb142d98537450a319625d13d2a0" - integrity sha512-i7ooMZFS+a/Om0crxZodrTzNEPJHZrlMVGMTEpFAj6rYY/bKCddB0Dk/YxfPuYXOopuhKk/e1jV6h+WUU9XN3A== - dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - -"@babel/plugin-syntax-top-level-await@^7.8.3": +"@babel/plugin-syntax-top-level-await@^7.12.1", "@babel/plugin-syntax-top-level-await@^7.8.3": version "7.12.1" resolved "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.12.1.tgz#dd6c0b357ac1bb142d98537450a319625d13d2a0" integrity sha512-i7ooMZFS+a/Om0crxZodrTzNEPJHZrlMVGMTEpFAj6rYY/bKCddB0Dk/YxfPuYXOopuhKk/e1jV6h+WUU9XN3A== @@ -5894,13 +5887,6 @@ dependencies: "@types/node" "*" -"@types/klaw@^3.0.1": - version "3.0.1" - resolved "https://registry.npmjs.org/@types/klaw/-/klaw-3.0.1.tgz#29f90021c0234976aa4eb97efced9cb6db9fa8b3" - integrity sha512-acnF3n9mYOr1aFJKFyvfNX0am9EtPUsYPq22QUCGdJE+MVt6UyAN1jwo+PmOPqXD4K7ZS9MtxDEp/un0lxFccA== - dependencies: - "@types/node" "*" - "@types/koa-compose@*": version "3.2.5" resolved "https://registry.npmjs.org/@types/koa-compose/-/koa-compose-3.2.5.tgz#85eb2e80ac50be95f37ccf8c407c09bbe3468e9d" @@ -16255,13 +16241,6 @@ klaw@^1.0.0: optionalDependencies: graceful-fs "^4.1.9" -klaw@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/klaw/-/klaw-3.0.0.tgz#b11bec9cf2492f06756d6e809ab73a2910259146" - integrity sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g== - dependencies: - graceful-fs "^4.1.9" - kleur@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e"