diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 9def24722a..349d6ead97 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -15,18 +15,19 @@ */ import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { - FileExistsResponse, - Storage, - UploadResponse, -} from '@google-cloud/storage'; +import { FileExistsResponse, Storage } from '@google-cloud/storage'; import express from 'express'; import JSON5 from 'json5'; -import createLimiter from 'p-limit'; import path from 'path'; import { Readable } from 'stream'; import { Logger } from 'winston'; -import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { + bulkStorageOperation, + getCloudPathForLocalPath, + getFileTreeRecursively, + getHeadersForFileExtension, + getStaleFiles, +} from './helpers'; import { MigrateWriteStream } from './migrations'; import { PublisherBase, @@ -114,49 +115,68 @@ export class GoogleGCSPublish implements PublisherBase { * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ async publish({ entity, directory }: PublishRequest): Promise { + const bucket = this.storageClient.bucket(this.bucketName); + + // First, retrieve a list of all individual files in currently existing + const remoteFolder = getCloudPathForLocalPath(entity); + const existingFiles = ( + await bucket.getFiles({ prefix: remoteFolder }) + )[0].map(file => file.name); + + // Then, merge new files into the same folder + let absoluteFilesToUpload; try { - // Note: GCS manages creation of parent directories if they do not exist. - // So collecting path of only the files is good enough. - const allFilesToUpload = await getFileTreeRecursively(directory); + // Remove the absolute path prefix of the source directory + // Path of all files to upload, relative to the root of the source directory + // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] + absoluteFilesToUpload = await getFileTreeRecursively(directory); - const limiter = createLimiter(10); - const uploadPromises: Array> = []; - allFilesToUpload.forEach(sourceFilePath => { - // Remove the absolute path prefix of the source directory - // Path of all files to upload, relative to the root of the source directory - // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] - const relativeFilePath = path.relative(directory, sourceFilePath); - - // Convert destination file path to a POSIX path for uploading. - // GCS expects / as path separator and relativeFilePath will contain \\ on Windows. - // https://cloud.google.com/storage/docs/gsutil/addlhelp/HowSubdirectoriesWork - const relativeFilePathPosix = relativeFilePath - .split(path.sep) - .join(path.posix.sep); - - // The / delimiter is intentional since it represents the cloud storage and not the local file system. - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path - - // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) - const uploadFile = limiter(() => - this.storageClient.bucket(this.bucketName).upload(sourceFilePath, { - destination, - }), - ); - uploadPromises.push(uploadFile); - }); - - await Promise.all(uploadPromises); + await bulkStorageOperation( + async absoluteFilePath => { + const relativeFilePath = path.relative(directory, absoluteFilePath); + return await bucket.upload(absoluteFilePath, { + destination: getCloudPathForLocalPath(entity, relativeFilePath), + }); + }, + absoluteFilesToUpload, + { concurrencyLimit: 10 }, + ); this.logger.info( - `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, + `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${absoluteFilesToUpload.length}`, ); } catch (e) { const errorMessage = `Unable to upload file(s) to Google Cloud Storage. ${e}`; this.logger.error(errorMessage); throw new Error(errorMessage); } + + // Last, try to remove the files that were *only* present previously + try { + const relativeFilesToUpload = absoluteFilesToUpload.map( + absoluteFilePath => + getCloudPathForLocalPath( + entity, + path.relative(directory, absoluteFilePath), + ), + ); + const staleFiles = getStaleFiles(relativeFilesToUpload, existingFiles); + + await bulkStorageOperation( + async relativeFilePath => { + return await bucket.file(relativeFilePath).delete(); + }, + staleFiles, + { concurrencyLimit: 10 }, + ); + + this.logger.info( + `Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: ${staleFiles.length}`, + ); + } catch (error) { + const errorMessage = `Unable to delete file(s) from Google Cloud Storage. ${error}`; + this.logger.error(errorMessage); + } } fetchTechDocsMetadata(entityName: EntityName): Promise { diff --git a/packages/techdocs-common/src/stages/publish/helpers.ts b/packages/techdocs-common/src/stages/publish/helpers.ts index ed681eb8ba..1f77d33971 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.ts @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import mime from 'mime-types'; +import path from 'path'; +import createLimiter from 'p-limit'; import recursiveReadDir from 'recursive-readdir'; /** @@ -110,4 +113,41 @@ export const lowerCaseEntityTripletInStoragePath = ( const lowerKind = kind.toLowerCase(); const lowerName = name.toLowerCase(); return [lowerNamespace, lowerKind, lowerName, ...parts].join('/'); +} + +// Only returns the files that existed previously and are not present anymore. +export const getStaleFiles = ( + newFiles: string[], + oldFiles: string[], +): string[] => { + const staleFiles = new Set(oldFiles); + newFiles.forEach(newFile => { + staleFiles.delete(newFile); + }); + return Array.from(staleFiles); +}; + +// Compose actual filename on remote bucket including entity information +export const getCloudPathForLocalPath = ( + entity: Entity, + localPath = '', +): string => { + // Convert destination file path to a POSIX path for uploading. + // GCS expects / as path separator and relativeFilePath will contain \\ on Windows. + // https://cloud.google.com/storage/docs/gsutil/addlhelp/HowSubdirectoriesWork + const relativeFilePathPosix = localPath.split(path.sep).join(path.posix.sep); + + // The / delimiter is intentional since it represents the cloud storage and not the local file system. + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + return `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path +}; + +// Perform rate limited generic operations by passing a function and a list of arguments +export const bulkStorageOperation = async ( + operation: (arg: T) => Promise, + args: T[], + { concurrencyLimit } = { concurrencyLimit: 25 }, +) => { + const limiter = createLimiter(concurrencyLimit); + await Promise.all(args.map(arg => limiter(operation, arg))); };