diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index fe8fe9c2ce..5dc868ba2b 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -26,8 +26,11 @@ import limiterFactory from 'p-limit'; import { default as path, default as platformPath } from 'path'; import { Logger } from 'winston'; import { + bulkStorageOperation, + getCloudPathForLocalPath, getFileTreeRecursively, getHeadersForFileExtension, + getStaleFiles, lowerCaseEntityTripletInStoragePath, } from './helpers'; import { @@ -133,75 +136,85 @@ export class AzureBlobStoragePublish implements PublisherBase { * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html */ async publish({ entity, directory }: PublishRequest): Promise { + // First, retrieve a list of all individual files in currently existing + const remoteFolder = getCloudPathForLocalPath(entity); + const existingFiles: string[] = []; + const container = this.storageClient.getContainerClient(this.containerName); try { - // Note: Azure Blob Storage manages creation of parent directories if they do not exist. - // So collecting path of only the files is good enough. - const allFilesToUpload = await getFileTreeRecursively(directory); - - // Bound the number of concurrent batches. We want a bit of concurrency for - // performance reasons, but not so much that we starve the connection pool - // or start thrashing. - const limiter = limiterFactory(BATCH_CONCURRENCY); - - const promises = allFilesToUpload.map(sourceFilePath => { - // Remove the absolute path prefix of the source directory - // Path of all files to upload, relative to the root of the source directory - // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] - const relativeFilePath = path.normalize( - path.relative(directory, sourceFilePath), - ); - - // Convert destination file path to a POSIX path for uploading. - // Azure Blob Storage expects / as path separator and relativeFilePath will contain \\ on Windows. - // https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#blob-names - const relativeFilePathPosix = relativeFilePath - .split(path.sep) - .join(path.posix.sep); - - // The / delimiter is intentional since it represents the cloud storage and not the local file system. - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Azure Blob Storage Container file relative path - return limiter(async () => { - const response = await this.storageClient - .getContainerClient(this.containerName) - .getBlockBlobClient(destination) - .uploadFile(sourceFilePath); - - if (response._response.status >= 400) { - return { - ...response, - error: new Error( - `Upload failed for ${sourceFilePath} with status code ${response._response.status}`, - ), - }; - } - return { - ...response, - error: undefined, - }; - }); - }); - - const responses = await Promise.all(promises); - - const failed = responses.filter(r => r.error); - if (failed.length === 0) { - this.logger.info( - `Successfully uploaded the ${responses.length} generated file(s) for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, - ); - } else { - throw new Error( - failed - .map(r => r.error?.message) - .filter(Boolean) - .join(' '), - ); + for await (const blob of container.listBlobsFlat()) { + existingFiles.push(blob.name); } } catch (e) { - const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e}`; + const errorMessage = `Unable to list file(s) to Azure. ${e}`; this.logger.error(errorMessage); throw new Error(errorMessage); } + + // Then, merge new files into the same folder + let absoluteFilesToUpload; + try { + // Remove the absolute path prefix of the source directory + // Path of all files to upload, relative to the root of the source directory + // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] + absoluteFilesToUpload = await getFileTreeRecursively(directory); + + await bulkStorageOperation( + async absoluteFilePath => { + // const relativeFilePath = path.relative(directory, absoluteFilePath); + const relativeFilePath = path.normalize( + path.relative(directory, absoluteFilePath), + ); + return await this.storageClient + .getContainerClient(this.containerName) + .getBlockBlobClient( + getCloudPathForLocalPath(entity, relativeFilePath), + ) + .uploadFile(absoluteFilePath); + }, + absoluteFilesToUpload, + { concurrencyLimit: BATCH_CONCURRENCY }, + ); + + this.logger.info( + `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${absoluteFilesToUpload.length}`, + ); + } catch (e) { + const errorMessage = `Unable to upload file(s) to Azure. ${e}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } + + // Last, try to remove the files that were *only* present previously + try { + const relativeFilesToUpload = absoluteFilesToUpload.map( + absoluteFilePath => + getCloudPathForLocalPath( + entity, + path.relative(directory, absoluteFilePath), + ), + ); + + const staleFiles = getStaleFiles( + relativeFilesToUpload, + existingFiles, + remoteFolder, + ); + + await bulkStorageOperation( + async relativeFilePath => { + return await container.deleteBlob(relativeFilePath); + }, + staleFiles, + { concurrencyLimit: BATCH_CONCURRENCY }, + ); + + this.logger.info( + `Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: ${staleFiles.length}`, + ); + } catch (error) { + const errorMessage = `Unable to delete file(s) from AWS S3. ${error}`; + this.logger.error(errorMessage); + } } private download(containerName: string, blobPath: string): Promise { diff --git a/packages/techdocs-common/src/stages/publish/helpers.ts b/packages/techdocs-common/src/stages/publish/helpers.ts index 1f77d33971..2aabcb3259 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.ts @@ -113,14 +113,21 @@ export const lowerCaseEntityTripletInStoragePath = ( const lowerKind = kind.toLowerCase(); const lowerName = name.toLowerCase(); return [lowerNamespace, lowerKind, lowerName, ...parts].join('/'); -} +}; // Only returns the files that existed previously and are not present anymore. export const getStaleFiles = ( newFiles: string[], oldFiles: string[], + remoteFolder?: string, ): string[] => { - const staleFiles = new Set(oldFiles); + let filteredFiles = [...oldFiles]; + if (remoteFolder) { + filteredFiles = filteredFiles.filter(filePath => + filePath.match(remoteFolder), + ); + } + const staleFiles = new Set(filteredFiles); newFiles.forEach(newFile => { staleFiles.delete(newFile); });