feat(techdocs): delete stale files when publishing to Azure

Signed-off-by: Camila Belo <camilaibs@gmail.com>

packages/techdocs-common/src/stages/publish/azureBlobStorage.ts

@@ -26,8 +26,11 @@ import limiterFactory from 'p-limit';
 import { default as path, default as platformPath } from 'path';
 import { Logger } from 'winston';
 import {
+  bulkStorageOperation,
+  getCloudPathForLocalPath,
   getFileTreeRecursively,
   getHeadersForFileExtension,
+  getStaleFiles,
   lowerCaseEntityTripletInStoragePath,
 } from './helpers';
 import {
@@ -133,75 +136,85 @@ export class AzureBlobStoragePublish implements PublisherBase {
    * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
    */
   async publish({ entity, directory }: PublishRequest): Promise<void> {
+    // First, retrieve a list of all individual files in currently existing
+    const remoteFolder = getCloudPathForLocalPath(entity);
+    const existingFiles: string[] = [];
+    const container = this.storageClient.getContainerClient(this.containerName);
     try {
-      // Note: Azure Blob Storage manages creation of parent directories if they do not exist.
-      // So collecting path of only the files is good enough.
-      const allFilesToUpload = await getFileTreeRecursively(directory);
-
-      // Bound the number of concurrent batches. We want a bit of concurrency for
-      // performance reasons, but not so much that we starve the connection pool
-      // or start thrashing.
-      const limiter = limiterFactory(BATCH_CONCURRENCY);
-
-      const promises = allFilesToUpload.map(sourceFilePath => {
-        // Remove the absolute path prefix of the source directory
-        // Path of all files to upload, relative to the root of the source directory
-        // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
-        const relativeFilePath = path.normalize(
-          path.relative(directory, sourceFilePath),
-        );
-
-        // Convert destination file path to a POSIX path for uploading.
-        // Azure Blob Storage expects / as path separator and relativeFilePath will contain \\ on Windows.
-        // https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#blob-names
-        const relativeFilePathPosix = relativeFilePath
-          .split(path.sep)
-          .join(path.posix.sep);
-
-        // The / delimiter is intentional since it represents the cloud storage and not the local file system.
-        const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
-        const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Azure Blob Storage Container file relative path
-        return limiter(async () => {
-          const response = await this.storageClient
-            .getContainerClient(this.containerName)
-            .getBlockBlobClient(destination)
-            .uploadFile(sourceFilePath);
-
-          if (response._response.status >= 400) {
-            return {
-              ...response,
-              error: new Error(
-                `Upload failed for ${sourceFilePath} with status code ${response._response.status}`,
-              ),
-            };
-          }
-          return {
-            ...response,
-            error: undefined,
-          };
-        });
-      });
-
-      const responses = await Promise.all(promises);
-
-      const failed = responses.filter(r => r.error);
-      if (failed.length === 0) {
-        this.logger.info(
-          `Successfully uploaded the ${responses.length} generated file(s) for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
-        );
-      } else {
-        throw new Error(
-          failed
-            .map(r => r.error?.message)
-            .filter(Boolean)
-            .join(' '),
-        );
+      for await (const blob of container.listBlobsFlat()) {
+        existingFiles.push(blob.name);
       }
     } catch (e) {
-      const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e}`;
+      const errorMessage = `Unable to list file(s) to Azure. ${e}`;
       this.logger.error(errorMessage);
       throw new Error(errorMessage);
     }
+
+    // Then, merge new files into the same folder
+    let absoluteFilesToUpload;
+    try {
+      // Remove the absolute path prefix of the source directory
+      // Path of all files to upload, relative to the root of the source directory
+      // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
+      absoluteFilesToUpload = await getFileTreeRecursively(directory);
+
+      await bulkStorageOperation(
+        async absoluteFilePath => {
+          // const relativeFilePath = path.relative(directory, absoluteFilePath);
+          const relativeFilePath = path.normalize(
+            path.relative(directory, absoluteFilePath),
+          );
+          return await this.storageClient
+            .getContainerClient(this.containerName)
+            .getBlockBlobClient(
+              getCloudPathForLocalPath(entity, relativeFilePath),
+            )
+            .uploadFile(absoluteFilePath);
+        },
+        absoluteFilesToUpload,
+        { concurrencyLimit: BATCH_CONCURRENCY },
+      );
+
+      this.logger.info(
+        `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${absoluteFilesToUpload.length}`,
+      );
+    } catch (e) {
+      const errorMessage = `Unable to upload file(s) to Azure. ${e}`;
+      this.logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+
+    // Last, try to remove the files that were *only* present previously
+    try {
+      const relativeFilesToUpload = absoluteFilesToUpload.map(
+        absoluteFilePath =>
+          getCloudPathForLocalPath(
+            entity,
+            path.relative(directory, absoluteFilePath),
+          ),
+      );
+
+      const staleFiles = getStaleFiles(
+        relativeFilesToUpload,
+        existingFiles,
+        remoteFolder,
+      );
+
+      await bulkStorageOperation(
+        async relativeFilePath => {
+          return await container.deleteBlob(relativeFilePath);
+        },
+        staleFiles,
+        { concurrencyLimit: BATCH_CONCURRENCY },
+      );
+
+      this.logger.info(
+        `Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: ${staleFiles.length}`,
+      );
+    } catch (error) {
+      const errorMessage = `Unable to delete file(s) from AWS S3. ${error}`;
+      this.logger.error(errorMessage);
+    }
   }
 
   private download(containerName: string, blobPath: string): Promise<Buffer> {

packages/techdocs-common/src/stages/publish/helpers.ts

@@ -113,14 +113,21 @@ export const lowerCaseEntityTripletInStoragePath = (
   const lowerKind = kind.toLowerCase();
   const lowerName = name.toLowerCase();
   return [lowerNamespace, lowerKind, lowerName, ...parts].join('/');
-}
+};
 
 // Only returns the files that existed previously and are not present anymore.
 export const getStaleFiles = (
   newFiles: string[],
   oldFiles: string[],
+  remoteFolder?: string,
 ): string[] => {
-  const staleFiles = new Set(oldFiles);
+  let filteredFiles = [...oldFiles];
+  if (remoteFolder) {
+    filteredFiles = filteredFiles.filter(filePath =>
+      filePath.match(remoteFolder),
+    );
+  }
+  const staleFiles = new Set(filteredFiles);
   newFiles.forEach(newFile => {
     staleFiles.delete(newFile);
   });