Merge pull request #6405 from backstage/mob/docs-storage-case-migration

[TechDocs] Provide a migration path from case-sensitive to lower-case entity triplets in docs storage

.changeset/techdocs-migration.md

@@ -0,0 +1,13 @@
+---
+'@backstage/plugin-techdocs': minor
+---
+
+Added a `migrateDocsCase()` method to TechDocs publishers, along with
+implementations for AWS, Azure, and GCS.
+
+This change is in support of a future update to TechDocs that will allow for
+case-insensitive entity triplet URL access to documentation pages which will
+require a migration of existing documentation objects in external storage
+solutions.
+
+See [#4367](https://github.com/backstage/backstage/issues/4367) for details.

docs/features/techdocs/using-cloud-storage.md

@@ -150,6 +150,39 @@ permissions to:
 - `s3:ListBucket` - To retrieve bucket metadata
 - `s3:GetObject` - To retrieve files from the bucket
 
+> Note: If you need to migrate documentation objects from an older-style path
+> format including case-sensitive entity metadata, you will need to add some
+> additional permissions to be able to perform the migration, including:
+>
+> - `s3:PutBucketAcl` (for copying files,
+>   [more info here](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html))
+> - `s3:DeleteObject` and `s3:DeleteObjectVersion` (for deleting migrated files,
+>   [more info here](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html))
+>
+> ...And you will need to ensure the permissions apply to the bucket itself, as
+> well as all resources under the bucket. See the example policy below.
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "TechDocsWithMigration",
+      "Effect": "Allow",
+      "Action": [
+        "s3:PutObject",
+        "s3:GetObject",
+        "s3:DeleteObjectVersion",
+        "s3:ListBucket",
+        "s3:DeleteObject",
+        "s3:PutObjectAcl"
+      ],
+      "Resource": ["arn:aws:s3:::your-bucket", "arn:aws:s3:::your-bucket/*"]
+    }
+  ]
+}
+```
+
 **4a. (Recommended) Setup authentication the AWS way, using environment
 variables**
 

packages/techdocs-common/api-report.md

@@ -194,6 +194,7 @@ export interface PublisherBase {
   fetchTechDocsMetadata(entityName: EntityName): Promise<TechDocsMetadata>;
   getReadiness(): Promise<ReadinessResponse>;
   hasDocsBeenGenerated(entityName: Entity): Promise<boolean>;
+  migrateDocsCase?(migrateRequest: MigrateRequest): Promise<void>;
   publish(request: PublishRequest): Promise<PublishResponse>;
 }
 

packages/techdocs-common/src/stages/publish/awsS3.ts

@@ -16,7 +16,7 @@
 import { Entity, EntityName } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import aws, { Credentials } from 'aws-sdk';
-import { ManagedUpload } from 'aws-sdk/clients/s3';
+import { ListObjectsV2Output, ManagedUpload } from 'aws-sdk/clients/s3';
 import { CredentialsOptions } from 'aws-sdk/lib/credentials';
 import express from 'express';
 import fs from 'fs-extra';
@@ -25,7 +25,11 @@ import createLimiter from 'p-limit';
 import path from 'path';
 import { Readable } from 'stream';
 import { Logger } from 'winston';
-import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
+import {
+  getFileTreeRecursively,
+  getHeadersForFileExtension,
+  lowerCaseEntityTripletInStoragePath,
+} from './helpers';
 import {
   PublisherBase,
   PublishRequest,
@@ -308,4 +312,78 @@ export class AwsS3Publish implements PublisherBase {
       return Promise.resolve(false);
     }
   }
+
+  async migrateDocsCase({
+    removeOriginal = false,
+    concurrency = 25,
+  }): Promise<void> {
+    // Iterate through every file in the root of the publisher.
+    const allObjects = await this.getAllObjectsFromBucket();
+    const limiter = createLimiter(concurrency);
+    await Promise.all(
+      allObjects.map(f =>
+        limiter(async file => {
+          let newPath;
+          try {
+            newPath = lowerCaseEntityTripletInStoragePath(file);
+          } catch (e) {
+            this.logger.warn(e.message);
+            return;
+          }
+
+          // If all parts are already lowercase, ignore.
+          if (file === newPath) {
+            return;
+          }
+
+          try {
+            this.logger.debug(`Migrating ${file}`);
+            await this.storageClient
+              .copyObject({
+                Bucket: this.bucketName,
+                CopySource: [this.bucketName, file].join('/'),
+                Key: newPath,
+              })
+              .promise();
+
+            if (removeOriginal) {
+              await this.storageClient
+                .deleteObject({
+                  Bucket: this.bucketName,
+                  Key: file,
+                })
+                .promise();
+            }
+          } catch (e) {
+            this.logger.warn(`Unable to migrate ${file}: ${e.message}`);
+          }
+        }, f),
+      ),
+    );
+  }
+
+  /**
+   * Returns a list of all object keys from the configured bucket.
+   */
+  protected async getAllObjectsFromBucket(): Promise<string[]> {
+    const objects: string[] = [];
+    let nextContinuation: string | undefined;
+    let allObjects: ListObjectsV2Output;
+
+    // Iterate through every file in the root of the publisher.
+    do {
+      allObjects = await this.storageClient
+        .listObjectsV2({
+          Bucket: this.bucketName,
+          ContinuationToken: nextContinuation,
+        })
+        .promise();
+      objects.push(
+        ...(allObjects.Contents || []).map(f => f.Key || '').filter(f => !!f),
+      );
+      nextContinuation = allObjects.NextContinuationToken;
+    } while (nextContinuation);
+
+    return objects;
+  }
 }

packages/techdocs-common/src/stages/publish/azureBlobStorage.ts

@@ -25,7 +25,11 @@ import JSON5 from 'json5';
 import limiterFactory from 'p-limit';
 import { default as path, default as platformPath } from 'path';
 import { Logger } from 'winston';
-import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
+import {
+  getFileTreeRecursively,
+  getHeadersForFileExtension,
+  lowerCaseEntityTripletInStoragePath,
+} from './helpers';
 import {
   PublisherBase,
   PublishRequest,
@@ -289,4 +293,61 @@ export class AzureBlobStoragePublish implements PublisherBase {
       .getBlockBlobClient(`${entityRootDir}/index.html`)
       .exists();
   }
+
+  protected async renameBlob(
+    originalName: string,
+    newName: string,
+    removeOriginal = false,
+  ): Promise<void> {
+    const container = this.storageClient.getContainerClient(this.containerName);
+    const blob = container.getBlobClient(newName);
+    const { url } = container.getBlobClient(originalName);
+    const response = await blob.beginCopyFromURL(url);
+    await response.pollUntilDone();
+    if (removeOriginal) {
+      await container.deleteBlob(originalName);
+    }
+  }
+
+  protected async renameBlobToLowerCase(
+    originalPath: string,
+    removeOriginal: boolean,
+  ) {
+    let newPath;
+    try {
+      newPath = lowerCaseEntityTripletInStoragePath(originalPath);
+    } catch (e) {
+      this.logger.warn(e.message);
+      return;
+    }
+
+    if (originalPath === newPath) return;
+    try {
+      this.logger.debug(`Migrating ${originalPath}`);
+      await this.renameBlob(originalPath, newPath, removeOriginal);
+    } catch (e) {
+      this.logger.warn(`Unable to migrate ${originalPath}: ${e.message}`);
+    }
+  }
+
+  async migrateDocsCase({
+    removeOriginal = false,
+    concurrency = 25,
+  }): Promise<void> {
+    const promises = [];
+    const limiter = limiterFactory(concurrency);
+    const container = this.storageClient.getContainerClient(this.containerName);
+
+    for await (const blob of container.listBlobsFlat()) {
+      promises.push(
+        limiter(
+          this.renameBlobToLowerCase.bind(this),
+          blob.name,
+          removeOriginal,
+        ),
+      );
+    }
+
+    await Promise.all(promises);
+  }
 }

packages/techdocs-common/src/stages/publish/googleStorage.ts

@@ -24,8 +24,10 @@ import express from 'express';
 import JSON5 from 'json5';
 import createLimiter from 'p-limit';
 import path from 'path';
+import { Readable } from 'stream';
 import { Logger } from 'winston';
 import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
+import { MigrateWriteStream } from './migrations';
 import {
   PublisherBase,
   PublishRequest,
@@ -235,4 +237,23 @@ export class GoogleGCSPublish implements PublisherBase {
         });
     });
   }
+
+  migrateDocsCase({ removeOriginal = false, concurrency = 25 }): Promise<void> {
+    return new Promise((resolve, reject) => {
+      // Iterate through every file in the root of the publisher.
+      const allFileMetadata: Readable = this.storageClient
+        .bucket(this.bucketName)
+        .getFilesStream();
+      const migrateFiles = new MigrateWriteStream(
+        this.logger,
+        removeOriginal,
+        concurrency,
+      );
+      migrateFiles.on('finish', resolve).on('error', reject);
+      allFileMetadata.pipe(migrateFiles).on('error', error => {
+        migrateFiles.destroy();
+        reject(error);
+      });
+    });
+  }
 }

packages/techdocs-common/src/stages/publish/helpers.test.ts

@@ -16,7 +16,11 @@
 import mockFs from 'mock-fs';
 import * as os from 'os';
 import * as path from 'path';
-import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
+import {
+  getFileTreeRecursively,
+  getHeadersForFileExtension,
+  lowerCaseEntityTripletInStoragePath,
+} from './helpers';
 
 describe('getHeadersForFileExtension', () => {
   const correctMapOfExtensions = [
@@ -73,3 +77,25 @@ describe('getFileTreeRecursively', () => {
     expect(fileList).toContain(path.resolve(root, 'subDirA/file2'));
   });
 });
+
+describe('lowerCaseEntityTripletInStoragePath', () => {
+  it('returns lower-cased entity triplet path', () => {
+    const originalPath = 'default/Component/backstage/index.html';
+    const actualPath = lowerCaseEntityTripletInStoragePath(originalPath);
+    expect(actualPath).toBe('default/component/backstage/index.html');
+  });
+
+  it('does not lowercase beyond the triplet', () => {
+    const originalPath = 'default/Component/backstage/assets/IMAGE.png';
+    const actualPath = lowerCaseEntityTripletInStoragePath(originalPath);
+    expect(actualPath).toBe('default/component/backstage/assets/IMAGE.png');
+  });
+
+  it('throws error when there is no triplet', () => {
+    const originalPath = '/default/component/IMAGE.png';
+    const error = `Encountered file unmanaged by TechDocs ${originalPath}. Skipping.`;
+    expect(() =>
+      lowerCaseEntityTripletInStoragePath(originalPath),
+    ).toThrowError(error);
+  });
+});

packages/techdocs-common/src/stages/publish/helpers.ts

@@ -82,3 +82,32 @@ export const getFileTreeRecursively = async (
   });
   return fileList;
 };
+
+/**
+ * Returns the version of an object's storage path where the first three parts
+ * of the path (the entity triplet of namespace, kind, and name) are
+ * lower-cased.
+ *
+ * Path must not include a starting slash.
+ *
+ * @example
+ * lowerCaseEntityTripletInStoragePath('default/Component/backstage')
+ * // return default/component/backstage
+ */
+export const lowerCaseEntityTripletInStoragePath = (
+  originalPath: string,
+): string => {
+  const trimmedPath =
+    originalPath[0] === '/' ? originalPath.substring(1) : originalPath;
+  const matches = trimmedPath.match(/\//g) || [];
+  if (matches.length <= 2) {
+    throw new Error(
+      `Encountered file unmanaged by TechDocs ${originalPath}. Skipping.`,
+    );
+  }
+  const [namespace, kind, name, ...parts] = originalPath.split('/');
+  const lowerNamespace = namespace.toLowerCase();
+  const lowerKind = kind.toLowerCase();
+  const lowerName = name.toLowerCase();
+  return [lowerNamespace, lowerKind, lowerName, ...parts].join('/');
+};

packages/techdocs-common/src/stages/publish/local.ts

@@ -22,6 +22,7 @@ import { Config } from '@backstage/config';
 import express from 'express';
 import fs from 'fs-extra';
 import os from 'os';
+import createLimiter from 'p-limit';
 import path from 'path';
 import { Logger } from 'winston';
 import {
@@ -31,7 +32,11 @@ import {
   ReadinessResponse,
   TechDocsMetadata,
 } from './types';
-import { getHeadersForFileExtension } from './helpers';
+import {
+  getFileTreeRecursively,
+  getHeadersForFileExtension,
+  lowerCaseEntityTripletInStoragePath,
+} from './helpers';
 
 // TODO: Use a more persistent storage than node_modules or /tmp directory.
 // Make it configurable with techdocs.publisher.local.publishDirectory
@@ -164,4 +169,45 @@ export class LocalPublish implements PublisherBase {
       return false;
     }
   }
+
+  /**
+   * This code will never run in practice. It is merely here to illustrate how
+   * to implement this method for other storage providers.
+   */
+  async migrateDocsCase({
+    removeOriginal = false,
+    concurrency = 25,
+  }): Promise<void> {
+    // Iterate through every file in the root of the publisher.
+    const files = await getFileTreeRecursively(staticDocsDir);
+    const limit = createLimiter(concurrency);
+
+    await Promise.all(
+      files.map(f =>
+        limit(async file => {
+          const relativeFile = file.replace(`${staticDocsDir}${path.sep}`, '');
+          const newFile = lowerCaseEntityTripletInStoragePath(relativeFile);
+
+          // If all parts are already lowercase, ignore.
+          if (relativeFile === newFile) {
+            return;
+          }
+
+          // Otherwise, copy or move the file.
+          await new Promise<void>(resolve => {
+            const migrate = removeOriginal ? fs.move : fs.copyFile;
+            this.logger.debug(`Migrating ${relativeFile}`);
+            migrate(file, newFile, err => {
+              if (err) {
+                this.logger.warn(
+                  `Unable to migrate ${relativeFile}: ${err.message}`,
+                );
+              }
+              resolve();
+            });
+          });
+        }, f),
+      ),
+    );
+  }
 }

packages/techdocs-common/src/stages/publish/migrations/GoogleMigration.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { File } from '@google-cloud/storage';
+import { Writable } from 'stream';
+import { Logger } from 'winston';
+import { lowerCaseEntityTripletInStoragePath } from '../helpers';
+
+/**
+ * Writable stream to handle object copy/move operations. This implementation
+ * ensures we don't read in files from GCS faster than GCS can copy/move them.
+ */
+export class MigrateWriteStream extends Writable {
+  protected logger: Logger;
+  protected removeOriginal: boolean;
+  protected maxConcurrency: number;
+  protected inFlight = 0;
+
+  constructor(logger: Logger, removeOriginal: boolean, concurrency: number) {
+    super({ objectMode: true });
+    this.logger = logger;
+    this.removeOriginal = removeOriginal;
+    this.maxConcurrency = concurrency;
+  }
+
+  _write(file: File, _encoding: BufferEncoding, next: Function) {
+    let shouldCallNext = true;
+    let newFile;
+    try {
+      newFile = lowerCaseEntityTripletInStoragePath(file.name);
+    } catch (e) {
+      this.logger.warn(e.message);
+      next();
+      return;
+    }
+
+    // If all parts are already lowercase, ignore.
+    if (newFile === file.name) {
+      next();
+      return;
+    }
+
+    // Allow up to n-many files to be migrated at a time.
+    this.inFlight++;
+    if (this.inFlight < this.maxConcurrency) {
+      next();
+      shouldCallNext = false;
+    }
+
+    // Otherwise, copy or move the file.
+    const migrate = this.removeOriginal
+      ? file.move.bind(file)
+      : file.copy.bind(file);
+    this.logger.debug(`Migrating ${file.name}`);
+    migrate(newFile)
+      .catch(e =>
+        this.logger.warn(`Unable to migrate ${file.name}: ${e.message}`),
+      )
+      .finally(() => {
+        this.inFlight--;
+        if (shouldCallNext) {
+          next();
+        }
+      });
+  }
+}

packages/techdocs-common/src/stages/publish/migrations/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { MigrateWriteStream } from './GoogleMigration';

packages/techdocs-common/src/stages/publish/types.ts

@@ -55,6 +55,19 @@ export type TechDocsMetadata = {
   etag: string;
 };
 
+export type MigrateRequest = {
+  /**
+   * Whether or not to remove the source file. Defaults to false (acting like a
+   * copy instead of a move).
+   */
+  removeOriginal?: boolean;
+
+  /**
+   * Maximum number of files/objects to migrate at once. Defaults to 25.
+   */
+  concurrency?: number;
+};
+
 /**
  * Base class for a TechDocs publisher (e.g. Local, Google GCS Bucket, AWS S3, etc.)
  * The publisher handles publishing of the generated static files after the prepare and generate steps of TechDocs.
@@ -92,4 +105,14 @@ export interface PublisherBase {
    * Check if the index.html is present for the Entity at the Storage location.
    */
   hasDocsBeenGenerated(entityName: Entity): Promise<boolean>;
+
+  /**
+   * Migrates documentation objects with case sensitive entity triplets to
+   * lowercase entity triplets. This was (will be) a change introduced in
+   * techdocs-cli v{0.x.y} and techdocs-backend v{0.x.y}.
+   *
+   * Implementation of this method is unnecessary in publishers introduced
+   * after v{0.x.y} of techdocs-common.
+   */
+  migrateDocsCase?(migrateRequest: MigrateRequest): Promise<void>;
 }