diff --git a/.changeset/techdocs-migration.md b/.changeset/techdocs-migration.md new file mode 100644 index 0000000000..939ace17a5 --- /dev/null +++ b/.changeset/techdocs-migration.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-techdocs': minor +--- + +Added a `migrateDocsCase()` method to TechDocs publishers, along with +implementations for AWS, Azure, and GCS. + +This change is in support of a future update to TechDocs that will allow for +case-insensitive entity triplet URL access to documentation pages which will +require a migration of existing documentation objects in external storage +solutions. + +See [#4367](https://github.com/backstage/backstage/issues/4367) for details. diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 119629915e..159137c879 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -150,6 +150,39 @@ permissions to: - `s3:ListBucket` - To retrieve bucket metadata - `s3:GetObject` - To retrieve files from the bucket +> Note: If you need to migrate documentation objects from an older-style path +> format including case-sensitive entity metadata, you will need to add some +> additional permissions to be able to perform the migration, including: +> +> - `s3:PutBucketAcl` (for copying files, +> [more info here](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html)) +> - `s3:DeleteObject` and `s3:DeleteObjectVersion` (for deleting migrated files, +> [more info here](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html)) +> +> ...And you will need to ensure the permissions apply to the bucket itself, as +> well as all resources under the bucket. See the example policy below. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "TechDocsWithMigration", + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObjectVersion", + "s3:ListBucket", + "s3:DeleteObject", + "s3:PutObjectAcl" + ], + "Resource": ["arn:aws:s3:::your-bucket", "arn:aws:s3:::your-bucket/*"] + } + ] +} +``` + **4a. (Recommended) Setup authentication the AWS way, using environment variables** diff --git a/packages/techdocs-common/api-report.md b/packages/techdocs-common/api-report.md index 8cca624e11..f3cfd29bcd 100644 --- a/packages/techdocs-common/api-report.md +++ b/packages/techdocs-common/api-report.md @@ -194,6 +194,7 @@ export interface PublisherBase { fetchTechDocsMetadata(entityName: EntityName): Promise; getReadiness(): Promise; hasDocsBeenGenerated(entityName: Entity): Promise; + migrateDocsCase?(migrateRequest: MigrateRequest): Promise; publish(request: PublishRequest): Promise; } diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 15fe5a467f..2942e33380 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -16,7 +16,7 @@ import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import aws, { Credentials } from 'aws-sdk'; -import { ManagedUpload } from 'aws-sdk/clients/s3'; +import { ListObjectsV2Output, ManagedUpload } from 'aws-sdk/clients/s3'; import { CredentialsOptions } from 'aws-sdk/lib/credentials'; import express from 'express'; import fs from 'fs-extra'; @@ -25,7 +25,11 @@ import createLimiter from 'p-limit'; import path from 'path'; import { Readable } from 'stream'; import { Logger } from 'winston'; -import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { + getFileTreeRecursively, + getHeadersForFileExtension, + lowerCaseEntityTripletInStoragePath, +} from './helpers'; import { PublisherBase, PublishRequest, @@ -308,4 +312,78 @@ export class AwsS3Publish implements PublisherBase { return Promise.resolve(false); } } + + async migrateDocsCase({ + removeOriginal = false, + concurrency = 25, + }): Promise { + // Iterate through every file in the root of the publisher. + const allObjects = await this.getAllObjectsFromBucket(); + const limiter = createLimiter(concurrency); + await Promise.all( + allObjects.map(f => + limiter(async file => { + let newPath; + try { + newPath = lowerCaseEntityTripletInStoragePath(file); + } catch (e) { + this.logger.warn(e.message); + return; + } + + // If all parts are already lowercase, ignore. + if (file === newPath) { + return; + } + + try { + this.logger.debug(`Migrating ${file}`); + await this.storageClient + .copyObject({ + Bucket: this.bucketName, + CopySource: [this.bucketName, file].join('/'), + Key: newPath, + }) + .promise(); + + if (removeOriginal) { + await this.storageClient + .deleteObject({ + Bucket: this.bucketName, + Key: file, + }) + .promise(); + } + } catch (e) { + this.logger.warn(`Unable to migrate ${file}: ${e.message}`); + } + }, f), + ), + ); + } + + /** + * Returns a list of all object keys from the configured bucket. + */ + protected async getAllObjectsFromBucket(): Promise { + const objects: string[] = []; + let nextContinuation: string | undefined; + let allObjects: ListObjectsV2Output; + + // Iterate through every file in the root of the publisher. + do { + allObjects = await this.storageClient + .listObjectsV2({ + Bucket: this.bucketName, + ContinuationToken: nextContinuation, + }) + .promise(); + objects.push( + ...(allObjects.Contents || []).map(f => f.Key || '').filter(f => !!f), + ); + nextContinuation = allObjects.NextContinuationToken; + } while (nextContinuation); + + return objects; + } } diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index 3ed75f532e..7ee67a0016 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -25,7 +25,11 @@ import JSON5 from 'json5'; import limiterFactory from 'p-limit'; import { default as path, default as platformPath } from 'path'; import { Logger } from 'winston'; -import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { + getFileTreeRecursively, + getHeadersForFileExtension, + lowerCaseEntityTripletInStoragePath, +} from './helpers'; import { PublisherBase, PublishRequest, @@ -289,4 +293,61 @@ export class AzureBlobStoragePublish implements PublisherBase { .getBlockBlobClient(`${entityRootDir}/index.html`) .exists(); } + + protected async renameBlob( + originalName: string, + newName: string, + removeOriginal = false, + ): Promise { + const container = this.storageClient.getContainerClient(this.containerName); + const blob = container.getBlobClient(newName); + const { url } = container.getBlobClient(originalName); + const response = await blob.beginCopyFromURL(url); + await response.pollUntilDone(); + if (removeOriginal) { + await container.deleteBlob(originalName); + } + } + + protected async renameBlobToLowerCase( + originalPath: string, + removeOriginal: boolean, + ) { + let newPath; + try { + newPath = lowerCaseEntityTripletInStoragePath(originalPath); + } catch (e) { + this.logger.warn(e.message); + return; + } + + if (originalPath === newPath) return; + try { + this.logger.debug(`Migrating ${originalPath}`); + await this.renameBlob(originalPath, newPath, removeOriginal); + } catch (e) { + this.logger.warn(`Unable to migrate ${originalPath}: ${e.message}`); + } + } + + async migrateDocsCase({ + removeOriginal = false, + concurrency = 25, + }): Promise { + const promises = []; + const limiter = limiterFactory(concurrency); + const container = this.storageClient.getContainerClient(this.containerName); + + for await (const blob of container.listBlobsFlat()) { + promises.push( + limiter( + this.renameBlobToLowerCase.bind(this), + blob.name, + removeOriginal, + ), + ); + } + + await Promise.all(promises); + } } diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index bca7b86856..9def24722a 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -24,8 +24,10 @@ import express from 'express'; import JSON5 from 'json5'; import createLimiter from 'p-limit'; import path from 'path'; +import { Readable } from 'stream'; import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { MigrateWriteStream } from './migrations'; import { PublisherBase, PublishRequest, @@ -235,4 +237,23 @@ export class GoogleGCSPublish implements PublisherBase { }); }); } + + migrateDocsCase({ removeOriginal = false, concurrency = 25 }): Promise { + return new Promise((resolve, reject) => { + // Iterate through every file in the root of the publisher. + const allFileMetadata: Readable = this.storageClient + .bucket(this.bucketName) + .getFilesStream(); + const migrateFiles = new MigrateWriteStream( + this.logger, + removeOriginal, + concurrency, + ); + migrateFiles.on('finish', resolve).on('error', reject); + allFileMetadata.pipe(migrateFiles).on('error', error => { + migrateFiles.destroy(); + reject(error); + }); + }); + } } diff --git a/packages/techdocs-common/src/stages/publish/helpers.test.ts b/packages/techdocs-common/src/stages/publish/helpers.test.ts index cfb9c7930d..27a56cd06d 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.test.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.test.ts @@ -16,7 +16,11 @@ import mockFs from 'mock-fs'; import * as os from 'os'; import * as path from 'path'; -import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { + getFileTreeRecursively, + getHeadersForFileExtension, + lowerCaseEntityTripletInStoragePath, +} from './helpers'; describe('getHeadersForFileExtension', () => { const correctMapOfExtensions = [ @@ -73,3 +77,25 @@ describe('getFileTreeRecursively', () => { expect(fileList).toContain(path.resolve(root, 'subDirA/file2')); }); }); + +describe('lowerCaseEntityTripletInStoragePath', () => { + it('returns lower-cased entity triplet path', () => { + const originalPath = 'default/Component/backstage/index.html'; + const actualPath = lowerCaseEntityTripletInStoragePath(originalPath); + expect(actualPath).toBe('default/component/backstage/index.html'); + }); + + it('does not lowercase beyond the triplet', () => { + const originalPath = 'default/Component/backstage/assets/IMAGE.png'; + const actualPath = lowerCaseEntityTripletInStoragePath(originalPath); + expect(actualPath).toBe('default/component/backstage/assets/IMAGE.png'); + }); + + it('throws error when there is no triplet', () => { + const originalPath = '/default/component/IMAGE.png'; + const error = `Encountered file unmanaged by TechDocs ${originalPath}. Skipping.`; + expect(() => + lowerCaseEntityTripletInStoragePath(originalPath), + ).toThrowError(error); + }); +}); diff --git a/packages/techdocs-common/src/stages/publish/helpers.ts b/packages/techdocs-common/src/stages/publish/helpers.ts index eac5ba05c8..ed681eb8ba 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.ts @@ -82,3 +82,32 @@ export const getFileTreeRecursively = async ( }); return fileList; }; + +/** + * Returns the version of an object's storage path where the first three parts + * of the path (the entity triplet of namespace, kind, and name) are + * lower-cased. + * + * Path must not include a starting slash. + * + * @example + * lowerCaseEntityTripletInStoragePath('default/Component/backstage') + * // return default/component/backstage + */ +export const lowerCaseEntityTripletInStoragePath = ( + originalPath: string, +): string => { + const trimmedPath = + originalPath[0] === '/' ? originalPath.substring(1) : originalPath; + const matches = trimmedPath.match(/\//g) || []; + if (matches.length <= 2) { + throw new Error( + `Encountered file unmanaged by TechDocs ${originalPath}. Skipping.`, + ); + } + const [namespace, kind, name, ...parts] = originalPath.split('/'); + const lowerNamespace = namespace.toLowerCase(); + const lowerKind = kind.toLowerCase(); + const lowerName = name.toLowerCase(); + return [lowerNamespace, lowerKind, lowerName, ...parts].join('/'); +}; diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 3ad0ed65ad..3414d273dd 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -22,6 +22,7 @@ import { Config } from '@backstage/config'; import express from 'express'; import fs from 'fs-extra'; import os from 'os'; +import createLimiter from 'p-limit'; import path from 'path'; import { Logger } from 'winston'; import { @@ -31,7 +32,11 @@ import { ReadinessResponse, TechDocsMetadata, } from './types'; -import { getHeadersForFileExtension } from './helpers'; +import { + getFileTreeRecursively, + getHeadersForFileExtension, + lowerCaseEntityTripletInStoragePath, +} from './helpers'; // TODO: Use a more persistent storage than node_modules or /tmp directory. // Make it configurable with techdocs.publisher.local.publishDirectory @@ -164,4 +169,45 @@ export class LocalPublish implements PublisherBase { return false; } } + + /** + * This code will never run in practice. It is merely here to illustrate how + * to implement this method for other storage providers. + */ + async migrateDocsCase({ + removeOriginal = false, + concurrency = 25, + }): Promise { + // Iterate through every file in the root of the publisher. + const files = await getFileTreeRecursively(staticDocsDir); + const limit = createLimiter(concurrency); + + await Promise.all( + files.map(f => + limit(async file => { + const relativeFile = file.replace(`${staticDocsDir}${path.sep}`, ''); + const newFile = lowerCaseEntityTripletInStoragePath(relativeFile); + + // If all parts are already lowercase, ignore. + if (relativeFile === newFile) { + return; + } + + // Otherwise, copy or move the file. + await new Promise(resolve => { + const migrate = removeOriginal ? fs.move : fs.copyFile; + this.logger.debug(`Migrating ${relativeFile}`); + migrate(file, newFile, err => { + if (err) { + this.logger.warn( + `Unable to migrate ${relativeFile}: ${err.message}`, + ); + } + resolve(); + }); + }); + }, f), + ), + ); + } } diff --git a/packages/techdocs-common/src/stages/publish/migrations/GoogleMigration.ts b/packages/techdocs-common/src/stages/publish/migrations/GoogleMigration.ts new file mode 100644 index 0000000000..cc35b351ea --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/migrations/GoogleMigration.ts @@ -0,0 +1,79 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { File } from '@google-cloud/storage'; +import { Writable } from 'stream'; +import { Logger } from 'winston'; +import { lowerCaseEntityTripletInStoragePath } from '../helpers'; + +/** + * Writable stream to handle object copy/move operations. This implementation + * ensures we don't read in files from GCS faster than GCS can copy/move them. + */ +export class MigrateWriteStream extends Writable { + protected logger: Logger; + protected removeOriginal: boolean; + protected maxConcurrency: number; + protected inFlight = 0; + + constructor(logger: Logger, removeOriginal: boolean, concurrency: number) { + super({ objectMode: true }); + this.logger = logger; + this.removeOriginal = removeOriginal; + this.maxConcurrency = concurrency; + } + + _write(file: File, _encoding: BufferEncoding, next: Function) { + let shouldCallNext = true; + let newFile; + try { + newFile = lowerCaseEntityTripletInStoragePath(file.name); + } catch (e) { + this.logger.warn(e.message); + next(); + return; + } + + // If all parts are already lowercase, ignore. + if (newFile === file.name) { + next(); + return; + } + + // Allow up to n-many files to be migrated at a time. + this.inFlight++; + if (this.inFlight < this.maxConcurrency) { + next(); + shouldCallNext = false; + } + + // Otherwise, copy or move the file. + const migrate = this.removeOriginal + ? file.move.bind(file) + : file.copy.bind(file); + this.logger.debug(`Migrating ${file.name}`); + migrate(newFile) + .catch(e => + this.logger.warn(`Unable to migrate ${file.name}: ${e.message}`), + ) + .finally(() => { + this.inFlight--; + if (shouldCallNext) { + next(); + } + }); + } +} diff --git a/packages/techdocs-common/src/stages/publish/migrations/index.ts b/packages/techdocs-common/src/stages/publish/migrations/index.ts new file mode 100644 index 0000000000..f82ab054a4 --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/migrations/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { MigrateWriteStream } from './GoogleMigration'; diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index a7f6ff7a45..229c853427 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -55,6 +55,19 @@ export type TechDocsMetadata = { etag: string; }; +export type MigrateRequest = { + /** + * Whether or not to remove the source file. Defaults to false (acting like a + * copy instead of a move). + */ + removeOriginal?: boolean; + + /** + * Maximum number of files/objects to migrate at once. Defaults to 25. + */ + concurrency?: number; +}; + /** * Base class for a TechDocs publisher (e.g. Local, Google GCS Bucket, AWS S3, etc.) * The publisher handles publishing of the generated static files after the prepare and generate steps of TechDocs. @@ -92,4 +105,14 @@ export interface PublisherBase { * Check if the index.html is present for the Entity at the Storage location. */ hasDocsBeenGenerated(entityName: Entity): Promise; + + /** + * Migrates documentation objects with case sensitive entity triplets to + * lowercase entity triplets. This was (will be) a change introduced in + * techdocs-cli v{0.x.y} and techdocs-backend v{0.x.y}. + * + * Implementation of this method is unnecessary in publishers introduced + * after v{0.x.y} of techdocs-common. + */ + migrateDocsCase?(migrateRequest: MigrateRequest): Promise; }