From 8fa94e3accbaaeee9b82b16977b692373d6cd6d8 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 8 Jul 2021 19:31:46 +0200 Subject: [PATCH] Google Cloud Storage implementation Signed-off-by: Eric Peterson --- .../src/stages/publish/googleStorage.ts | 21 +++++ .../publish/migrations/GoogleMigration.ts | 80 +++++++++++++++++++ .../src/stages/publish/migrations/index.ts | 17 ++++ .../techdocs-backend/src/service/router.ts | 2 +- 4 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 packages/techdocs-common/src/stages/publish/migrations/GoogleMigration.ts create mode 100644 packages/techdocs-common/src/stages/publish/migrations/index.ts diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index bca7b86856..9def24722a 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -24,8 +24,10 @@ import express from 'express'; import JSON5 from 'json5'; import createLimiter from 'p-limit'; import path from 'path'; +import { Readable } from 'stream'; import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { MigrateWriteStream } from './migrations'; import { PublisherBase, PublishRequest, @@ -235,4 +237,23 @@ export class GoogleGCSPublish implements PublisherBase { }); }); } + + migrateDocsCase({ removeOriginal = false, concurrency = 25 }): Promise { + return new Promise((resolve, reject) => { + // Iterate through every file in the root of the publisher. + const allFileMetadata: Readable = this.storageClient + .bucket(this.bucketName) + .getFilesStream(); + const migrateFiles = new MigrateWriteStream( + this.logger, + removeOriginal, + concurrency, + ); + migrateFiles.on('finish', resolve).on('error', reject); + allFileMetadata.pipe(migrateFiles).on('error', error => { + migrateFiles.destroy(); + reject(error); + }); + }); + } } diff --git a/packages/techdocs-common/src/stages/publish/migrations/GoogleMigration.ts b/packages/techdocs-common/src/stages/publish/migrations/GoogleMigration.ts new file mode 100644 index 0000000000..6bac7ace4a --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/migrations/GoogleMigration.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { File } from '@google-cloud/storage'; +import { Writable } from 'stream'; +import { Logger } from 'winston'; + +/** + * Writable stream to handle object copy/move operations. This implementation + * ensures we don't read in files from GCS faster than GCS can copy/move them. + */ +export class MigrateWriteStream extends Writable { + protected logger: Logger; + protected removeOriginal: boolean; + protected maxConcurrency: number; + protected inFlight = 0; + + constructor(logger: Logger, removeOriginal: boolean, concurrency: number) { + super({ objectMode: true }); + this.logger = logger; + this.removeOriginal = removeOriginal; + this.maxConcurrency = concurrency; + } + + _write(file: File, _encoding: BufferEncoding, next: Function) { + let shouldCallNext = true; + const [namespace, kind, name, ...parts] = file.name.split('/'); + const lowerNamespace = namespace.toLowerCase(); + const lowerKind = kind.toLowerCase(); + const lowerName = name.toLowerCase(); + + // If all parts are already lowercase, ignore. + if ( + namespace === lowerNamespace && + kind === lowerKind && + name === lowerName + ) { + next(); + return; + } + + // Allow up to n-many files to be migrated at a time. + this.inFlight++; + if (this.inFlight < this.maxConcurrency) { + next(); + shouldCallNext = false; + } + + // Otherwise, copy or move the file. + const newFile = [lowerNamespace, lowerKind, lowerName, ...parts].join('/'); + // todo: Use file.move instead of file.copy when removeOriginal is true. + const migrate = this.removeOriginal + ? file.copy.bind(file) + : file.copy.bind(file); + this.logger.debug(`Migrating ${file.name}`); + migrate(newFile) + .catch(e => + this.logger.warn(`Unable to migrate ${file.name}: ${e.message}`), + ) + .finally(() => { + this.inFlight--; + if (shouldCallNext) { + next(); + } + }); + } +} diff --git a/packages/techdocs-common/src/stages/publish/migrations/index.ts b/packages/techdocs-common/src/stages/publish/migrations/index.ts new file mode 100644 index 0000000000..f82ab054a4 --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/migrations/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { MigrateWriteStream } from './GoogleMigration'; diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index a4b5d73650..a5a4a3f647 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -256,7 +256,7 @@ export async function createRouter( await publisher.migrateDocsCase({ // removeOriginal: !!req.query?.removeOriginal, concurrency: - parseInt((req.query?.concurrency as string) || '', 10) || undefined, + parseInt((req.query?.concurrency as string) || '', 25) || undefined, }); res.status(200).send('Good job'); } catch (e) {