Google Cloud Storage implementation

Signed-off-by: Eric Peterson <ericpeterson@spotify.com>

packages/techdocs-common/src/stages/publish/googleStorage.ts

@@ -24,8 +24,10 @@ import express from 'express';
 import JSON5 from 'json5';
 import createLimiter from 'p-limit';
 import path from 'path';
+import { Readable } from 'stream';
 import { Logger } from 'winston';
 import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
+import { MigrateWriteStream } from './migrations';
 import {
   PublisherBase,
   PublishRequest,
@@ -235,4 +237,23 @@ export class GoogleGCSPublish implements PublisherBase {
         });
     });
   }
+
+  migrateDocsCase({ removeOriginal = false, concurrency = 25 }): Promise<void> {
+    return new Promise((resolve, reject) => {
+      // Iterate through every file in the root of the publisher.
+      const allFileMetadata: Readable = this.storageClient
+        .bucket(this.bucketName)
+        .getFilesStream();
+      const migrateFiles = new MigrateWriteStream(
+        this.logger,
+        removeOriginal,
+        concurrency,
+      );
+      migrateFiles.on('finish', resolve).on('error', reject);
+      allFileMetadata.pipe(migrateFiles).on('error', error => {
+        migrateFiles.destroy();
+        reject(error);
+      });
+    });
+  }
 }

packages/techdocs-common/src/stages/publish/migrations/GoogleMigration.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { File } from '@google-cloud/storage';
+import { Writable } from 'stream';
+import { Logger } from 'winston';
+
+/**
+ * Writable stream to handle object copy/move operations. This implementation
+ * ensures we don't read in files from GCS faster than GCS can copy/move them.
+ */
+export class MigrateWriteStream extends Writable {
+  protected logger: Logger;
+  protected removeOriginal: boolean;
+  protected maxConcurrency: number;
+  protected inFlight = 0;
+
+  constructor(logger: Logger, removeOriginal: boolean, concurrency: number) {
+    super({ objectMode: true });
+    this.logger = logger;
+    this.removeOriginal = removeOriginal;
+    this.maxConcurrency = concurrency;
+  }
+
+  _write(file: File, _encoding: BufferEncoding, next: Function) {
+    let shouldCallNext = true;
+    const [namespace, kind, name, ...parts] = file.name.split('/');
+    const lowerNamespace = namespace.toLowerCase();
+    const lowerKind = kind.toLowerCase();
+    const lowerName = name.toLowerCase();
+
+    // If all parts are already lowercase, ignore.
+    if (
+      namespace === lowerNamespace &&
+      kind === lowerKind &&
+      name === lowerName
+    ) {
+      next();
+      return;
+    }
+
+    // Allow up to n-many files to be migrated at a time.
+    this.inFlight++;
+    if (this.inFlight < this.maxConcurrency) {
+      next();
+      shouldCallNext = false;
+    }
+
+    // Otherwise, copy or move the file.
+    const newFile = [lowerNamespace, lowerKind, lowerName, ...parts].join('/');
+    // todo: Use file.move instead of file.copy when removeOriginal is true.
+    const migrate = this.removeOriginal
+      ? file.copy.bind(file)
+      : file.copy.bind(file);
+    this.logger.debug(`Migrating ${file.name}`);
+    migrate(newFile)
+      .catch(e =>
+        this.logger.warn(`Unable to migrate ${file.name}: ${e.message}`),
+      )
+      .finally(() => {
+        this.inFlight--;
+        if (shouldCallNext) {
+          next();
+        }
+      });
+  }
+}

packages/techdocs-common/src/stages/publish/migrations/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { MigrateWriteStream } from './GoogleMigration';

plugins/techdocs-backend/src/service/router.ts

@@ -256,7 +256,7 @@ export async function createRouter(
         await publisher.migrateDocsCase({
           // removeOriginal: !!req.query?.removeOriginal,
           concurrency:
-            parseInt((req.query?.concurrency as string) || '', 10) || undefined,
+            parseInt((req.query?.concurrency as string) || '', 25) || undefined,
         });
         res.status(200).send('Good job');
       } catch (e) {