Merge pull request #7554 from cmpadden/techdocs-storage-subfolder

Support optional bucketRootPath in GCS and S3 techdocs publishers
2021-10-25 15:34:10 +02:00
parent 96a2a23167 e571311f8a
commit d066399739
8 changed files with 380 additions and 12 deletions
@@ -43,9 +43,11 @@ const loggerErrorSpy = jest.spyOn(logger, 'error');

 const createPublisherFromConfig = ({
  bucketName = 'bucketName',
+  bucketRootPath = '/',
  legacyUseCaseSensitiveTripletPaths = false,
 }: {
  bucketName?: string;
+  bucketRootPath?: string;
  legacyUseCaseSensitiveTripletPaths?: boolean;
 } = {}) => {
  const mockConfig = new ConfigReader({
@@ -59,6 +61,7 @@ const createPublisherFromConfig = ({
            secretAccessKey: 'secretAccessKey',
          },
          bucketName,
+          bucketRootPath,
        },
      },
      legacyUseCaseSensitiveTripletPaths,
@@ -153,6 +156,21 @@ describe('AwsS3Publish', () => {
      expect(await publisher.publish({ entity, directory })).toBeUndefined();
    });

+    it('should publish a directory when root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+    });
+
+    it('should publish a directory when root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+    });
+
    it('should fail to publish a directory', async () => {
      const wrongPathToGeneratedDirectory = path.join(
        rootDir,
@@ -214,6 +232,23 @@ describe('AwsS3Publish', () => {
      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
    });

+    it('should return true if docs has been generated if root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should return true if docs has been generated if root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
    it('should return false if docs has not been generated', async () => {
      const publisher = createPublisherFromConfig();
      expect(
@@ -247,6 +282,27 @@ describe('AwsS3Publish', () => {
      );
    });

+    it('should return tech docs metadata even if root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata if root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
    it('should return tech docs metadata when json encoded with single quotes', async () => {
      const techdocsMetadataPath = path.join(
        directory,
@@ -335,6 +391,47 @@ describe('AwsS3Publish', () => {
      expect(jsResponse.text).toEqual('found it too');
    });

+    it('should pass expected object path to bucket if root path is specified', async () => {
+      const rootPath = 'backstage-data/techdocs';
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: rootPath,
+      });
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+
+      const pngResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass expected object path to bucket if root path is specified and legacy case is enabled', async () => {
+      const rootPath = 'backstage-data/techdocs';
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: rootPath,
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+
+      const pngResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
    it('should pass text/plain content-type for html', async () => {
      const htmlResponse = await request(app).get(
        `/${entityTripletPath}/html/unsafe.html`,
@@ -34,6 +34,7 @@ import {
  getStaleFiles,
  lowerCaseEntityTriplet,
  lowerCaseEntityTripletInStoragePath,
+  normalizeExternalStorageRootPath,
 } from './helpers';
 import {
  PublisherBase,
@@ -67,6 +68,10 @@ export class AwsS3Publish implements PublisherBase {
      );
    }

+    const bucketRootPath = normalizeExternalStorageRootPath(
+      config.getOptionalString('techdocs.publisher.awsS3.bucketRootPath') || '',
+    );
+
    // Credentials is an optional config. If missing, the default ways of authenticating AWS SDK V2 will be used.
    // 1. AWS environment variables
    // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html
@@ -112,6 +117,7 @@ export class AwsS3Publish implements PublisherBase {
      bucketName,
      legacyPathCasing,
      logger,
+      bucketRootPath,
    );
  }

@@ -151,11 +157,13 @@ export class AwsS3Publish implements PublisherBase {
    private readonly bucketName: string,
    private readonly legacyPathCasing: boolean,
    private readonly logger: Logger,
+    private readonly bucketRootPath: string,
  ) {
    this.storageClient = storageClient;
    this.bucketName = bucketName;
    this.legacyPathCasing = legacyPathCasing;
    this.logger = logger;
+    this.bucketRootPath = bucketRootPath;
  }

  /**
@@ -193,6 +201,8 @@ export class AwsS3Publish implements PublisherBase {
   */
  async publish({ entity, directory }: PublishRequest): Promise<void> {
    const useLegacyPathCasing = this.legacyPathCasing;
+    const bucketRootPath = this.bucketRootPath;
+
    // First, try to retrieve a list of all individual files currently existing
    let existingFiles: string[] = [];
    try {
@@ -200,6 +210,7 @@ export class AwsS3Publish implements PublisherBase {
        entity,
        undefined,
        useLegacyPathCasing,
+        bucketRootPath,
      );
      existingFiles = await this.getAllObjectsFromBucket({
        prefix: remoteFolder,
@@ -230,6 +241,7 @@ export class AwsS3Publish implements PublisherBase {
              entity,
              relativeFilePath,
              useLegacyPathCasing,
+              bucketRootPath,
            ),
            Body: fileStream,
          };
@@ -257,6 +269,7 @@ export class AwsS3Publish implements PublisherBase {
            entity,
            path.relative(directory, absoluteFilePath),
            useLegacyPathCasing,
+            bucketRootPath,
          ),
      );
      const staleFiles = getStaleFiles(relativeFilesToUpload, existingFiles);
@@ -289,10 +302,12 @@ export class AwsS3Publish implements PublisherBase {
    try {
      return await new Promise<TechDocsMetadata>(async (resolve, reject) => {
        const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
-        const entityRootDir = this.legacyPathCasing
+        const entityDir = this.legacyPathCasing
          ? entityTriplet
          : lowerCaseEntityTriplet(entityTriplet);

+        const entityRootDir = path.join(this.bucketRootPath, entityDir);
+
        const stream = this.storageClient
          .getObject({
            Bucket: this.bucketName,
@@ -332,10 +347,17 @@ export class AwsS3Publish implements PublisherBase {
      // Decode and trim the leading forward slash
      const decodedUri = decodeURI(req.path.replace(/^\//, ''));

+      // Root path is removed from the Uri so that legacy casing can be applied
+      // to the entity triplet without manipulating the root path
+      const decodedUriNoRoot = path.relative(this.bucketRootPath, decodedUri);
+
      // filePath example - /default/component/documented-component/index.html
-      const filePath = this.legacyPathCasing
-        ? decodedUri
-        : lowerCaseEntityTripletInStoragePath(decodedUri);
+      const filePathNoRoot = this.legacyPathCasing
+        ? decodedUriNoRoot
+        : lowerCaseEntityTripletInStoragePath(decodedUriNoRoot);
+
+      // Re-prepend the root path to the relative file path
+      const filePath = path.join(this.bucketRootPath, filePathNoRoot);

      // Files with different extensions (CSS, HTML) need to be served with different headers
      const fileExtension = path.extname(filePath);
@@ -370,10 +392,12 @@ export class AwsS3Publish implements PublisherBase {
  async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
    try {
      const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
-      const entityRootDir = this.legacyPathCasing
+      const entityDir = this.legacyPathCasing
        ? entityTriplet
        : lowerCaseEntityTriplet(entityTriplet);

+      const entityRootDir = path.join(this.bucketRootPath, entityDir);
+
      await this.storageClient
        .headObject({
          Bucket: this.bucketName,
@@ -43,9 +43,11 @@ jest.spyOn(logger, 'error').mockReturnValue(logger);

 const createPublisherFromConfig = ({
  bucketName = 'bucketName',
+  bucketRootPath = '/',
  legacyUseCaseSensitiveTripletPaths = false,
 }: {
  bucketName?: string;
+  bucketRootPath?: string;
  legacyUseCaseSensitiveTripletPaths?: boolean;
 } = {}) => {
  const config = new ConfigReader({
@@ -56,6 +58,7 @@ const createPublisherFromConfig = ({
        googleGcs: {
          credentials: '{}',
          bucketName,
+          bucketRootPath,
        },
      },
      legacyUseCaseSensitiveTripletPaths,
@@ -149,6 +152,21 @@ describe('GoogleGCSPublish', () => {
      expect(await publisher.publish({ entity, directory })).toBeUndefined();
    });

+    it('should publish a directory when root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+    });
+
+    it('should publish a directory when root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      expect(await publisher.publish({ entity, directory })).toBeUndefined();
+    });
+
    it('should fail to publish a directory', async () => {
      const wrongPathToGeneratedDirectory = path.join(
        rootDir,
@@ -212,6 +230,23 @@ describe('GoogleGCSPublish', () => {
      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
    });

+    it('should return true if docs has been generated if root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should return true if docs has been generated if root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
    it('should return false if docs has not been generated', async () => {
      const publisher = createPublisherFromConfig();
      expect(
@@ -245,6 +280,27 @@ describe('GoogleGCSPublish', () => {
      );
    });

+    it('should return tech docs metadata even if root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata if root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
    it('should return tech docs metadata when json encoded with single quotes', async () => {
      const techdocsMetadataPath = path.join(
        directory,
@@ -291,6 +347,7 @@ describe('GoogleGCSPublish', () => {

  describe('docsRouter', () => {
    const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+    // const entityTripletPath =

    let app: Express.Application;

@@ -334,6 +391,47 @@ describe('GoogleGCSPublish', () => {
      expect(jsResponse.text).toEqual('found it too');
    });

+    it('should pass expected object path to bucket if root path is specified', async () => {
+      const rootPath = 'backstage-data/techdocs';
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: rootPath,
+      });
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+
+      const pngResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass expected object path to bucket if root path is specified and legacy case is enabled', async () => {
+      const rootPath = 'backstage-data/techdocs';
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: rootPath,
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+
+      const pngResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
    it('should pass text/plain content-type for html', async () => {
      const htmlResponse = await request(app).get(
        `/${entityTripletPath}/html/unsafe.html`,
@@ -30,6 +30,7 @@ import {
  bulkStorageOperation,
  getCloudPathForLocalPath,
  getStaleFiles,
+  normalizeExternalStorageRootPath,
 } from './helpers';
 import { MigrateWriteStream } from './migrations';
 import {
@@ -51,6 +52,11 @@ export class GoogleGCSPublish implements PublisherBase {
      );
    }

+    const bucketRootPath = normalizeExternalStorageRootPath(
+      config.getOptionalString('techdocs.publisher.googleGcs.bucketRootPath') ||
+        '',
+    );
+
    // Credentials is an optional config. If missing, default GCS environment variables will be used.
    // Read more here https://cloud.google.com/docs/authentication/production
    const credentials = config.getOptionalString(
@@ -83,6 +89,7 @@ export class GoogleGCSPublish implements PublisherBase {
      bucketName,
      legacyPathCasing,
      logger,
+      bucketRootPath,
    );
  }

@@ -91,11 +98,13 @@ export class GoogleGCSPublish implements PublisherBase {
    private readonly bucketName: string,
    private readonly legacyPathCasing: boolean,
    private readonly logger: Logger,
+    private readonly bucketRootPath: string,
  ) {
    this.storageClient = storageClient;
    this.bucketName = bucketName;
    this.legacyPathCasing = legacyPathCasing;
    this.logger = logger;
+    this.bucketRootPath = bucketRootPath;
  }

  /**
@@ -133,6 +142,7 @@ export class GoogleGCSPublish implements PublisherBase {
  async publish({ entity, directory }: PublishRequest): Promise<void> {
    const useLegacyPathCasing = this.legacyPathCasing;
    const bucket = this.storageClient.bucket(this.bucketName);
+    const bucketRootPath = this.bucketRootPath;

    // First, try to retrieve a list of all individual files currently existing
    let existingFiles: string[] = [];
@@ -141,6 +151,7 @@ export class GoogleGCSPublish implements PublisherBase {
        entity,
        undefined,
        useLegacyPathCasing,
+        bucketRootPath,
      );
      existingFiles = await this.getFilesForFolder(remoteFolder);
    } catch (e) {
@@ -166,6 +177,7 @@ export class GoogleGCSPublish implements PublisherBase {
              entity,
              relativeFilePath,
              useLegacyPathCasing,
+              bucketRootPath,
            ),
          });
        },
@@ -190,6 +202,7 @@ export class GoogleGCSPublish implements PublisherBase {
            entity,
            path.relative(directory, absoluteFilePath),
            useLegacyPathCasing,
+            bucketRootPath,
          ),
      );
      const staleFiles = getStaleFiles(relativeFilesToUpload, existingFiles);
@@ -214,10 +227,12 @@ export class GoogleGCSPublish implements PublisherBase {
  fetchTechDocsMetadata(entityName: EntityName): Promise<TechDocsMetadata> {
    return new Promise((resolve, reject) => {
      const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
-      const entityRootDir = this.legacyPathCasing
+      const entityDir = this.legacyPathCasing
        ? entityTriplet
        : lowerCaseEntityTriplet(entityTriplet);

+      const entityRootDir = path.join(this.bucketRootPath, entityDir);
+
      const fileStreamChunks: Array<any> = [];
      this.storageClient
        .bucket(this.bucketName)
@@ -246,10 +261,16 @@ export class GoogleGCSPublish implements PublisherBase {
      // Decode and trim the leading forward slash
      const decodedUri = decodeURI(req.path.replace(/^\//, ''));

-      // filePath example - /default/component/documented-component/index.html
-      const filePath = this.legacyPathCasing
-        ? decodedUri
-        : lowerCaseEntityTripletInStoragePath(decodedUri);
+      // Root path is removed from the Uri so that legacy casing can be applied
+      // to the entity triplet without manipulating the root path
+      const decodedUriNoRoot = path.relative(this.bucketRootPath, decodedUri);
+
+      const filePathNoRoot = this.legacyPathCasing
+        ? decodedUriNoRoot
+        : lowerCaseEntityTripletInStoragePath(decodedUriNoRoot);
+
+      // Re-prepend the root path to the relative file path
+      const filePath = path.join(this.bucketRootPath, filePathNoRoot);

      // Files with different extensions (CSS, HTML) need to be served with different headers
      const fileExtension = path.extname(filePath);
@@ -285,10 +306,12 @@ export class GoogleGCSPublish implements PublisherBase {
  async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
    return new Promise(resolve => {
      const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
-      const entityRootDir = this.legacyPathCasing
+      const entityDir = this.legacyPathCasing
        ? entityTriplet
        : lowerCaseEntityTriplet(entityTriplet);

+      const entityRootDir = path.join(this.bucketRootPath, entityDir);
+
      this.storageClient
        .bucket(this.bucketName)
        .file(`${entityRootDir}/index.html`)
@@ -25,6 +25,7 @@ import {
  bulkStorageOperation,
  lowerCaseEntityTriplet,
  lowerCaseEntityTripletInStoragePath,
+  normalizeExternalStorageRootPath,
 } from './helpers';

 describe('getHeadersForFileExtension', () => {
@@ -107,6 +108,34 @@ describe('lowerCaseEntityTripletInStoragePath', () => {
  });
 });

+describe('normalizeExternalStorageRootPath', () => {
+  it('returns an empty string when empty string provided', () => {
+    const originalPath = '';
+    const normalPath = normalizeExternalStorageRootPath(originalPath);
+    expect(normalPath).toBe('');
+  });
+  it('returns an empty string when only separator is provided', () => {
+    const originalPath = '/';
+    const normalPath = normalizeExternalStorageRootPath(originalPath);
+    expect(normalPath).toBe('');
+  });
+  it('returns normalized path from path with leading and trailing sep', () => {
+    const originalPath = '/backstage-data/techdocs/';
+    const normalPath = normalizeExternalStorageRootPath(originalPath);
+    expect(normalPath).toBe('backstage-data/techdocs');
+  });
+  it('returns normalized path from path without leading and trailing sep', () => {
+    const originalPath = 'backstage-data/techdocs';
+    const normalPath = normalizeExternalStorageRootPath(originalPath);
+    expect(normalPath).toBe('backstage-data/techdocs');
+  });
+  it('returns normalized path from path with trailing sep', () => {
+    const originalPath = 'backstage-data/techdocs/';
+    const normalPath = normalizeExternalStorageRootPath(originalPath);
+    expect(normalPath).toBe('backstage-data/techdocs');
+  });
+});
+
 describe('getStaleFiles', () => {
  const defaultFiles = [
    'default/Component/backstage/index.html',
@@ -172,6 +201,60 @@ describe('getCloudPathForLocalPath', () => {
  it('should throw error when entity is invalid', () => {
    expect(() => getCloudPathForLocalPath({} as Entity)).toThrow();
  });
+
+  it('should prepend root directory to destination', () => {
+    const localPath = 'index/html';
+    const rootPath = 'backstage-data/techdocs/';
+    const remoteBucket = getCloudPathForLocalPath(
+      entity,
+      localPath,
+      false,
+      rootPath,
+    );
+    expect(remoteBucket).toBe(
+      `backstage-data/techdocs/custom/component/backstage/${localPath}`,
+    );
+  });
+
+  it('should add trailing seperator to root directory', () => {
+    const localPath = 'index/html';
+    const rootPath = 'backstage-data/techdocs';
+    const remoteBucket = getCloudPathForLocalPath(
+      entity,
+      localPath,
+      false,
+      rootPath,
+    );
+    expect(remoteBucket).toBe(
+      `backstage-data/techdocs/custom/component/backstage/${localPath}`,
+    );
+  });
+
+  it('should remove leading seperator from root directory', () => {
+    const localPath = 'index/html';
+    const rootPath = '/backstage-data/techdocs/';
+    const remoteBucket = getCloudPathForLocalPath(
+      entity,
+      localPath,
+      false,
+      rootPath,
+    );
+    expect(remoteBucket).toBe(
+      `backstage-data/techdocs/custom/component/backstage/${localPath}`,
+    );
+  });
+
+  it('should ignore seperator if root directory is explicitly defined', () => {
+    const localPath = 'index/html';
+    const rootPath = '/';
+    const remoteBucket = getCloudPathForLocalPath(
+      entity,
+      localPath,
+      false,
+      rootPath,
+    );
+    expect(remoteBucket).toBe(`custom/component/backstage/${localPath}`);
+  });
 });

 describe('bulkStorageOperation', () => {
@@ -140,6 +140,29 @@ export const lowerCaseEntityTripletInStoragePath = (
  return lowerCaseEntityTriplet(parts.join(path.posix.sep));
 };

+/**
+ * Take a posix path and return a path without leading and trailing
+ * separators
+ *
+ * @example
+ * normalizeExternalStorageRootPath('/backstage-data/techdocs/')
+ * // return backstage-data/techdocs
+ */
+export const normalizeExternalStorageRootPath = (posixPath: string): string => {
+  // remove leading slash
+  let normalizedPath = posixPath;
+  if (posixPath.startsWith(path.posix.sep)) {
+    normalizedPath = posixPath.slice(1);
+  }
+
+  // remove trailing slash
+  if (normalizedPath.endsWith(path.posix.sep)) {
+    normalizedPath = normalizedPath.slice(0, normalizedPath.length - 1);
+  }
+
+  return normalizedPath;
+};
+
 // Only returns the files that existed previously and are not present anymore.
 export const getStaleFiles = (
  newFiles: string[],
@@ -157,6 +180,7 @@ export const getCloudPathForLocalPath = (
  entity: Entity,
  localPath = '',
  useLegacyPathCasing = false,
+  externalStorageRootPath = '',
 ): string => {
  // Convert destination file path to a POSIX path for uploading.
  // GCS expects / as path separator and relativeFilePath will contain \\ on Windows.
@@ -169,11 +193,17 @@ export const getCloudPathForLocalPath = (
  }/${entity.kind}/${entity.metadata.name}`;

  const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`;
+
  const destination = useLegacyPathCasing
    ? relativeFilePathTriplet
    : lowerCaseEntityTriplet(relativeFilePathTriplet);

-  return destination; // Remote storage file relative path
+  const destinationWithRoot = path.join(
+    ...externalStorageRootPath.split(path.posix.sep),
+    destination,
+  );
+
+  return destinationWithRoot; // Remote storage file relative path
 };

 // Perform rate limited generic operations by passing a function and a list of arguments