diff --git a/.changeset/quick-walls-know.md b/.changeset/quick-walls-know.md new file mode 100644 index 0000000000..14dd78364f --- /dev/null +++ b/.changeset/quick-walls-know.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Support optional bucketRootPath configuration parameter in S3 and GCS publishers diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 46cbc2a8a2..fbe9f36540 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -63,6 +63,10 @@ techdocs: # (Required) Cloud Storage Bucket Name bucketName: 'techdocs-storage' + # (Optional) Location in storage bucket to save files + # If not set, the default location will be the root of the storage bucket + bucketRootPath: '/' + # (Optional) An API key is required to write to a storage bucket. # If missing, GOOGLE_APPLICATION_CREDENTIALS environment variable will be used. # https://cloud.google.com/docs/authentication/production @@ -75,6 +79,10 @@ techdocs: # (Required) AWS S3 Bucket Name bucketName: 'techdocs-storage' + # (Optional) Location in storage bucket to save files + # If not set, the default location will be the root of the storage bucket + bucketRootPath: '/' + # (Optional) An API key is required to write to a storage bucket. # If not set, environment variables or aws config file will be used to authenticate. # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index 3a637ebbc6..9b74d89dda 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -43,9 +43,11 @@ const loggerErrorSpy = jest.spyOn(logger, 'error'); const createPublisherFromConfig = ({ bucketName = 'bucketName', + bucketRootPath = '/', legacyUseCaseSensitiveTripletPaths = false, }: { bucketName?: string; + bucketRootPath?: string; legacyUseCaseSensitiveTripletPaths?: boolean; } = {}) => { const mockConfig = new ConfigReader({ @@ -59,6 +61,7 @@ const createPublisherFromConfig = ({ secretAccessKey: 'secretAccessKey', }, bucketName, + bucketRootPath, }, }, legacyUseCaseSensitiveTripletPaths, @@ -153,6 +156,21 @@ describe('AwsS3Publish', () => { expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); + it('should publish a directory when root path is specified', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + }); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); + }); + + it('should publish a directory when root path is specified and legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + legacyUseCaseSensitiveTripletPaths: true, + }); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); + }); + it('should fail to publish a directory', async () => { const wrongPathToGeneratedDirectory = path.join( rootDir, @@ -214,6 +232,23 @@ describe('AwsS3Publish', () => { expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); + it('should return true if docs has been generated if root path is specified', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + }); + await publisher.publish({ entity, directory }); + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + }); + + it('should return true if docs has been generated if root path is specified and legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + legacyUseCaseSensitiveTripletPaths: true, + }); + await publisher.publish({ entity, directory }); + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + }); + it('should return false if docs has not been generated', async () => { const publisher = createPublisherFromConfig(); expect( @@ -247,6 +282,27 @@ describe('AwsS3Publish', () => { ); }); + it('should return tech docs metadata even if root path is specified', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + }); + await publisher.publish({ entity, directory }); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + }); + + it('should return tech docs metadata if root path is specified and legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + legacyUseCaseSensitiveTripletPaths: true, + }); + await publisher.publish({ entity, directory }); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + }); + it('should return tech docs metadata when json encoded with single quotes', async () => { const techdocsMetadataPath = path.join( directory, @@ -335,6 +391,47 @@ describe('AwsS3Publish', () => { expect(jsResponse.text).toEqual('found it too'); }); + it('should pass expected object path to bucket if root path is specified', async () => { + const rootPath = 'backstage-data/techdocs'; + const publisher = createPublisherFromConfig({ + bucketRootPath: rootPath, + }); + await publisher.publish({ entity, directory }); + app = express().use(publisher.docsRouter()); + + const pngResponse = await request(app).get( + `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`, + ); + expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( + 'found it', + ); + const jsResponse = await request(app).get( + `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, + ); + expect(jsResponse.text).toEqual('found it too'); + }); + + it('should pass expected object path to bucket if root path is specified and legacy case is enabled', async () => { + const rootPath = 'backstage-data/techdocs'; + const publisher = createPublisherFromConfig({ + bucketRootPath: rootPath, + legacyUseCaseSensitiveTripletPaths: true, + }); + await publisher.publish({ entity, directory }); + app = express().use(publisher.docsRouter()); + + const pngResponse = await request(app).get( + `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`, + ); + expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( + 'found it', + ); + const jsResponse = await request(app).get( + `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, + ); + expect(jsResponse.text).toEqual('found it too'); + }); + it('should pass text/plain content-type for html', async () => { const htmlResponse = await request(app).get( `/${entityTripletPath}/html/unsafe.html`, diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index ee48db575f..29eecb8d03 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -34,6 +34,7 @@ import { getStaleFiles, lowerCaseEntityTriplet, lowerCaseEntityTripletInStoragePath, + normalizeExternalStorageRootPath, } from './helpers'; import { PublisherBase, @@ -67,6 +68,10 @@ export class AwsS3Publish implements PublisherBase { ); } + const bucketRootPath = normalizeExternalStorageRootPath( + config.getOptionalString('techdocs.publisher.awsS3.bucketRootPath') || '', + ); + // Credentials is an optional config. If missing, the default ways of authenticating AWS SDK V2 will be used. // 1. AWS environment variables // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html @@ -112,6 +117,7 @@ export class AwsS3Publish implements PublisherBase { bucketName, legacyPathCasing, logger, + bucketRootPath, ); } @@ -151,11 +157,13 @@ export class AwsS3Publish implements PublisherBase { private readonly bucketName: string, private readonly legacyPathCasing: boolean, private readonly logger: Logger, + private readonly bucketRootPath: string, ) { this.storageClient = storageClient; this.bucketName = bucketName; this.legacyPathCasing = legacyPathCasing; this.logger = logger; + this.bucketRootPath = bucketRootPath; } /** @@ -193,6 +201,8 @@ export class AwsS3Publish implements PublisherBase { */ async publish({ entity, directory }: PublishRequest): Promise { const useLegacyPathCasing = this.legacyPathCasing; + const bucketRootPath = this.bucketRootPath; + // First, try to retrieve a list of all individual files currently existing let existingFiles: string[] = []; try { @@ -200,6 +210,7 @@ export class AwsS3Publish implements PublisherBase { entity, undefined, useLegacyPathCasing, + bucketRootPath, ); existingFiles = await this.getAllObjectsFromBucket({ prefix: remoteFolder, @@ -230,6 +241,7 @@ export class AwsS3Publish implements PublisherBase { entity, relativeFilePath, useLegacyPathCasing, + bucketRootPath, ), Body: fileStream, }; @@ -257,6 +269,7 @@ export class AwsS3Publish implements PublisherBase { entity, path.relative(directory, absoluteFilePath), useLegacyPathCasing, + bucketRootPath, ), ); const staleFiles = getStaleFiles(relativeFilesToUpload, existingFiles); @@ -289,10 +302,12 @@ export class AwsS3Publish implements PublisherBase { try { return await new Promise(async (resolve, reject) => { const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; - const entityRootDir = this.legacyPathCasing + const entityDir = this.legacyPathCasing ? entityTriplet : lowerCaseEntityTriplet(entityTriplet); + const entityRootDir = path.join(this.bucketRootPath, entityDir); + const stream = this.storageClient .getObject({ Bucket: this.bucketName, @@ -332,10 +347,17 @@ export class AwsS3Publish implements PublisherBase { // Decode and trim the leading forward slash const decodedUri = decodeURI(req.path.replace(/^\//, '')); + // Root path is removed from the Uri so that legacy casing can be applied + // to the entity triplet without manipulating the root path + const decodedUriNoRoot = path.relative(this.bucketRootPath, decodedUri); + // filePath example - /default/component/documented-component/index.html - const filePath = this.legacyPathCasing - ? decodedUri - : lowerCaseEntityTripletInStoragePath(decodedUri); + const filePathNoRoot = this.legacyPathCasing + ? decodedUriNoRoot + : lowerCaseEntityTripletInStoragePath(decodedUriNoRoot); + + // Re-prepend the root path to the relative file path + const filePath = path.join(this.bucketRootPath, filePathNoRoot); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); @@ -370,10 +392,12 @@ export class AwsS3Publish implements PublisherBase { async hasDocsBeenGenerated(entity: Entity): Promise { try { const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const entityRootDir = this.legacyPathCasing + const entityDir = this.legacyPathCasing ? entityTriplet : lowerCaseEntityTriplet(entityTriplet); + const entityRootDir = path.join(this.bucketRootPath, entityDir); + await this.storageClient .headObject({ Bucket: this.bucketName, diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index a04f61ac11..d31995ea2e 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -43,9 +43,11 @@ jest.spyOn(logger, 'error').mockReturnValue(logger); const createPublisherFromConfig = ({ bucketName = 'bucketName', + bucketRootPath = '/', legacyUseCaseSensitiveTripletPaths = false, }: { bucketName?: string; + bucketRootPath?: string; legacyUseCaseSensitiveTripletPaths?: boolean; } = {}) => { const config = new ConfigReader({ @@ -56,6 +58,7 @@ const createPublisherFromConfig = ({ googleGcs: { credentials: '{}', bucketName, + bucketRootPath, }, }, legacyUseCaseSensitiveTripletPaths, @@ -149,6 +152,21 @@ describe('GoogleGCSPublish', () => { expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); + it('should publish a directory when root path is specified', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + }); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); + }); + + it('should publish a directory when root path is specified and legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + legacyUseCaseSensitiveTripletPaths: true, + }); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); + }); + it('should fail to publish a directory', async () => { const wrongPathToGeneratedDirectory = path.join( rootDir, @@ -212,6 +230,23 @@ describe('GoogleGCSPublish', () => { expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); + it('should return true if docs has been generated if root path is specified', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + }); + await publisher.publish({ entity, directory }); + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + }); + + it('should return true if docs has been generated if root path is specified and legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + legacyUseCaseSensitiveTripletPaths: true, + }); + await publisher.publish({ entity, directory }); + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + }); + it('should return false if docs has not been generated', async () => { const publisher = createPublisherFromConfig(); expect( @@ -245,6 +280,27 @@ describe('GoogleGCSPublish', () => { ); }); + it('should return tech docs metadata even if root path is specified', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + }); + await publisher.publish({ entity, directory }); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + }); + + it('should return tech docs metadata if root path is specified and legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + bucketRootPath: 'backstage-data/techdocs', + legacyUseCaseSensitiveTripletPaths: true, + }); + await publisher.publish({ entity, directory }); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + }); + it('should return tech docs metadata when json encoded with single quotes', async () => { const techdocsMetadataPath = path.join( directory, @@ -291,6 +347,7 @@ describe('GoogleGCSPublish', () => { describe('docsRouter', () => { const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + // const entityTripletPath = let app: Express.Application; @@ -334,6 +391,47 @@ describe('GoogleGCSPublish', () => { expect(jsResponse.text).toEqual('found it too'); }); + it('should pass expected object path to bucket if root path is specified', async () => { + const rootPath = 'backstage-data/techdocs'; + const publisher = createPublisherFromConfig({ + bucketRootPath: rootPath, + }); + await publisher.publish({ entity, directory }); + app = express().use(publisher.docsRouter()); + + const pngResponse = await request(app).get( + `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`, + ); + expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( + 'found it', + ); + const jsResponse = await request(app).get( + `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, + ); + expect(jsResponse.text).toEqual('found it too'); + }); + + it('should pass expected object path to bucket if root path is specified and legacy case is enabled', async () => { + const rootPath = 'backstage-data/techdocs'; + const publisher = createPublisherFromConfig({ + bucketRootPath: rootPath, + legacyUseCaseSensitiveTripletPaths: true, + }); + await publisher.publish({ entity, directory }); + app = express().use(publisher.docsRouter()); + + const pngResponse = await request(app).get( + `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`, + ); + expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( + 'found it', + ); + const jsResponse = await request(app).get( + `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, + ); + expect(jsResponse.text).toEqual('found it too'); + }); + it('should pass text/plain content-type for html', async () => { const htmlResponse = await request(app).get( `/${entityTripletPath}/html/unsafe.html`, diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index dcce1b8ee9..5a142e800a 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -30,6 +30,7 @@ import { bulkStorageOperation, getCloudPathForLocalPath, getStaleFiles, + normalizeExternalStorageRootPath, } from './helpers'; import { MigrateWriteStream } from './migrations'; import { @@ -51,6 +52,11 @@ export class GoogleGCSPublish implements PublisherBase { ); } + const bucketRootPath = normalizeExternalStorageRootPath( + config.getOptionalString('techdocs.publisher.googleGcs.bucketRootPath') || + '', + ); + // Credentials is an optional config. If missing, default GCS environment variables will be used. // Read more here https://cloud.google.com/docs/authentication/production const credentials = config.getOptionalString( @@ -83,6 +89,7 @@ export class GoogleGCSPublish implements PublisherBase { bucketName, legacyPathCasing, logger, + bucketRootPath, ); } @@ -91,11 +98,13 @@ export class GoogleGCSPublish implements PublisherBase { private readonly bucketName: string, private readonly legacyPathCasing: boolean, private readonly logger: Logger, + private readonly bucketRootPath: string, ) { this.storageClient = storageClient; this.bucketName = bucketName; this.legacyPathCasing = legacyPathCasing; this.logger = logger; + this.bucketRootPath = bucketRootPath; } /** @@ -133,6 +142,7 @@ export class GoogleGCSPublish implements PublisherBase { async publish({ entity, directory }: PublishRequest): Promise { const useLegacyPathCasing = this.legacyPathCasing; const bucket = this.storageClient.bucket(this.bucketName); + const bucketRootPath = this.bucketRootPath; // First, try to retrieve a list of all individual files currently existing let existingFiles: string[] = []; @@ -141,6 +151,7 @@ export class GoogleGCSPublish implements PublisherBase { entity, undefined, useLegacyPathCasing, + bucketRootPath, ); existingFiles = await this.getFilesForFolder(remoteFolder); } catch (e) { @@ -166,6 +177,7 @@ export class GoogleGCSPublish implements PublisherBase { entity, relativeFilePath, useLegacyPathCasing, + bucketRootPath, ), }); }, @@ -190,6 +202,7 @@ export class GoogleGCSPublish implements PublisherBase { entity, path.relative(directory, absoluteFilePath), useLegacyPathCasing, + bucketRootPath, ), ); const staleFiles = getStaleFiles(relativeFilesToUpload, existingFiles); @@ -214,10 +227,12 @@ export class GoogleGCSPublish implements PublisherBase { fetchTechDocsMetadata(entityName: EntityName): Promise { return new Promise((resolve, reject) => { const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; - const entityRootDir = this.legacyPathCasing + const entityDir = this.legacyPathCasing ? entityTriplet : lowerCaseEntityTriplet(entityTriplet); + const entityRootDir = path.join(this.bucketRootPath, entityDir); + const fileStreamChunks: Array = []; this.storageClient .bucket(this.bucketName) @@ -246,10 +261,16 @@ export class GoogleGCSPublish implements PublisherBase { // Decode and trim the leading forward slash const decodedUri = decodeURI(req.path.replace(/^\//, '')); - // filePath example - /default/component/documented-component/index.html - const filePath = this.legacyPathCasing - ? decodedUri - : lowerCaseEntityTripletInStoragePath(decodedUri); + // Root path is removed from the Uri so that legacy casing can be applied + // to the entity triplet without manipulating the root path + const decodedUriNoRoot = path.relative(this.bucketRootPath, decodedUri); + + const filePathNoRoot = this.legacyPathCasing + ? decodedUriNoRoot + : lowerCaseEntityTripletInStoragePath(decodedUriNoRoot); + + // Re-prepend the root path to the relative file path + const filePath = path.join(this.bucketRootPath, filePathNoRoot); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); @@ -285,10 +306,12 @@ export class GoogleGCSPublish implements PublisherBase { async hasDocsBeenGenerated(entity: Entity): Promise { return new Promise(resolve => { const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const entityRootDir = this.legacyPathCasing + const entityDir = this.legacyPathCasing ? entityTriplet : lowerCaseEntityTriplet(entityTriplet); + const entityRootDir = path.join(this.bucketRootPath, entityDir); + this.storageClient .bucket(this.bucketName) .file(`${entityRootDir}/index.html`) diff --git a/packages/techdocs-common/src/stages/publish/helpers.test.ts b/packages/techdocs-common/src/stages/publish/helpers.test.ts index a3571ef0ab..6245898586 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.test.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.test.ts @@ -25,6 +25,7 @@ import { bulkStorageOperation, lowerCaseEntityTriplet, lowerCaseEntityTripletInStoragePath, + normalizeExternalStorageRootPath, } from './helpers'; describe('getHeadersForFileExtension', () => { @@ -107,6 +108,34 @@ describe('lowerCaseEntityTripletInStoragePath', () => { }); }); +describe('normalizeExternalStorageRootPath', () => { + it('returns an empty string when empty string provided', () => { + const originalPath = ''; + const normalPath = normalizeExternalStorageRootPath(originalPath); + expect(normalPath).toBe(''); + }); + it('returns an empty string when only separator is provided', () => { + const originalPath = '/'; + const normalPath = normalizeExternalStorageRootPath(originalPath); + expect(normalPath).toBe(''); + }); + it('returns normalized path from path with leading and trailing sep', () => { + const originalPath = '/backstage-data/techdocs/'; + const normalPath = normalizeExternalStorageRootPath(originalPath); + expect(normalPath).toBe('backstage-data/techdocs'); + }); + it('returns normalized path from path without leading and trailing sep', () => { + const originalPath = 'backstage-data/techdocs'; + const normalPath = normalizeExternalStorageRootPath(originalPath); + expect(normalPath).toBe('backstage-data/techdocs'); + }); + it('returns normalized path from path with trailing sep', () => { + const originalPath = 'backstage-data/techdocs/'; + const normalPath = normalizeExternalStorageRootPath(originalPath); + expect(normalPath).toBe('backstage-data/techdocs'); + }); +}); + describe('getStaleFiles', () => { const defaultFiles = [ 'default/Component/backstage/index.html', @@ -172,6 +201,60 @@ describe('getCloudPathForLocalPath', () => { it('should throw error when entity is invalid', () => { expect(() => getCloudPathForLocalPath({} as Entity)).toThrow(); }); + + it('should prepend root directory to destination', () => { + const localPath = 'index/html'; + const rootPath = 'backstage-data/techdocs/'; + const remoteBucket = getCloudPathForLocalPath( + entity, + localPath, + false, + rootPath, + ); + expect(remoteBucket).toBe( + `backstage-data/techdocs/custom/component/backstage/${localPath}`, + ); + }); + + it('should add trailing seperator to root directory', () => { + const localPath = 'index/html'; + const rootPath = 'backstage-data/techdocs'; + const remoteBucket = getCloudPathForLocalPath( + entity, + localPath, + false, + rootPath, + ); + expect(remoteBucket).toBe( + `backstage-data/techdocs/custom/component/backstage/${localPath}`, + ); + }); + + it('should remove leading seperator from root directory', () => { + const localPath = 'index/html'; + const rootPath = '/backstage-data/techdocs/'; + const remoteBucket = getCloudPathForLocalPath( + entity, + localPath, + false, + rootPath, + ); + expect(remoteBucket).toBe( + `backstage-data/techdocs/custom/component/backstage/${localPath}`, + ); + }); + + it('should ignore seperator if root directory is explicitly defined', () => { + const localPath = 'index/html'; + const rootPath = '/'; + const remoteBucket = getCloudPathForLocalPath( + entity, + localPath, + false, + rootPath, + ); + expect(remoteBucket).toBe(`custom/component/backstage/${localPath}`); + }); }); describe('bulkStorageOperation', () => { diff --git a/packages/techdocs-common/src/stages/publish/helpers.ts b/packages/techdocs-common/src/stages/publish/helpers.ts index 9b935ccbfd..c23c72af38 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.ts @@ -140,6 +140,29 @@ export const lowerCaseEntityTripletInStoragePath = ( return lowerCaseEntityTriplet(parts.join(path.posix.sep)); }; +/** + * Take a posix path and return a path without leading and trailing + * separators + * + * @example + * normalizeExternalStorageRootPath('/backstage-data/techdocs/') + * // return backstage-data/techdocs + */ +export const normalizeExternalStorageRootPath = (posixPath: string): string => { + // remove leading slash + let normalizedPath = posixPath; + if (posixPath.startsWith(path.posix.sep)) { + normalizedPath = posixPath.slice(1); + } + + // remove trailing slash + if (normalizedPath.endsWith(path.posix.sep)) { + normalizedPath = normalizedPath.slice(0, normalizedPath.length - 1); + } + + return normalizedPath; +}; + // Only returns the files that existed previously and are not present anymore. export const getStaleFiles = ( newFiles: string[], @@ -157,6 +180,7 @@ export const getCloudPathForLocalPath = ( entity: Entity, localPath = '', useLegacyPathCasing = false, + externalStorageRootPath = '', ): string => { // Convert destination file path to a POSIX path for uploading. // GCS expects / as path separator and relativeFilePath will contain \\ on Windows. @@ -169,11 +193,17 @@ export const getCloudPathForLocalPath = ( }/${entity.kind}/${entity.metadata.name}`; const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`; + const destination = useLegacyPathCasing ? relativeFilePathTriplet : lowerCaseEntityTriplet(relativeFilePathTriplet); - return destination; // Remote storage file relative path + const destinationWithRoot = path.join( + ...externalStorageRootPath.split(path.posix.sep), + destination, + ); + + return destinationWithRoot; // Remote storage file relative path }; // Perform rate limited generic operations by passing a function and a list of arguments