From 5f37569058b21cc2945aaa2501e546fa693294c9 Mon Sep 17 00:00:00 2001 From: Camila Loiola Date: Thu, 26 Aug 2021 12:53:10 +0200 Subject: [PATCH 1/5] fix: force using posix path for cloud storages Co-authored-by: Emma Indal Co-authored-by: Otto Sichert Signed-off-by: Camila Loiola --- .../src/stages/publish/awsS3.test.ts | 2 +- .../stages/publish/azureBlobStorage.test.ts | 2 +- .../src/stages/publish/googleStorage.test.ts | 2 +- .../src/stages/publish/helpers.test.ts | 5 ++- .../src/stages/publish/helpers.ts | 41 ++++++++++++------- .../src/stages/publish/openStackSwift.test.ts | 18 +++++++- 6 files changed, 48 insertions(+), 22 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index 028be87e33..dbc81a25b4 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -278,7 +278,7 @@ describe('AwsS3Publish', () => { name: 'path', }; - const techDocsMetadaFilePath = path.join( + const techDocsMetadaFilePath = path.posix.join( ...Object.values(invalidEntityName), 'techdocs_metadata.json', ); diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 200e6a3054..914cc24986 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -297,7 +297,7 @@ describe('AzureBlobStoragePublish', () => { name: 'path', }; - const techDocsMetadaFilePath = path.join( + const techDocsMetadaFilePath = path.posix.join( ...Object.values(invalidEntityName), 'techdocs_metadata.json', ); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index aaa1f6deb5..4247442c54 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -276,7 +276,7 @@ describe('GoogleGCSPublish', () => { name: 'path', }; - const techDocsMetadaFilePath = path.join( + const techDocsMetadaFilePath = path.posix.join( ...Object.values(invalidEntityName), 'techdocs_metadata.json', ); diff --git a/packages/techdocs-common/src/stages/publish/helpers.test.ts b/packages/techdocs-common/src/stages/publish/helpers.test.ts index 2babf547e4..c30f649c8d 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.test.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.test.ts @@ -23,6 +23,7 @@ import { getCloudPathForLocalPath, getHeadersForFileExtension, bulkStorageOperation, + lowerCaseEntityTriplet, lowerCaseEntityTripletInStoragePath, } from './helpers'; @@ -82,10 +83,10 @@ describe('getFileTreeRecursively', () => { }); }); -describe('lowerCaseEntityTripletInStoragePath', () => { +describe('lowerCaseEntityTriplet', () => { it('returns lower-cased entity triplet path', () => { const originalPath = 'default/Component/backstage/index.html'; - const actualPath = lowerCaseEntityTripletInStoragePath(originalPath); + const actualPath = lowerCaseEntityTriplet(originalPath); expect(actualPath).toBe('default/component/backstage/index.html'); }); diff --git a/packages/techdocs-common/src/stages/publish/helpers.ts b/packages/techdocs-common/src/stages/publish/helpers.ts index 0294ed9127..26a16893ed 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.ts @@ -87,7 +87,8 @@ export const getFileTreeRecursively = async ( }; /** - * Returns a lower-cased version of entity's triplet in the form of a path. + * Takes a posix path and returns a lower-cased version of entity's triplet + * with the remaining path in posix. * * Path must not include a starting slash. * @@ -95,39 +96,49 @@ export const getFileTreeRecursively = async ( * lowerCaseEntityTriplet('default/Component/backstage') * // return default/component/backstage */ -export const lowerCaseEntityTriplet = (originalPath: string): string => { - const [namespace, kind, name, ...parts] = originalPath.split('/'); +export const lowerCaseEntityTriplet = (posixPath: string): string => { + const [namespace, kind, name, ...rest] = posixPath.split(path.posix.sep); const lowerNamespace = namespace.toLowerCase(); const lowerKind = kind.toLowerCase(); const lowerName = name.toLowerCase(); - return [lowerNamespace, lowerKind, lowerName, ...parts].join('/'); + return [lowerNamespace, lowerKind, lowerName, ...rest].join(path.posix.sep); }; /** - * Returns the version of an object's storage path where the first three parts - * of the path (the entity triplet of namespace, kind, and name) are - * lower-cased. + * Takes either a win32 or posix path and returns a lower-cased version of entity's triplet + * with the remaining path in posix. * - * Path must not include a starting slash. + * Starting slashes will be trimmed. * * Throws an error if the path does not appear to be an entity triplet. * * @example - * lowerCaseEntityTripletInStoragePath('default/Component/backstage/file.txt') + * lowerCaseEntityTripletInStoragePath('/default/Component/backstage/file.txt') * // return default/component/backstage/file.txt */ export const lowerCaseEntityTripletInStoragePath = ( originalPath: string, ): string => { - const trimmedPath = - originalPath[0] === '/' ? originalPath.substring(1) : originalPath; - const matches = trimmedPath.match(/\//g) || []; - if (matches.length <= 2) { + // normalize path using path.sep (as its support both windows and posix) + let posixPath = originalPath; + if (originalPath.includes(path.win32.sep)) { + posixPath = originalPath.split(path.win32.sep).join(path.posix.sep); + } + + // remove leading slash + const parts = posixPath.split(path.posix.sep); + if (parts[0] === '') { + parts.shift(); + } + + // check if all parts of the entity exist (name, namespace, kind) plus filename + if (parts.length <= 3) { throw new Error( `Encountered file unmanaged by TechDocs ${originalPath}. Skipping.`, ); } - return lowerCaseEntityTriplet(originalPath); + + return lowerCaseEntityTriplet(parts.join(path.posix.sep)); }; // Only returns the files that existed previously and are not present anymore. @@ -161,7 +172,7 @@ export const getCloudPathForLocalPath = ( const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`; const destination = useLegacyPathCasing ? relativeFilePathTriplet - : lowerCaseEntityTripletInStoragePath(relativeFilePathTriplet); + : lowerCaseEntityTriplet(relativeFilePathTriplet); return destination; // Remote storage file relative path }; diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts index 9a33fe08cd..84007c0f5b 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts @@ -62,6 +62,20 @@ const getEntityRootDir = (entity: Entity) => { return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name); }; +const getPosixEntityRootDir = (entity: Entity) => { + const { + kind, + metadata: { namespace, name }, + } = entity; + + return path.posix.join( + '/rootDir', + namespace || ENTITY_DEFAULT_NAMESPACE, + kind, + name, + ); +}; + const logger = getVoidLogger(); let publisher: PublisherBase; @@ -267,12 +281,12 @@ describe('OpenStackSwiftPublish', () => { it('should return an error if the techdocs_metadata.json file is not present', async () => { const entityNameMock = createMockEntityName(); const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const entityRootDir = getPosixEntityRootDir(entity); const fails = publisher.fetchTechDocsMetadata(entityNameMock); await expect(fails).rejects.toMatchObject({ - message: `TechDocs metadata fetch failed, The file ${path.join( + message: `TechDocs metadata fetch failed, The file ${path.posix.join( entityRootDir, 'techdocs_metadata.json', )} does not exist !`, From 8eab6be6a4725a6ec3b101b661504ad23ac38dc8 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Thu, 26 Aug 2021 13:06:07 +0200 Subject: [PATCH 2/5] add changeset file Signed-off-by: Camila Belo --- .changeset/techdocs-trains-beg.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/techdocs-trains-beg.md diff --git a/.changeset/techdocs-trains-beg.md b/.changeset/techdocs-trains-beg.md new file mode 100644 index 0000000000..1618dad07b --- /dev/null +++ b/.changeset/techdocs-trains-beg.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': minor +--- + +Force using posix path for cloud storages From b281d3a7f92c74225e1e024bd4f282f52b440265 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Thu, 26 Aug 2021 13:08:11 +0200 Subject: [PATCH 3/5] apply review suggestions Signed-off-by: Camila Belo --- packages/techdocs-common/src/stages/publish/helpers.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/techdocs-common/src/stages/publish/helpers.ts b/packages/techdocs-common/src/stages/publish/helpers.ts index 26a16893ed..9b935ccbfd 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.ts @@ -119,7 +119,6 @@ export const lowerCaseEntityTriplet = (posixPath: string): string => { export const lowerCaseEntityTripletInStoragePath = ( originalPath: string, ): string => { - // normalize path using path.sep (as its support both windows and posix) let posixPath = originalPath; if (originalPath.includes(path.win32.sep)) { posixPath = originalPath.split(path.win32.sep).join(path.posix.sep); From b257ca7d8bc51738a64a8cecc0ec1339c1e44290 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Thu, 26 Aug 2021 13:14:26 +0200 Subject: [PATCH 4/5] Fix changeset content Signed-off-by: Camila Belo --- .changeset/techdocs-trains-beg.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/techdocs-trains-beg.md b/.changeset/techdocs-trains-beg.md index 1618dad07b..05161b7093 100644 --- a/.changeset/techdocs-trains-beg.md +++ b/.changeset/techdocs-trains-beg.md @@ -1,5 +1,5 @@ --- -'@backstage/techdocs-common': minor +'@backstage/techdocs-common': patch --- -Force using posix path for cloud storages +Force using `posix` path for cloud storage From 38b9bd9c6385b25ed0d2011a2c0c3f5c0f6e5404 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Thu, 26 Aug 2021 13:26:55 +0200 Subject: [PATCH 5/5] Fix test describes Signed-off-by: Camila Belo --- packages/techdocs-common/src/stages/publish/helpers.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/techdocs-common/src/stages/publish/helpers.test.ts b/packages/techdocs-common/src/stages/publish/helpers.test.ts index c30f649c8d..a3571ef0ab 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.test.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.test.ts @@ -89,7 +89,9 @@ describe('lowerCaseEntityTriplet', () => { const actualPath = lowerCaseEntityTriplet(originalPath); expect(actualPath).toBe('default/component/backstage/index.html'); }); +}); +describe('lowerCaseEntityTripletInStoragePath', () => { it('does not lowercase beyond the triplet', () => { const originalPath = 'default/Component/backstage/assets/IMAGE.png'; const actualPath = lowerCaseEntityTripletInStoragePath(originalPath);