From 41e5f28ab77b8f0b60d2e8d6aba62bcfccc4dd72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Can=20Bilgi=C3=A7?= Date: Tue, 9 Nov 2021 12:36:18 +0300 Subject: [PATCH] OSS lowercase migration added MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mert Can Bilgiç --- packages/techdocs-common/package.json | 2 +- .../src/stages/publish/openStackSwift.ts | 66 +++++++++++++++++-- yarn.lock | 64 +++++++++++++++--- 3 files changed, 117 insertions(+), 15 deletions(-) diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 152b9b04f1..2bc3e80f26 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -45,7 +45,7 @@ "@backstage/search-common": "^0.2.1", "@backstage/integration": "^0.6.9", "@google-cloud/storage": "^5.6.0", - "@trendyol-js/openstack-swift-sdk": "^0.0.4", + "@trendyol-js/openstack-swift-sdk": "^0.0.5", "@types/express": "^4.17.6", "aws-sdk": "^2.840.0", "express": "^4.17.1", diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.ts index a463157552..83e17fa466 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.ts @@ -24,7 +24,7 @@ import { SwiftClient } from '@trendyol-js/openstack-swift-sdk'; import { NotFound } from '@trendyol-js/openstack-swift-sdk/lib/types'; import { Stream, Readable } from 'stream'; import { Logger } from 'winston'; -import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { getFileTreeRecursively, getHeadersForFileExtension, lowerCaseEntityTripletInStoragePath } from './helpers'; import { PublisherBase, PublishRequest, @@ -77,7 +77,7 @@ export class OpenStackSwiftPublish implements PublisherBase { } catch (error) { throw new Error( "Since techdocs.publisher.type is set to 'openStackSwift' in your app config, " + - 'techdocs.publisher.openStackSwift.containerName is required.', + 'techdocs.publisher.openStackSwift.containerName is required.', ); } @@ -115,9 +115,9 @@ export class OpenStackSwiftPublish implements PublisherBase { } this.logger.error( `Could not retrieve metadata about the OpenStack Swift container ${this.containerName}. ` + - 'Make sure the container exists. Also make sure that authentication is setup either by ' + - 'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' + - 'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + 'Make sure the container exists. Also make sure that authentication is setup either by ' + + 'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' + + 'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', ); return { isAvailable: false, @@ -289,4 +289,60 @@ export class OpenStackSwiftPublish implements PublisherBase { return false; } } + + + async migrateDocsCase({ + removeOriginal = false, + concurrency = 25, + }): Promise { + // Iterate through every file in the root of the publisher. + const allObjects = await this.getAllObjectsFromContainer(); + const limiter = createLimiter(concurrency); + await Promise.all( + allObjects.map(f => + limiter(async file => { + let newPath; + try { + newPath = lowerCaseEntityTripletInStoragePath(file); + } catch (e) { + assertError(e); + this.logger.warn(e.message); + return; + } + + // If all parts are already lowercase, ignore. + if (file === newPath) { + return; + } + + try { + this.logger.verbose(`Migrating ${file} to ${newPath}`); + await this.storageClient.copy(this.containerName, file, this.containerName, newPath); + if (removeOriginal) { + await this.storageClient.delete(this.containerName, file); + } + } catch (e) { + assertError(e); + this.logger.warn(`Unable to migrate ${file}: ${e.message}`); + } + }, f), + ), + ); + } + + /** + * Returns a list of all object keys from the configured container. + */ + protected async getAllObjectsFromContainer( + { prefix } = { prefix: '' }, + ): Promise { + let objects: string[] = []; + let allObjects: any; + const OSS_MAX_LIMIT = Math.pow(2, 31) - 1; + + allObjects = await this.storageClient.list(this.containerName, prefix, OSS_MAX_LIMIT) + objects = allObjects.map((object: any) => object = object.name) + + return objects; + } } diff --git a/yarn.lock b/yarn.lock index 42c43fe4b7..915dfdfbda 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4613,7 +4613,7 @@ call-me-maybe "^1.0.1" glob-to-regexp "^0.3.0" -"@mswjs/cookies@^0.1.6": +"@mswjs/cookies@^0.1.5", "@mswjs/cookies@^0.1.6": version "0.1.6" resolved "https://registry.npmjs.org/@mswjs/cookies/-/cookies-0.1.6.tgz#176f77034ab6d7373ae5c94bcbac36fee8869249" integrity sha512-A53XD5TOfwhpqAmwKdPtg1dva5wrng2gH5xMvklzbd9WLTSVU953eCRa8rtrrm6G7Cy60BOGsBRN89YQK0mlKA== @@ -4621,6 +4621,17 @@ "@types/set-cookie-parser" "^2.4.0" set-cookie-parser "^2.4.6" +"@mswjs/interceptors@^0.10.0": + version "0.10.0" + resolved "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.10.0.tgz#f5aad03c2c0591d164e3ed178b21942f1c2f8061" + integrity sha512-/M0GGpid5q2EDI+Keas1sLYF3VZFXHDE5gCmX/jHdp+OJFruVNca3PUk7A8KnGdPpuycZogdPsmRBSOXwjyA7A== + dependencies: + "@open-draft/until" "^1.0.3" + debug "^4.3.0" + headers-utils "^3.0.2" + strict-event-emitter "^0.2.0" + xmldom "^0.6.0" + "@mswjs/interceptors@^0.12.6": version "0.12.7" resolved "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.12.7.tgz#0d1cd4cd31a0f663e0455993951201faa09d0909" @@ -6695,10 +6706,10 @@ resolved "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== -"@trendyol-js/openstack-swift-sdk@^0.0.4": - version "0.0.4" - resolved "https://registry.npmjs.org/@trendyol-js/openstack-swift-sdk/-/openstack-swift-sdk-0.0.4.tgz#570c6ab950319156c175ace005b4fb4d9f895d47" - integrity sha512-9YKOjov+V+yzptei6+B9QPuC5pOMTBTg/NQpb1ZbxvlOaYpWU4HHpSH2BkIFYZ8vYyAfzFNG1T2rjpQ2ZQDUtQ== +"@trendyol-js/openstack-swift-sdk@^0.0.5": + version "0.0.5" + resolved "https://registry.npmjs.org/@trendyol-js/openstack-swift-sdk/-/openstack-swift-sdk-0.0.5.tgz#65be3c42b8dbafc57f2f2a46c327e2ad51e5a70e" + integrity sha512-KS5nz0cvd35UUyMzhZm+btGV4prtA1KNE7CCMOGBdVxoMGl06Qidli3HgHoc2I9jLPmky1SPp5yzQUwrsyWa0g== dependencies: agentkeepalive "^4.1.4" axios "^0.21.1" @@ -6912,7 +6923,7 @@ dependencies: "@types/express" "*" -"@types/cookie@^0.4.1": +"@types/cookie@^0.4.0", "@types/cookie@^0.4.1": version "0.4.1" resolved "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz#bfd02c1f2224567676c1545199f87c3a861d878d" integrity sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q== @@ -12586,7 +12597,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.0, debug@^2.6.9: dependencies: ms "2.0.0" -debug@4, debug@4.3.2, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2: +debug@4, debug@4.3.2, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.0, debug@^4.3.1, debug@^4.3.2: version "4.3.2" resolved "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== @@ -15722,6 +15733,11 @@ graphql@^15.3.0: resolved "https://registry.npmjs.org/graphql/-/graphql-15.5.1.tgz#f2f84415d8985e7b84731e7f3536f8bb9d383aad" integrity sha512-FeTRX67T3LoE3LWAxxOlW2K3Bz+rMYAC18rRguK4wgXaTZMiJwSUwDmPFo3UadAKbzirKIg5Qy+sNJXbpPRnQw== +graphql@^15.4.0: + version "15.7.2" + resolved "https://registry.npmjs.org/graphql/-/graphql-15.7.2.tgz#85ab0eeb83722977151b3feb4d631b5f2ab287ef" + integrity sha512-AnnKk7hFQFmU/2I9YSQf3xw44ctnSFCfp3zE0N6W174gqe9fWG/2rKaKxROK7CcI3XtERpjEKFqts8o319Kf7A== + graphql@^15.5.1: version "15.6.1" resolved "https://registry.npmjs.org/graphql/-/graphql-15.6.1.tgz#9125bdf057553525da251e19e96dab3d3855ddfc" @@ -16649,7 +16665,7 @@ inquirer@^8.0.0: strip-ansi "^6.0.0" through "^2.3.6" -inquirer@^8.1.1: +inquirer@^8.1.0, inquirer@^8.1.1: version "8.2.0" resolved "https://registry.npmjs.org/inquirer/-/inquirer-8.2.0.tgz#f44f008dd344bbfc4b30031f45d984e034a3ac3a" integrity sha512-0crLweprevJ02tTuA6ThpoAERAGyVILC4sS74uib58Xf/zSr1/ZWtmm7D5CI+bSQEaA04f0K7idaHpQbSWgiVQ== @@ -20678,6 +20694,31 @@ msal@^1.0.2: dependencies: tslib "^1.9.3" +msw@^0.29.0: + version "0.29.0" + resolved "https://registry.npmjs.org/msw/-/msw-0.29.0.tgz#7242d575cb01db0c925241587df1fc2b79230d78" + integrity sha512-C/wz1d5uAEZRvAPAYrXG1rwLxXl0+BOs+JPrCzasoABZW3ATwS6ifSze+/DAgA93e9M86RXwvy6yDtZeZWmCFQ== + dependencies: + "@mswjs/cookies" "^0.1.5" + "@mswjs/interceptors" "^0.10.0" + "@open-draft/until" "^1.0.3" + "@types/cookie" "^0.4.0" + "@types/inquirer" "^7.3.1" + "@types/js-levenshtein" "^1.1.0" + chalk "^4.1.1" + chokidar "^3.4.2" + cookie "^0.4.1" + graphql "^15.4.0" + headers-utils "^3.0.2" + inquirer "^8.1.0" + js-levenshtein "^1.1.6" + node-fetch "^2.6.1" + node-match-path "^0.6.3" + statuses "^2.0.0" + strict-event-emitter "^0.2.0" + type-fest "^1.1.3" + yargs "^17.0.1" + msw@^0.35.0: version "0.35.0" resolved "https://registry.npmjs.org/msw/-/msw-0.35.0.tgz#18a4ceb6c822ef226a30421d434413bc45030d38" @@ -27640,7 +27681,7 @@ type-fest@^0.8.1: resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== -type-fest@^1.2.2: +type-fest@^1.1.3, type-fest@^1.2.2: version "1.4.0" resolved "https://registry.npmjs.org/type-fest/-/type-fest-1.4.0.tgz#e9fb813fe3bf1744ec359d55d1affefa76f14be1" integrity sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA== @@ -29155,6 +29196,11 @@ xmlchars@^2.2.0: resolved "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw== +xmldom@^0.6.0: + version "0.6.0" + resolved "https://registry.npmjs.org/xmldom/-/xmldom-0.6.0.tgz#43a96ecb8beece991cef382c08397d82d4d0c46f" + integrity sha512-iAcin401y58LckRZ0TkI4k0VSM1Qg0KGSc3i8rU+xrxe19A/BN1zHyVSJY7uoutVlaTSzYyk/v5AmkewAP7jtg== + xpath@0.0.32: version "0.0.32" resolved "https://registry.npmjs.org/xpath/-/xpath-0.0.32.tgz#1b73d3351af736e17ec078d6da4b8175405c48af"