OSS lowercase migration added
Signed-off-by: Mert Can Bilgiç <mert.bilgic@trendyol.com>
This commit is contained in:
Mert Can Bilgiç
@@ -45,7 +45,7 @@
|
||||
"@backstage/search-common": "^0.2.1",
|
||||
"@backstage/integration": "^0.6.9",
|
||||
"@google-cloud/storage": "^5.6.0",
|
||||
"@trendyol-js/openstack-swift-sdk": "^0.0.4",
|
||||
"@trendyol-js/openstack-swift-sdk": "^0.0.5",
|
||||
"@types/express": "^4.17.6",
|
||||
"aws-sdk": "^2.840.0",
|
||||
"express": "^4.17.1",
|
||||
|
||||
@@ -24,7 +24,7 @@ import { SwiftClient } from '@trendyol-js/openstack-swift-sdk';
|
||||
import { NotFound } from '@trendyol-js/openstack-swift-sdk/lib/types';
|
||||
import { Stream, Readable } from 'stream';
|
||||
import { Logger } from 'winston';
|
||||
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
|
||||
import { getFileTreeRecursively, getHeadersForFileExtension, lowerCaseEntityTripletInStoragePath } from './helpers';
|
||||
import {
|
||||
PublisherBase,
|
||||
PublishRequest,
|
||||
@@ -77,7 +77,7 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
"Since techdocs.publisher.type is set to 'openStackSwift' in your app config, " +
|
||||
'techdocs.publisher.openStackSwift.containerName is required.',
|
||||
'techdocs.publisher.openStackSwift.containerName is required.',
|
||||
);
|
||||
}
|
||||
|
||||
@@ -115,9 +115,9 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
}
|
||||
this.logger.error(
|
||||
`Could not retrieve metadata about the OpenStack Swift container ${this.containerName}. ` +
|
||||
'Make sure the container exists. Also make sure that authentication is setup either by ' +
|
||||
'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
|
||||
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
'Make sure the container exists. Also make sure that authentication is setup either by ' +
|
||||
'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
|
||||
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
return {
|
||||
isAvailable: false,
|
||||
@@ -289,4 +289,60 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async migrateDocsCase({
|
||||
removeOriginal = false,
|
||||
concurrency = 25,
|
||||
}): Promise<void> {
|
||||
// Iterate through every file in the root of the publisher.
|
||||
const allObjects = await this.getAllObjectsFromContainer();
|
||||
const limiter = createLimiter(concurrency);
|
||||
await Promise.all(
|
||||
allObjects.map(f =>
|
||||
limiter(async file => {
|
||||
let newPath;
|
||||
try {
|
||||
newPath = lowerCaseEntityTripletInStoragePath(file);
|
||||
} catch (e) {
|
||||
assertError(e);
|
||||
this.logger.warn(e.message);
|
||||
return;
|
||||
}
|
||||
|
||||
// If all parts are already lowercase, ignore.
|
||||
if (file === newPath) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
this.logger.verbose(`Migrating ${file} to ${newPath}`);
|
||||
await this.storageClient.copy(this.containerName, file, this.containerName, newPath);
|
||||
if (removeOriginal) {
|
||||
await this.storageClient.delete(this.containerName, file);
|
||||
}
|
||||
} catch (e) {
|
||||
assertError(e);
|
||||
this.logger.warn(`Unable to migrate ${file}: ${e.message}`);
|
||||
}
|
||||
}, f),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a list of all object keys from the configured container.
|
||||
*/
|
||||
protected async getAllObjectsFromContainer(
|
||||
{ prefix } = { prefix: '' },
|
||||
): Promise<string[]> {
|
||||
let objects: string[] = [];
|
||||
let allObjects: any;
|
||||
const OSS_MAX_LIMIT = Math.pow(2, 31) - 1;
|
||||
|
||||
allObjects = await this.storageClient.list(this.containerName, prefix, OSS_MAX_LIMIT)
|
||||
objects = allObjects.map((object: any) => object = object.name)
|
||||
|
||||
return objects;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4613,7 +4613,7 @@
|
||||
call-me-maybe "^1.0.1"
|
||||
glob-to-regexp "^0.3.0"
|
||||
|
||||
"@mswjs/cookies@^0.1.6":
|
||||
"@mswjs/cookies@^0.1.5", "@mswjs/cookies@^0.1.6":
|
||||
version "0.1.6"
|
||||
resolved "https://registry.npmjs.org/@mswjs/cookies/-/cookies-0.1.6.tgz#176f77034ab6d7373ae5c94bcbac36fee8869249"
|
||||
integrity sha512-A53XD5TOfwhpqAmwKdPtg1dva5wrng2gH5xMvklzbd9WLTSVU953eCRa8rtrrm6G7Cy60BOGsBRN89YQK0mlKA==
|
||||
@@ -4621,6 +4621,17 @@
|
||||
"@types/set-cookie-parser" "^2.4.0"
|
||||
set-cookie-parser "^2.4.6"
|
||||
|
||||
"@mswjs/interceptors@^0.10.0":
|
||||
version "0.10.0"
|
||||
resolved "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.10.0.tgz#f5aad03c2c0591d164e3ed178b21942f1c2f8061"
|
||||
integrity sha512-/M0GGpid5q2EDI+Keas1sLYF3VZFXHDE5gCmX/jHdp+OJFruVNca3PUk7A8KnGdPpuycZogdPsmRBSOXwjyA7A==
|
||||
dependencies:
|
||||
"@open-draft/until" "^1.0.3"
|
||||
debug "^4.3.0"
|
||||
headers-utils "^3.0.2"
|
||||
strict-event-emitter "^0.2.0"
|
||||
xmldom "^0.6.0"
|
||||
|
||||
"@mswjs/interceptors@^0.12.6":
|
||||
version "0.12.7"
|
||||
resolved "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.12.7.tgz#0d1cd4cd31a0f663e0455993951201faa09d0909"
|
||||
@@ -6695,10 +6706,10 @@
|
||||
resolved "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
|
||||
integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==
|
||||
|
||||
"@trendyol-js/openstack-swift-sdk@^0.0.4":
|
||||
version "0.0.4"
|
||||
resolved "https://registry.npmjs.org/@trendyol-js/openstack-swift-sdk/-/openstack-swift-sdk-0.0.4.tgz#570c6ab950319156c175ace005b4fb4d9f895d47"
|
||||
integrity sha512-9YKOjov+V+yzptei6+B9QPuC5pOMTBTg/NQpb1ZbxvlOaYpWU4HHpSH2BkIFYZ8vYyAfzFNG1T2rjpQ2ZQDUtQ==
|
||||
"@trendyol-js/openstack-swift-sdk@^0.0.5":
|
||||
version "0.0.5"
|
||||
resolved "https://registry.npmjs.org/@trendyol-js/openstack-swift-sdk/-/openstack-swift-sdk-0.0.5.tgz#65be3c42b8dbafc57f2f2a46c327e2ad51e5a70e"
|
||||
integrity sha512-KS5nz0cvd35UUyMzhZm+btGV4prtA1KNE7CCMOGBdVxoMGl06Qidli3HgHoc2I9jLPmky1SPp5yzQUwrsyWa0g==
|
||||
dependencies:
|
||||
agentkeepalive "^4.1.4"
|
||||
axios "^0.21.1"
|
||||
@@ -6912,7 +6923,7 @@
|
||||
dependencies:
|
||||
"@types/express" "*"
|
||||
|
||||
"@types/cookie@^0.4.1":
|
||||
"@types/cookie@^0.4.0", "@types/cookie@^0.4.1":
|
||||
version "0.4.1"
|
||||
resolved "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz#bfd02c1f2224567676c1545199f87c3a861d878d"
|
||||
integrity sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q==
|
||||
@@ -12586,7 +12597,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.0, debug@^2.6.9:
|
||||
dependencies:
|
||||
ms "2.0.0"
|
||||
|
||||
debug@4, debug@4.3.2, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2:
|
||||
debug@4, debug@4.3.2, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.0, debug@^4.3.1, debug@^4.3.2:
|
||||
version "4.3.2"
|
||||
resolved "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b"
|
||||
integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==
|
||||
@@ -15722,6 +15733,11 @@ graphql@^15.3.0:
|
||||
resolved "https://registry.npmjs.org/graphql/-/graphql-15.5.1.tgz#f2f84415d8985e7b84731e7f3536f8bb9d383aad"
|
||||
integrity sha512-FeTRX67T3LoE3LWAxxOlW2K3Bz+rMYAC18rRguK4wgXaTZMiJwSUwDmPFo3UadAKbzirKIg5Qy+sNJXbpPRnQw==
|
||||
|
||||
graphql@^15.4.0:
|
||||
version "15.7.2"
|
||||
resolved "https://registry.npmjs.org/graphql/-/graphql-15.7.2.tgz#85ab0eeb83722977151b3feb4d631b5f2ab287ef"
|
||||
integrity sha512-AnnKk7hFQFmU/2I9YSQf3xw44ctnSFCfp3zE0N6W174gqe9fWG/2rKaKxROK7CcI3XtERpjEKFqts8o319Kf7A==
|
||||
|
||||
graphql@^15.5.1:
|
||||
version "15.6.1"
|
||||
resolved "https://registry.npmjs.org/graphql/-/graphql-15.6.1.tgz#9125bdf057553525da251e19e96dab3d3855ddfc"
|
||||
@@ -16649,7 +16665,7 @@ inquirer@^8.0.0:
|
||||
strip-ansi "^6.0.0"
|
||||
through "^2.3.6"
|
||||
|
||||
inquirer@^8.1.1:
|
||||
inquirer@^8.1.0, inquirer@^8.1.1:
|
||||
version "8.2.0"
|
||||
resolved "https://registry.npmjs.org/inquirer/-/inquirer-8.2.0.tgz#f44f008dd344bbfc4b30031f45d984e034a3ac3a"
|
||||
integrity sha512-0crLweprevJ02tTuA6ThpoAERAGyVILC4sS74uib58Xf/zSr1/ZWtmm7D5CI+bSQEaA04f0K7idaHpQbSWgiVQ==
|
||||
@@ -20678,6 +20694,31 @@ msal@^1.0.2:
|
||||
dependencies:
|
||||
tslib "^1.9.3"
|
||||
|
||||
msw@^0.29.0:
|
||||
version "0.29.0"
|
||||
resolved "https://registry.npmjs.org/msw/-/msw-0.29.0.tgz#7242d575cb01db0c925241587df1fc2b79230d78"
|
||||
integrity sha512-C/wz1d5uAEZRvAPAYrXG1rwLxXl0+BOs+JPrCzasoABZW3ATwS6ifSze+/DAgA93e9M86RXwvy6yDtZeZWmCFQ==
|
||||
dependencies:
|
||||
"@mswjs/cookies" "^0.1.5"
|
||||
"@mswjs/interceptors" "^0.10.0"
|
||||
"@open-draft/until" "^1.0.3"
|
||||
"@types/cookie" "^0.4.0"
|
||||
"@types/inquirer" "^7.3.1"
|
||||
"@types/js-levenshtein" "^1.1.0"
|
||||
chalk "^4.1.1"
|
||||
chokidar "^3.4.2"
|
||||
cookie "^0.4.1"
|
||||
graphql "^15.4.0"
|
||||
headers-utils "^3.0.2"
|
||||
inquirer "^8.1.0"
|
||||
js-levenshtein "^1.1.6"
|
||||
node-fetch "^2.6.1"
|
||||
node-match-path "^0.6.3"
|
||||
statuses "^2.0.0"
|
||||
strict-event-emitter "^0.2.0"
|
||||
type-fest "^1.1.3"
|
||||
yargs "^17.0.1"
|
||||
|
||||
msw@^0.35.0:
|
||||
version "0.35.0"
|
||||
resolved "https://registry.npmjs.org/msw/-/msw-0.35.0.tgz#18a4ceb6c822ef226a30421d434413bc45030d38"
|
||||
@@ -27640,7 +27681,7 @@ type-fest@^0.8.1:
|
||||
resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d"
|
||||
integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==
|
||||
|
||||
type-fest@^1.2.2:
|
||||
type-fest@^1.1.3, type-fest@^1.2.2:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.npmjs.org/type-fest/-/type-fest-1.4.0.tgz#e9fb813fe3bf1744ec359d55d1affefa76f14be1"
|
||||
integrity sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==
|
||||
@@ -29155,6 +29196,11 @@ xmlchars@^2.2.0:
|
||||
resolved "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"
|
||||
integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==
|
||||
|
||||
xmldom@^0.6.0:
|
||||
version "0.6.0"
|
||||
resolved "https://registry.npmjs.org/xmldom/-/xmldom-0.6.0.tgz#43a96ecb8beece991cef382c08397d82d4d0c46f"
|
||||
integrity sha512-iAcin401y58LckRZ0TkI4k0VSM1Qg0KGSc3i8rU+xrxe19A/BN1zHyVSJY7uoutVlaTSzYyk/v5AmkewAP7jtg==
|
||||
|
||||
xpath@0.0.32:
|
||||
version "0.0.32"
|
||||
resolved "https://registry.npmjs.org/xpath/-/xpath-0.0.32.tgz#1b73d3351af736e17ec078d6da4b8175405c48af"
|
||||
|
||||
Reference in New Issue
Block a user