OpenStack Swift SDK changed from "pkgcloud" to Trendyol's own OpenStack Swift SDK (#6839)
This commit is contained in:
@@ -0,0 +1,64 @@
|
||||
---
|
||||
'@backstage/techdocs-common': minor
|
||||
'@backstage/plugin-techdocs-backend': minor
|
||||
---
|
||||
|
||||
OpenStack Swift Client changed with Trendyol's OpenStack Swift SDK.
|
||||
|
||||
## Migration from old OpenStack Swift Configuration
|
||||
|
||||
Let's assume we have the old OpenStack Swift configuration here.
|
||||
|
||||
```yaml
|
||||
techdocs:
|
||||
publisher:
|
||||
type: 'openStackSwift'
|
||||
openStackSwift:
|
||||
containerName: 'name-of-techdocs-storage-bucket'
|
||||
credentials:
|
||||
username: ${OPENSTACK_SWIFT_STORAGE_USERNAME}
|
||||
password: ${OPENSTACK_SWIFT_STORAGE_PASSWORD}
|
||||
authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL}
|
||||
keystoneAuthVersion: ${OPENSTACK_SWIFT_STORAGE_AUTH_VERSION}
|
||||
domainId: ${OPENSTACK_SWIFT_STORAGE_DOMAIN_ID}
|
||||
domainName: ${OPENSTACK_SWIFT_STORAGE_DOMAIN_NAME}
|
||||
region: ${OPENSTACK_SWIFT_STORAGE_REGION}
|
||||
```
|
||||
|
||||
##### Step 1: Change the credential keys
|
||||
|
||||
Since the new SDK uses _Application Credentials_ to authenticate OpenStack, we
|
||||
need to change the keys `credentials.username` to `credentials.id`,
|
||||
`credentials.password` to `credentials.secret` and use Application Credential ID
|
||||
and secret here. For more detail about credentials look
|
||||
[here](https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail,authenticating-with-an-application-credential-detail#authenticating-with-an-application-credential).
|
||||
|
||||
##### Step 2: Remove the unused keys
|
||||
|
||||
Since the new SDK doesn't use the old way authentication, we don't need the keys
|
||||
`openStackSwift.keystoneAuthVersion`, `openStackSwift.domainId`,
|
||||
`openStackSwift.domainName` and `openStackSwift.region`. So you can remove them.
|
||||
|
||||
##### Step 3: Add Swift URL
|
||||
|
||||
The new SDK needs the OpenStack Swift connection URL for connecting the Swift.
|
||||
So you need to add a new key called `openStackSwift.swiftUrl` and give the
|
||||
OpenStack Swift url here. Example url should look like that:
|
||||
`https://example.com:6780/swift/v1`
|
||||
|
||||
##### That's it!
|
||||
|
||||
Your new configuration should look like that!
|
||||
|
||||
```yaml
|
||||
techdocs:
|
||||
publisher:
|
||||
type: 'openStackSwift'
|
||||
openStackSwift:
|
||||
containerName: 'name-of-techdocs-storage-bucket'
|
||||
credentials:
|
||||
id: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_ID}
|
||||
secret: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_SECRET}
|
||||
authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL}
|
||||
swiftUrl: ${OPENSTACK_SWIFT_STORAGE_SWIFT_URL}
|
||||
```
|
||||
@@ -399,9 +399,34 @@ techdocs:
|
||||
|
||||
Set the configs in your `app-config.yaml` to point to your container name.
|
||||
|
||||
https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail#password-authentication-with-unscoped-authorization
|
||||
https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail,authenticating-with-an-application-credential-detail#authenticating-with-an-application-credential
|
||||
for more details.
|
||||
|
||||
```yaml
|
||||
techdocs:
|
||||
publisher:
|
||||
type: 'openStackSwift'
|
||||
openStackSwift:
|
||||
containerName: 'name-of-techdocs-storage-bucket'
|
||||
credentials:
|
||||
id: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_ID}
|
||||
secret: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_SECRET}
|
||||
authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL}
|
||||
swiftUrl: ${OPENSTACK_SWIFT_STORAGE_SWIFT_URL}
|
||||
```
|
||||
|
||||
**4. That's it!**
|
||||
|
||||
Your Backstage app is now ready to use OpenStack Swift Storage for TechDocs, to
|
||||
store and read the static generated documentation files. When you start the
|
||||
backend of the app, you should be able to see
|
||||
`techdocs info Successfully connected to the OpenStack Swift Storage container`
|
||||
in the logs.
|
||||
|
||||
## Bonus: Migration from old OpenStack Swift Configuration
|
||||
|
||||
Let's assume we have the old OpenStack Swift configuration here.
|
||||
|
||||
```yaml
|
||||
techdocs:
|
||||
publisher:
|
||||
@@ -418,10 +443,40 @@ techdocs:
|
||||
region: ${OPENSTACK_SWIFT_STORAGE_REGION}
|
||||
```
|
||||
|
||||
**4. That's it!**
|
||||
##### Step 1: Change the credential keys
|
||||
|
||||
Your Backstage app is now ready to use OpenStack Swift Storage for TechDocs, to
|
||||
store and read the static generated documentation files. When you start the
|
||||
backend of the app, you should be able to see
|
||||
`techdocs info Successfully connected to the OpenStack Swift Storage container`
|
||||
in the logs.
|
||||
Since the new SDK uses _Application Credentials_ to authenticate OpenStack, we
|
||||
need to change the keys `credentials.username` to `credentials.id`,
|
||||
`credentials.password` to `credentials.secret` and use Application Credential ID
|
||||
and secret here. For more detail about credentials look
|
||||
[here](https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail,authenticating-with-an-application-credential-detail#authenticating-with-an-application-credential).
|
||||
|
||||
##### Step 2: Remove the unused keys
|
||||
|
||||
Since the new SDK doesn't use the old way authentication, we don't need the keys
|
||||
`openStackSwift.keystoneAuthVersion`, `openStackSwift.domainId`,
|
||||
`openStackSwift.domainName` and `openStackSwift.region`. So you can remove them.
|
||||
|
||||
##### Step 3: Add Swift URL
|
||||
|
||||
The new SDK needs the OpenStack Swift connection URL for connecting the Swift.
|
||||
So you need to add a new key called `openStackSwift.swiftUrl` and give the
|
||||
OpenStack Swift url here. Example url should look like that:
|
||||
`https://example.com:6780/swift/v1`
|
||||
|
||||
##### That's it!
|
||||
|
||||
Your new configuration should look like that!
|
||||
|
||||
```yaml
|
||||
techdocs:
|
||||
publisher:
|
||||
type: 'openStackSwift'
|
||||
openStackSwift:
|
||||
containerName: 'name-of-techdocs-storage-bucket'
|
||||
credentials:
|
||||
id: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_ID}
|
||||
secret: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_SECRET}
|
||||
authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL}
|
||||
swiftUrl: ${OPENSTACK_SWIFT_STORAGE_SWIFT_URL}
|
||||
```
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import fs from 'fs-extra';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import {
|
||||
ContainerMetaResponse,
|
||||
DownloadResponse,
|
||||
NotFound,
|
||||
ObjectMetaResponse,
|
||||
UploadResponse,
|
||||
} from '@trendyol-js/openstack-swift-sdk';
|
||||
import { Stream, Readable } from 'stream';
|
||||
|
||||
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
|
||||
|
||||
const checkFileExists = async (Key: string): Promise<boolean> => {
|
||||
// Key will always have / as file separator irrespective of OS since cloud providers expects /.
|
||||
// Normalize Key to OS specific path before checking if file exists.
|
||||
const filePath = path.join(rootDir, Key);
|
||||
|
||||
try {
|
||||
await fs.access(filePath, fs.constants.F_OK);
|
||||
return true;
|
||||
} catch (err) {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const streamToBuffer = (stream: Stream | Readable): Promise<Buffer> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
const chunks: any[] = [];
|
||||
stream.on('data', chunk => chunks.push(chunk));
|
||||
stream.on('error', reject);
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to parse the response data ${e.message}`);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
export class SwiftClient {
|
||||
async getMetadata(_containerName: string, file: string) {
|
||||
const fileExists = await checkFileExists(file);
|
||||
if (fileExists) {
|
||||
return new ObjectMetaResponse({
|
||||
fullPath: file,
|
||||
});
|
||||
}
|
||||
return new NotFound();
|
||||
}
|
||||
|
||||
async getContainerMetadata(containerName: string) {
|
||||
if (containerName === 'mock') {
|
||||
return new ContainerMetaResponse({
|
||||
size: 10,
|
||||
});
|
||||
}
|
||||
return new NotFound();
|
||||
}
|
||||
|
||||
async upload(_containerName: string, destination: string, stream: Readable) {
|
||||
try {
|
||||
const filePath = path.join(rootDir, destination);
|
||||
const fileBuffer = await streamToBuffer(stream);
|
||||
|
||||
await fs.writeFile(filePath, fileBuffer);
|
||||
const fileExists = await checkFileExists(destination);
|
||||
|
||||
if (fileExists) {
|
||||
return new UploadResponse(filePath);
|
||||
}
|
||||
const errorMessage = `Unable to upload file(s) to OpenStack Swift.`;
|
||||
throw new Error(errorMessage);
|
||||
} catch (error) {
|
||||
const errorMessage = `Unable to upload file(s) to OpenStack Swift. ${error}`;
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
async download(_containerName: string, file: string) {
|
||||
const filePath = path.join(rootDir, file);
|
||||
const fileExists = await checkFileExists(file);
|
||||
if (!fileExists) {
|
||||
return new NotFound();
|
||||
}
|
||||
return new DownloadResponse([], fs.createReadStream(filePath));
|
||||
}
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { EventEmitter } from 'events';
|
||||
import fs from 'fs-extra';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { ClientError } from 'pkgcloud';
|
||||
|
||||
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
|
||||
|
||||
const checkFileExists = async (Key: string): Promise<boolean> => {
|
||||
// Key will always have / as file separator irrespective of OS since cloud providers expects /.
|
||||
// Normalize Key to OS specific path before checking if file exists.
|
||||
const filePath = path.join(rootDir, Key);
|
||||
|
||||
try {
|
||||
fs.accessSync(filePath, fs.constants.F_OK);
|
||||
return true;
|
||||
} catch (err) {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
class PkgCloudStorageClient {
|
||||
getFile(
|
||||
containerName: string,
|
||||
file: string,
|
||||
callback: (err: any, file: any) => any,
|
||||
) {
|
||||
checkFileExists(file).then(res => {
|
||||
if (!res) {
|
||||
callback('File does not exist', undefined);
|
||||
} else {
|
||||
callback(undefined, 'success');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
getContainer(
|
||||
containerName: string,
|
||||
callback: (err: ClientError, container: any) => any,
|
||||
) {
|
||||
if (containerName !== 'mock') {
|
||||
callback(new Error('Container does not exist'), undefined);
|
||||
} else {
|
||||
callback(undefined, 'success');
|
||||
}
|
||||
}
|
||||
|
||||
upload({ remote }: { remote: string }) {
|
||||
const filePath = path.join(rootDir, remote);
|
||||
|
||||
const emitter = new EventEmitter();
|
||||
|
||||
process.nextTick(() => {
|
||||
if (fs.existsSync(filePath)) {
|
||||
emitter.emit('success');
|
||||
(emitter as any).end = () => true;
|
||||
} else {
|
||||
emitter.emit(
|
||||
'error',
|
||||
new Error(`The file ${filePath} does not exist !`),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return emitter;
|
||||
}
|
||||
|
||||
download({ remote }: { remote: string }) {
|
||||
const filePath = path.join(rootDir, remote);
|
||||
|
||||
const emitter = new EventEmitter();
|
||||
|
||||
process.nextTick(() => {
|
||||
if (fs.existsSync(filePath)) {
|
||||
emitter.emit('data', Buffer.from(fs.readFileSync(filePath)));
|
||||
emitter.emit('end');
|
||||
} else {
|
||||
emitter.emit(
|
||||
'error',
|
||||
new Error(`The file ${filePath} does not exist !`),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return emitter;
|
||||
}
|
||||
}
|
||||
|
||||
export class storage {
|
||||
static createClient() {
|
||||
return new PkgCloudStorageClient();
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,7 @@
|
||||
"@backstage/errors": "^0.1.1",
|
||||
"@backstage/integration": "^0.6.0",
|
||||
"@google-cloud/storage": "^5.6.0",
|
||||
"@trendyol-js/openstack-swift-sdk": "^0.0.4",
|
||||
"@types/express": "^4.17.6",
|
||||
"aws-sdk": "^2.840.0",
|
||||
"express": "^4.17.1",
|
||||
@@ -53,7 +54,6 @@
|
||||
"mime-types": "^2.1.27",
|
||||
"mock-fs": "^4.13.0",
|
||||
"p-limit": "^3.1.0",
|
||||
"pkgcloud": "^2.2.0",
|
||||
"recursive-readdir": "^2.2.2",
|
||||
"winston": "^3.2.1"
|
||||
},
|
||||
@@ -63,7 +63,6 @@
|
||||
"@types/js-yaml": "^4.0.0",
|
||||
"@types/mime-types": "^2.1.0",
|
||||
"@types/mock-fs": "^4.13.0",
|
||||
"@types/pkgcloud": "^1.7.4",
|
||||
"@types/recursive-readdir": "^2.2.0",
|
||||
"@types/supertest": "^2.0.8",
|
||||
"supertest": "^6.1.3"
|
||||
|
||||
@@ -29,7 +29,7 @@ import path from 'path';
|
||||
import { OpenStackSwiftPublish } from './openStackSwift';
|
||||
import { PublisherBase, TechDocsMetadata } from './types';
|
||||
|
||||
// NOTE: /packages/techdocs-common/__mocks__ is being used to mock pkgcloud client library
|
||||
// NOTE: /packages/techdocs-common/__mocks__ is being used to mock @trendyol-js/openstack-swift-sdk client library
|
||||
|
||||
const createMockEntity = (annotations = {}): Entity => {
|
||||
return {
|
||||
@@ -75,11 +75,11 @@ beforeEach(() => {
|
||||
type: 'openStackSwift',
|
||||
openStackSwift: {
|
||||
credentials: {
|
||||
username: 'mockuser',
|
||||
password: 'verystrongpass',
|
||||
id: 'mockid',
|
||||
secret: 'verystrongsecret',
|
||||
},
|
||||
authUrl: 'mockauthurl',
|
||||
region: 'mockregion',
|
||||
swiftUrl: 'mockSwiftUrl',
|
||||
containerName: 'mock',
|
||||
},
|
||||
},
|
||||
@@ -105,11 +105,11 @@ describe('OpenStackSwiftPublish', () => {
|
||||
type: 'openStackSwift',
|
||||
openStackSwift: {
|
||||
credentials: {
|
||||
username: 'mockuser',
|
||||
password: 'verystrongpass',
|
||||
id: 'mockId',
|
||||
secret: 'mockSecret',
|
||||
},
|
||||
authUrl: 'mockauthurl',
|
||||
region: 'mockregion',
|
||||
swiftUrl: 'mockSwiftUrl',
|
||||
containerName: 'errorBucket',
|
||||
},
|
||||
},
|
||||
|
||||
@@ -20,8 +20,9 @@ import fs from 'fs-extra';
|
||||
import JSON5 from 'json5';
|
||||
import createLimiter from 'p-limit';
|
||||
import path from 'path';
|
||||
import { storage } from 'pkgcloud';
|
||||
import { Readable } from 'stream';
|
||||
import { SwiftClient } from '@trendyol-js/openstack-swift-sdk';
|
||||
import { NotFound } from '@trendyol-js/openstack-swift-sdk/lib/types';
|
||||
import { Stream, Readable } from 'stream';
|
||||
import { Logger } from 'winston';
|
||||
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
|
||||
import {
|
||||
@@ -31,7 +32,7 @@ import {
|
||||
TechDocsMetadata,
|
||||
} from './types';
|
||||
|
||||
const streamToBuffer = (stream: Readable): Promise<Buffer> => {
|
||||
const streamToBuffer = (stream: Stream | Readable): Promise<Buffer> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
const chunks: any[] = [];
|
||||
@@ -44,6 +45,13 @@ const streamToBuffer = (stream: Readable): Promise<Buffer> => {
|
||||
});
|
||||
};
|
||||
|
||||
const bufferToStream = (buffer: Buffer): Readable => {
|
||||
const stream = new Readable();
|
||||
stream.push(buffer);
|
||||
stream.push(null);
|
||||
return stream;
|
||||
};
|
||||
|
||||
export class OpenStackSwiftPublish implements PublisherBase {
|
||||
static fromConfig(config: Config, logger: Logger): PublisherBase {
|
||||
let containerName = '';
|
||||
@@ -62,24 +70,18 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
'techdocs.publisher.openStackSwift',
|
||||
);
|
||||
|
||||
const storageClient = storage.createClient({
|
||||
provider: 'openstack',
|
||||
username: openStackSwiftConfig.getString('credentials.username'),
|
||||
password: openStackSwiftConfig.getString('credentials.password'),
|
||||
authUrl: openStackSwiftConfig.getString('authUrl'),
|
||||
keystoneAuthVersion:
|
||||
openStackSwiftConfig.getOptionalString('keystoneAuthVersion') || 'v3',
|
||||
domainId: openStackSwiftConfig.getOptionalString('domainId') || 'default',
|
||||
domainName:
|
||||
openStackSwiftConfig.getOptionalString('domainName') || 'Default',
|
||||
region: openStackSwiftConfig.getString('region'),
|
||||
const storageClient = new SwiftClient({
|
||||
authEndpoint: openStackSwiftConfig.getString('authUrl'),
|
||||
swiftEndpoint: openStackSwiftConfig.getString('swiftUrl'),
|
||||
credentialId: openStackSwiftConfig.getString('credentials.id'),
|
||||
secret: openStackSwiftConfig.getString('credentials.secret'),
|
||||
});
|
||||
|
||||
return new OpenStackSwiftPublish(storageClient, containerName, logger);
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly storageClient: storage.Client,
|
||||
private readonly storageClient: SwiftClient,
|
||||
private readonly containerName: string,
|
||||
private readonly logger: Logger,
|
||||
) {
|
||||
@@ -92,31 +94,35 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
* Check if the defined container exists. Being able to connect means the configuration is good
|
||||
* and the storage client will work.
|
||||
*/
|
||||
getReadiness(): Promise<ReadinessResponse> {
|
||||
return new Promise(resolve => {
|
||||
this.storageClient.getContainer(this.containerName, (err, container) => {
|
||||
if (container) {
|
||||
this.logger.info(
|
||||
`Successfully connected to the OpenStack Swift container ${this.containerName}.`,
|
||||
);
|
||||
resolve({
|
||||
isAvailable: true,
|
||||
});
|
||||
} else {
|
||||
this.logger.error(
|
||||
`Could not retrieve metadata about the OpenStack Swift container ${this.containerName}. ` +
|
||||
'Make sure the container exists. Also make sure that authentication is setup either by ' +
|
||||
'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
|
||||
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
async getReadiness(): Promise<ReadinessResponse> {
|
||||
try {
|
||||
const container = await this.storageClient.getContainerMetadata(
|
||||
this.containerName,
|
||||
);
|
||||
|
||||
this.logger.error(`from OpenStack client library: ${err.message}`);
|
||||
resolve({
|
||||
isAvailable: false,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
if (!(container instanceof NotFound)) {
|
||||
this.logger.info(
|
||||
`Successfully connected to the OpenStack Swift container ${this.containerName}.`,
|
||||
);
|
||||
return {
|
||||
isAvailable: true,
|
||||
};
|
||||
}
|
||||
this.logger.error(
|
||||
`Could not retrieve metadata about the OpenStack Swift container ${this.containerName}. ` +
|
||||
'Make sure the container exists. Also make sure that authentication is setup either by ' +
|
||||
'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
|
||||
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
return {
|
||||
isAvailable: false,
|
||||
};
|
||||
} catch (err) {
|
||||
this.logger.error(`from OpenStack client library: ${err.message}`);
|
||||
return {
|
||||
isAvailable: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -135,7 +141,6 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
// Path of all files to upload, relative to the root of the source directory
|
||||
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
|
||||
const relativeFilePath = path.relative(directory, filePath);
|
||||
|
||||
// Convert destination file path to a POSIX path for uploading.
|
||||
// Swift expects / as path separator and relativeFilePath will contain \\ on Windows.
|
||||
// https://docs.openstack.org/python-openstackclient/pike/cli/man/openstack.html
|
||||
@@ -147,26 +152,16 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
|
||||
const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Swift container file relative path
|
||||
|
||||
const params = {
|
||||
container: this.containerName,
|
||||
remote: destination,
|
||||
};
|
||||
|
||||
// Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
|
||||
const uploadFile = limiter(
|
||||
() =>
|
||||
new Promise((res, rej) => {
|
||||
const readStream = fs.createReadStream(filePath);
|
||||
|
||||
const writeStream = this.storageClient.upload(params);
|
||||
|
||||
writeStream.on('error', rej);
|
||||
|
||||
writeStream.on('success', res);
|
||||
|
||||
readStream.pipe(writeStream);
|
||||
}),
|
||||
);
|
||||
const uploadFile = limiter(async () => {
|
||||
const fileBuffer = await fs.readFile(filePath);
|
||||
const stream = bufferToStream(fileBuffer);
|
||||
return this.storageClient.upload(
|
||||
this.containerName,
|
||||
destination,
|
||||
stream,
|
||||
);
|
||||
});
|
||||
uploadPromises.push(uploadFile);
|
||||
}
|
||||
await Promise.all(uploadPromises);
|
||||
@@ -184,15 +179,16 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
async fetchTechDocsMetadata(
|
||||
entityName: EntityName,
|
||||
): Promise<TechDocsMetadata> {
|
||||
try {
|
||||
return await new Promise<TechDocsMetadata>(async (resolve, reject) => {
|
||||
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
|
||||
return await new Promise<TechDocsMetadata>(async (resolve, reject) => {
|
||||
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
|
||||
|
||||
const stream = this.storageClient.download({
|
||||
container: this.containerName,
|
||||
remote: `${entityRootDir}/techdocs_metadata.json`,
|
||||
});
|
||||
const downloadResponse = await this.storageClient.download(
|
||||
this.containerName,
|
||||
`${entityRootDir}/techdocs_metadata.json`,
|
||||
);
|
||||
|
||||
if (!(downloadResponse instanceof NotFound)) {
|
||||
const stream = downloadResponse.data;
|
||||
try {
|
||||
const techdocsMetadataJson = await streamToBuffer(stream);
|
||||
if (!techdocsMetadataJson) {
|
||||
@@ -210,10 +206,12 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
this.logger.error(err.message);
|
||||
reject(new Error(err.message));
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`TechDocs metadata fetch failed, ${e.message}`);
|
||||
}
|
||||
} else {
|
||||
reject({
|
||||
message: `TechDocs metadata fetch failed, The file /rootDir/${entityRootDir}/techdocs_metadata.json does not exist !`,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -229,23 +227,29 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
const fileExtension = path.extname(filePath);
|
||||
const responseHeaders = getHeadersForFileExtension(fileExtension);
|
||||
|
||||
const stream = this.storageClient.download({
|
||||
container: this.containerName,
|
||||
remote: filePath,
|
||||
});
|
||||
const downloadResponse = await this.storageClient.download(
|
||||
this.containerName,
|
||||
filePath,
|
||||
);
|
||||
|
||||
try {
|
||||
// Inject response headers
|
||||
for (const [headerKey, headerValue] of Object.entries(
|
||||
responseHeaders,
|
||||
)) {
|
||||
res.setHeader(headerKey, headerValue);
|
||||
if (!(downloadResponse instanceof NotFound)) {
|
||||
const stream = downloadResponse.data;
|
||||
|
||||
try {
|
||||
// Inject response headers
|
||||
for (const [headerKey, headerValue] of Object.entries(
|
||||
responseHeaders,
|
||||
)) {
|
||||
res.setHeader(headerKey, headerValue);
|
||||
}
|
||||
|
||||
res.send(await streamToBuffer(stream));
|
||||
} catch (err) {
|
||||
this.logger.warn(err.message);
|
||||
res.status(404).send(err.message);
|
||||
}
|
||||
|
||||
res.send(await streamToBuffer(stream));
|
||||
} catch (err) {
|
||||
this.logger.warn(err.message);
|
||||
res.status(404).send(err.message);
|
||||
} else {
|
||||
res.status(404).send('File Not Found');
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -255,25 +259,20 @@ export class OpenStackSwiftPublish implements PublisherBase {
|
||||
* can be used to verify if there are any pre-generated docs available to serve.
|
||||
*/
|
||||
async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
|
||||
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
|
||||
try {
|
||||
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
|
||||
const fileResponse = await this.storageClient.getMetadata(
|
||||
this.containerName,
|
||||
`${entityRootDir}/index.html`,
|
||||
);
|
||||
|
||||
return new Promise(res => {
|
||||
this.storageClient.getFile(
|
||||
this.containerName,
|
||||
`${entityRootDir}/index.html`,
|
||||
(err, file) => {
|
||||
if (!err && file) {
|
||||
res(true);
|
||||
} else {
|
||||
res(false);
|
||||
this.logger.warn(err.message);
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
} catch (e) {
|
||||
return Promise.resolve(false);
|
||||
if (!(fileResponse instanceof NotFound)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
} catch (err) {
|
||||
this.logger.warn(err.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,11 +171,11 @@ describe('Publisher', () => {
|
||||
type: 'openStackSwift',
|
||||
openStackSwift: {
|
||||
credentials: {
|
||||
username: 'mockuser',
|
||||
password: 'verystrongpass',
|
||||
id: 'mockId',
|
||||
secret: 'mockSecret',
|
||||
},
|
||||
authUrl: 'mockauthurl',
|
||||
region: 'mockregion',
|
||||
swiftUrl: 'mockSwiftUrl',
|
||||
containerName: 'mock',
|
||||
},
|
||||
},
|
||||
|
||||
Vendored
+6
-22
@@ -137,15 +137,15 @@ export interface Config {
|
||||
*/
|
||||
credentials: {
|
||||
/**
|
||||
* (Required) Root user name
|
||||
* (Required) Application Credential ID
|
||||
* @visibility secret
|
||||
*/
|
||||
username: string;
|
||||
id: string;
|
||||
/**
|
||||
* (Required) Root user password
|
||||
* (Required) Application Credential Secret
|
||||
* @visibility secret
|
||||
*/
|
||||
password: string; // required
|
||||
secret: string; // required
|
||||
};
|
||||
/**
|
||||
* (Required) Cloud Storage Container Name
|
||||
@@ -158,26 +158,10 @@ export interface Config {
|
||||
*/
|
||||
authUrl: string;
|
||||
/**
|
||||
* (Optional) Auth version
|
||||
* If not set, 'v2.0' will be used.
|
||||
* (Required) Swift URL
|
||||
* @visibility backend
|
||||
*/
|
||||
keystoneAuthVersion: string;
|
||||
/**
|
||||
* (Required) Domain Id
|
||||
* @visibility backend
|
||||
*/
|
||||
domainId: string;
|
||||
/**
|
||||
* (Required) Domain Name
|
||||
* @visibility backend
|
||||
*/
|
||||
domainName: string;
|
||||
/**
|
||||
* (Required) Region
|
||||
* @visibility backend
|
||||
*/
|
||||
region: string;
|
||||
swiftUrl: string;
|
||||
};
|
||||
}
|
||||
| {
|
||||
|
||||
Reference in New Issue
Block a user