paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 44 Packages Projects Releases Wiki Activity

OpenStack Swift SDK changed from "pkgcloud" to Trendyol's own OpenStack Swift SDK (#6839)

Browse Source
This commit is contained in:
Mert Can Bilgiç
2021-08-25 23:16:10 +03:00
committed by GitHub
parent 8b066c47d5
commit 58452cdb72
10 changed files with 542 additions and 699 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/techdocs-common/__mocks__/@trendyol-js/openstack-swift-sdk.ts
+103
View File
@@ -0,0 +1,103 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
import {
ContainerMetaResponse,
DownloadResponse,
NotFound,
ObjectMetaResponse,
UploadResponse,
} from '@trendyol-js/openstack-swift-sdk';
import { Stream, Readable } from 'stream';
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
const checkFileExists = async (Key: string): Promise<boolean> => {
// Key will always have / as file separator irrespective of OS since cloud providers expects /.
// Normalize Key to OS specific path before checking if file exists.
const filePath = path.join(rootDir, Key);
try {
await fs.access(filePath, fs.constants.F_OK);
return true;
} catch (err) {
return false;
}
};
const streamToBuffer = (stream: Stream | Readable): Promise<Buffer> => {
return new Promise((resolve, reject) => {
try {
const chunks: any[] = [];
stream.on('data', chunk => chunks.push(chunk));
stream.on('error', reject);
stream.on('end', () => resolve(Buffer.concat(chunks)));
} catch (e) {
throw new Error(`Unable to parse the response data ${e.message}`);
}
});
};
export class SwiftClient {
async getMetadata(_containerName: string, file: string) {
const fileExists = await checkFileExists(file);
if (fileExists) {
return new ObjectMetaResponse({
fullPath: file,
});
}
return new NotFound();
}
async getContainerMetadata(containerName: string) {
if (containerName === 'mock') {
return new ContainerMetaResponse({
size: 10,
});
}
return new NotFound();
}
async upload(_containerName: string, destination: string, stream: Readable) {
try {
const filePath = path.join(rootDir, destination);
const fileBuffer = await streamToBuffer(stream);
await fs.writeFile(filePath, fileBuffer);
const fileExists = await checkFileExists(destination);
if (fileExists) {
return new UploadResponse(filePath);
}
const errorMessage = `Unable to upload file(s) to OpenStack Swift.`;
throw new Error(errorMessage);
} catch (error) {
const errorMessage = `Unable to upload file(s) to OpenStack Swift. ${error}`;
throw new Error(errorMessage);
}
}
async download(_containerName: string, file: string) {
const filePath = path.join(rootDir, file);
const fileExists = await checkFileExists(file);
if (!fileExists) {
return new NotFound();
}
return new DownloadResponse([], fs.createReadStream(filePath));
}
}
packages/techdocs-common/__mocks__/pkgcloud.ts
-108
View File
@@ -1,108 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EventEmitter } from 'events';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
import { ClientError } from 'pkgcloud';
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
const checkFileExists = async (Key: string): Promise<boolean> => {
// Key will always have / as file separator irrespective of OS since cloud providers expects /.
// Normalize Key to OS specific path before checking if file exists.
const filePath = path.join(rootDir, Key);
try {
fs.accessSync(filePath, fs.constants.F_OK);
return true;
} catch (err) {
return false;
}
};
class PkgCloudStorageClient {
getFile(
containerName: string,
file: string,
callback: (err: any, file: any) => any,
) {
checkFileExists(file).then(res => {
if (!res) {
callback('File does not exist', undefined);
} else {
callback(undefined, 'success');
}
});
}
getContainer(
containerName: string,
callback: (err: ClientError, container: any) => any,
) {
if (containerName !== 'mock') {
callback(new Error('Container does not exist'), undefined);
} else {
callback(undefined, 'success');
}
}
upload({ remote }: { remote: string }) {
const filePath = path.join(rootDir, remote);
const emitter = new EventEmitter();
process.nextTick(() => {
if (fs.existsSync(filePath)) {
emitter.emit('success');
(emitter as any).end = () => true;
} else {
emitter.emit(
'error',
new Error(`The file ${filePath} does not exist !`),
);
}
});
return emitter;
}
download({ remote }: { remote: string }) {
const filePath = path.join(rootDir, remote);
const emitter = new EventEmitter();
process.nextTick(() => {
if (fs.existsSync(filePath)) {
emitter.emit('data', Buffer.from(fs.readFileSync(filePath)));
emitter.emit('end');
} else {
emitter.emit(
'error',
new Error(`The file ${filePath} does not exist !`),
);
}
});
return emitter;
}
}
export class storage {
static createClient() {
return new PkgCloudStorageClient();
}
}
packages/techdocs-common/package.json
+1 -2
View File
@@ -44,6 +44,7 @@
"@backstage/errors": "^0.1.1",
"@backstage/integration": "^0.6.0",
"@google-cloud/storage": "^5.6.0",
"@trendyol-js/openstack-swift-sdk": "^0.0.4",
"@types/express": "^4.17.6",
"aws-sdk": "^2.840.0",
"express": "^4.17.1",
@@ -53,7 +54,6 @@
"mime-types": "^2.1.27",
"mock-fs": "^4.13.0",
"p-limit": "^3.1.0",
"pkgcloud": "^2.2.0",
"recursive-readdir": "^2.2.2",
"winston": "^3.2.1"
},
@@ -63,7 +63,6 @@
"@types/js-yaml": "^4.0.0",
"@types/mime-types": "^2.1.0",
"@types/mock-fs": "^4.13.0",
"@types/pkgcloud": "^1.7.4",
"@types/recursive-readdir": "^2.2.0",
"@types/supertest": "^2.0.8",
"supertest": "^6.1.3"
packages/techdocs-common/src/stages/publish/openStackSwift.test.ts
+7 -7
View File
@@ -29,7 +29,7 @@ import path from 'path';
import { OpenStackSwiftPublish } from './openStackSwift';
import { PublisherBase, TechDocsMetadata } from './types';
// NOTE: /packages/techdocs-common/__mocks__ is being used to mock pkgcloud client library
// NOTE: /packages/techdocs-common/__mocks__ is being used to mock @trendyol-js/openstack-swift-sdk client library
const createMockEntity = (annotations = {}): Entity => {
return {
@@ -75,11 +75,11 @@ beforeEach(() => {
type: 'openStackSwift',
openStackSwift: {
credentials: {
username: 'mockuser',
password: 'verystrongpass',
id: 'mockid',
secret: 'verystrongsecret',
},
authUrl: 'mockauthurl',
region: 'mockregion',
swiftUrl: 'mockSwiftUrl',
containerName: 'mock',
},
},
@@ -105,11 +105,11 @@ describe('OpenStackSwiftPublish', () => {
type: 'openStackSwift',
openStackSwift: {
credentials: {
username: 'mockuser',
password: 'verystrongpass',
id: 'mockId',
secret: 'mockSecret',
},
authUrl: 'mockauthurl',
region: 'mockregion',
swiftUrl: 'mockSwiftUrl',
containerName: 'errorBucket',
},
},
packages/techdocs-common/src/stages/publish/openStackSwift.ts
+101 -102
View File
@@ -20,8 +20,9 @@ import fs from 'fs-extra';
import JSON5 from 'json5';
import createLimiter from 'p-limit';
import path from 'path';
import { storage } from 'pkgcloud';
import { Readable } from 'stream';
import { SwiftClient } from '@trendyol-js/openstack-swift-sdk';
import { NotFound } from '@trendyol-js/openstack-swift-sdk/lib/types';
import { Stream, Readable } from 'stream';
import { Logger } from 'winston';
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
import {
@@ -31,7 +32,7 @@ import {
TechDocsMetadata,
} from './types';
const streamToBuffer = (stream: Readable): Promise<Buffer> => {
const streamToBuffer = (stream: Stream | Readable): Promise<Buffer> => {
return new Promise((resolve, reject) => {
try {
const chunks: any[] = [];
@@ -44,6 +45,13 @@ const streamToBuffer = (stream: Readable): Promise<Buffer> => {
});
};
const bufferToStream = (buffer: Buffer): Readable => {
const stream = new Readable();
stream.push(buffer);
stream.push(null);
return stream;
};
export class OpenStackSwiftPublish implements PublisherBase {
static fromConfig(config: Config, logger: Logger): PublisherBase {
let containerName = '';
@@ -62,24 +70,18 @@ export class OpenStackSwiftPublish implements PublisherBase {
'techdocs.publisher.openStackSwift',
);
const storageClient = storage.createClient({
provider: 'openstack',
username: openStackSwiftConfig.getString('credentials.username'),
password: openStackSwiftConfig.getString('credentials.password'),
authUrl: openStackSwiftConfig.getString('authUrl'),
keystoneAuthVersion:
openStackSwiftConfig.getOptionalString('keystoneAuthVersion') || 'v3',
domainId: openStackSwiftConfig.getOptionalString('domainId') || 'default',
domainName:
openStackSwiftConfig.getOptionalString('domainName') || 'Default',
region: openStackSwiftConfig.getString('region'),
const storageClient = new SwiftClient({
authEndpoint: openStackSwiftConfig.getString('authUrl'),
swiftEndpoint: openStackSwiftConfig.getString('swiftUrl'),
credentialId: openStackSwiftConfig.getString('credentials.id'),
secret: openStackSwiftConfig.getString('credentials.secret'),
});
return new OpenStackSwiftPublish(storageClient, containerName, logger);
}
constructor(
private readonly storageClient: storage.Client,
private readonly storageClient: SwiftClient,
private readonly containerName: string,
private readonly logger: Logger,
) {
@@ -92,31 +94,35 @@ export class OpenStackSwiftPublish implements PublisherBase {
* Check if the defined container exists. Being able to connect means the configuration is good
* and the storage client will work.
*/
getReadiness(): Promise<ReadinessResponse> {
return new Promise(resolve => {
this.storageClient.getContainer(this.containerName, (err, container) => {
if (container) {
this.logger.info(
`Successfully connected to the OpenStack Swift container ${this.containerName}.`,
);
resolve({
isAvailable: true,
});
} else {
this.logger.error(
`Could not retrieve metadata about the OpenStack Swift container ${this.containerName}. ` +
'Make sure the container exists. Also make sure that authentication is setup either by ' +
'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
async getReadiness(): Promise<ReadinessResponse> {
try {
const container = await this.storageClient.getContainerMetadata(
this.containerName,
);
this.logger.error(`from OpenStack client library: ${err.message}`);
resolve({
isAvailable: false,
});
}
});
});
if (!(container instanceof NotFound)) {
this.logger.info(
`Successfully connected to the OpenStack Swift container ${this.containerName}.`,
);
return {
isAvailable: true,
};
}
this.logger.error(
`Could not retrieve metadata about the OpenStack Swift container ${this.containerName}. ` +
'Make sure the container exists. Also make sure that authentication is setup either by ' +
'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
return {
isAvailable: false,
};
} catch (err) {
this.logger.error(`from OpenStack client library: ${err.message}`);
return {
isAvailable: false,
};
}
}
/**
@@ -135,7 +141,6 @@ export class OpenStackSwiftPublish implements PublisherBase {
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
const relativeFilePath = path.relative(directory, filePath);
// Convert destination file path to a POSIX path for uploading.
// Swift expects / as path separator and relativeFilePath will contain \\ on Windows.
// https://docs.openstack.org/python-openstackclient/pike/cli/man/openstack.html
@@ -147,26 +152,16 @@ export class OpenStackSwiftPublish implements PublisherBase {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Swift container file relative path
const params = {
container: this.containerName,
remote: destination,
};
// Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
const uploadFile = limiter(
() =>
new Promise((res, rej) => {
const readStream = fs.createReadStream(filePath);
const writeStream = this.storageClient.upload(params);
writeStream.on('error', rej);
writeStream.on('success', res);
readStream.pipe(writeStream);
}),
);
const uploadFile = limiter(async () => {
const fileBuffer = await fs.readFile(filePath);
const stream = bufferToStream(fileBuffer);
return this.storageClient.upload(
this.containerName,
destination,
stream,
);
});
uploadPromises.push(uploadFile);
}
await Promise.all(uploadPromises);
@@ -184,15 +179,16 @@ export class OpenStackSwiftPublish implements PublisherBase {
async fetchTechDocsMetadata(
entityName: EntityName,
): Promise<TechDocsMetadata> {
try {
return await new Promise<TechDocsMetadata>(async (resolve, reject) => {
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
return await new Promise<TechDocsMetadata>(async (resolve, reject) => {
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
const stream = this.storageClient.download({
container: this.containerName,
remote: `${entityRootDir}/techdocs_metadata.json`,
});
const downloadResponse = await this.storageClient.download(
this.containerName,
`${entityRootDir}/techdocs_metadata.json`,
);
if (!(downloadResponse instanceof NotFound)) {
const stream = downloadResponse.data;
try {
const techdocsMetadataJson = await streamToBuffer(stream);
if (!techdocsMetadataJson) {
@@ -210,10 +206,12 @@ export class OpenStackSwiftPublish implements PublisherBase {
this.logger.error(err.message);
reject(new Error(err.message));
}
});
} catch (e) {
throw new Error(`TechDocs metadata fetch failed, ${e.message}`);
}
} else {
reject({
message: `TechDocs metadata fetch failed, The file /rootDir/${entityRootDir}/techdocs_metadata.json does not exist !`,
});
}
});
}
/**
@@ -229,23 +227,29 @@ export class OpenStackSwiftPublish implements PublisherBase {
const fileExtension = path.extname(filePath);
const responseHeaders = getHeadersForFileExtension(fileExtension);
const stream = this.storageClient.download({
container: this.containerName,
remote: filePath,
});
const downloadResponse = await this.storageClient.download(
this.containerName,
filePath,
);
try {
// Inject response headers
for (const [headerKey, headerValue] of Object.entries(
responseHeaders,
)) {
res.setHeader(headerKey, headerValue);
if (!(downloadResponse instanceof NotFound)) {
const stream = downloadResponse.data;
try {
// Inject response headers
for (const [headerKey, headerValue] of Object.entries(
responseHeaders,
)) {
res.setHeader(headerKey, headerValue);
}
res.send(await streamToBuffer(stream));
} catch (err) {
this.logger.warn(err.message);
res.status(404).send(err.message);
}
res.send(await streamToBuffer(stream));
} catch (err) {
this.logger.warn(err.message);
res.status(404).send(err.message);
} else {
res.status(404).send('File Not Found');
}
};
}
@@ -255,25 +259,20 @@ export class OpenStackSwiftPublish implements PublisherBase {
* can be used to verify if there are any pre-generated docs available to serve.
*/
async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
try {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const fileResponse = await this.storageClient.getMetadata(
this.containerName,
`${entityRootDir}/index.html`,
);
return new Promise(res => {
this.storageClient.getFile(
this.containerName,
`${entityRootDir}/index.html`,
(err, file) => {
if (!err && file) {
res(true);
} else {
res(false);
this.logger.warn(err.message);
}
},
);
});
} catch (e) {
return Promise.resolve(false);
if (!(fileResponse instanceof NotFound)) {
return true;
}
return false;
} catch (err) {
this.logger.warn(err.message);
return false;
}
}
}
packages/techdocs-common/src/stages/publish/publish.test.ts
+3 -3
View File
@@ -171,11 +171,11 @@ describe('Publisher', () => {
type: 'openStackSwift',
openStackSwift: {
credentials: {
username: 'mockuser',
password: 'verystrongpass',
id: 'mockId',
secret: 'mockSecret',
},
authUrl: 'mockauthurl',
region: 'mockregion',
swiftUrl: 'mockSwiftUrl',
containerName: 'mock',
},
},