paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 39 Packages Projects Releases Wiki Activity

Merge pull request #4587 from backstage/orkohunter/techdocs-fix-publishers-on-windows

Browse Source
This commit is contained in:
Himanshu Mishra
2021-02-19 11:29:06 +01:00
committed by GitHub
parent d8bce7cdc7 5851bd5d2b
commit 75b63e92b0
10 changed files with 527 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/techdocs-wild-zoos-do.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/techdocs-common': patch
---
Fix AWS, GCS and Azure publisher to work on Windows.
packages/techdocs-common/__mocks__/@azure/storage-blob.ts
+23 -2
View File
@@ -13,11 +13,32 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import type {
BlobUploadCommonResponse,
ContainerGetPropertiesResponse,
} from '@azure/storage-blob';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
/**
* @param sourceFile Relative path to entity root dir. Contains either / or \ as file separator
* depending upon the OS.
*/
const checkFileExists = async (sourceFile: string): Promise<boolean> => {
// sourceFile will always have / as file separator irrespective of OS since Azure expects /.
// Normalize sourceFile to OS specific path before checking if file exists.
const relativeFilePath = sourceFile.split(path.posix.sep).join(path.sep);
const filePath = path.join(rootDir, sourceFile);
try {
await fs.access(filePath, fs.constants.F_OK);
return true;
} catch (err) {
return false;
}
};
export class BlockBlobClient {
private readonly blobName;
@@ -39,7 +60,7 @@ export class BlockBlobClient {
}
exists() {
return Promise.resolve(fs.existsSync(this.blobName));
return checkFileExists(this.blobName);
}
}
packages/techdocs-common/__mocks__/@google-cloud/storage.ts
+67 -2
View File
@@ -13,10 +13,67 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EventEmitter } from 'events';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
type storageOptions = {
keyFilename?: string;
};
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
/**
* @param sourceFile Absolute path. Contains either / or \ as file separator depending upon the OS.
*/
const checkFileExists = async (sourceFile: string): Promise<boolean> => {
// sourceFile will always have / as file separator irrespective of OS since GCS expects /.
// Normalize sourceFile to OS specific path before checking if file exists.
const filePath = sourceFile.split(path.posix.sep).join(path.sep);
try {
await fs.access(filePath, fs.constants.F_OK);
return true;
} catch (err) {
return false;
}
};
class GCSFile {
private readonly localFilePath: string;
constructor(private readonly destinationFilePath: string) {
this.destinationFilePath = destinationFilePath;
this.localFilePath = path.join(rootDir, this.destinationFilePath);
}
exists() {
return new Promise(async (resolve, reject) => {
if (await checkFileExists(this.localFilePath)) {
resolve([true]);
} else {
reject();
}
});
}
createReadStream() {
const emitter = new EventEmitter();
process.nextTick(() => {
if (fs.existsSync(this.localFilePath)) {
emitter.emit('data', Buffer.from(fs.readFileSync(this.localFilePath)));
emitter.emit('end');
} else {
emitter.emit(
'error',
new Error(`The file ${this.localFilePath} does not exist !`),
);
}
});
return emitter;
}
}
class Bucket {
private readonly bucketName;
@@ -31,10 +88,18 @@ class Bucket {
}
upload(source: string, { destination }) {
return new Promise(resolve => {
resolve({ source, destination });
return new Promise(async (resolve, reject) => {
if (await checkFileExists(source)) {
resolve({ source, destination });
} else {
reject(`Source file ${source} does not exist.`);
}
});
}
file(destinationFilePath: string) {
return new GCSFile(destinationFilePath);
}
}
export class Storage {
packages/techdocs-common/__mocks__/aws-sdk.ts
+53 -32
View File
@@ -15,7 +15,29 @@
*/
import type { S3 as S3Types } from 'aws-sdk';
import { EventEmitter } from 'events';
import fs from 'fs';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
/**
* @param Key Relative path to entity root dir. Contains either / or \ as file separator
* depending upon the OS.
*/
const checkFileExists = async (Key: string): Promise<boolean> => {
// Key will always have / as file separator irrespective of OS since S3 expects /.
// Normalize Key to OS specific path before checking if file exists.
const relativeFilePath = Key.split(path.posix.sep).join(path.sep);
const filePath = path.join(rootDir, Key);
try {
await fs.access(filePath, fs.constants.F_OK);
return true;
} catch (err) {
return false;
}
};
export class S3 {
private readonly options;
@@ -26,39 +48,34 @@ export class S3 {
headObject({ Key }: { Key: string }) {
return {
promise: () => this.checkFileExists(Key),
};
}
getObject({ Key }: { Key: string }) {
return {
promise: () => this.checkFileExists(Key),
createReadStream: () => {
const emitter = new EventEmitter();
process.nextTick(() => {
if (fs.existsSync(Key)) {
emitter.emit('data', Buffer.from(fs.readFileSync(Key)));
} else {
emitter.emit(
'error',
new Error(`The file ${Key} doest not exist !`),
);
}
emitter.emit('end');
});
return emitter;
promise: async () => {
if (!(await checkFileExists(Key))) {
throw new Error('File does not exist');
}
},
};
}
checkFileExists(Key: string) {
return new Promise((resolve, reject) => {
if (fs.existsSync(Key)) {
resolve('');
} else {
reject({ message: 'The object doest not exist !' });
}
});
getObject({ Key }: { Key: string }) {
const filePath = path.join(rootDir, Key);
return {
promise: () => checkFileExists(filePath),
createReadStream: () => {
const emitter = new EventEmitter();
process.nextTick(() => {
if (fs.existsSync(filePath)) {
emitter.emit('data', Buffer.from(fs.readFileSync(filePath)));
emitter.emit('end');
} else {
emitter.emit(
'error',
new Error(`The file ${filePath} does not exist !`),
);
}
});
return emitter;
},
};
}
headBucket() {
@@ -70,8 +87,12 @@ export class S3 {
upload({ Key }: { Key: string }) {
return {
promise: () =>
new Promise((resolve, reject) => {
resolve('');
new Promise(async (resolve, reject) => {
if (!(await checkFileExists(Key))) {
reject(`The file ${Key} does not exist`);
} else {
resolve('');
}
}),
};
}
packages/techdocs-common/src/stages/publish/awsS3.test.ts
+50 -22
View File
@@ -13,14 +13,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { Entity, EntityName } from '@backstage/catalog-model';
import {
Entity,
EntityName,
ENTITY_DEFAULT_NAMESPACE,
} from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import mockFs from 'mock-fs';
import os from 'os';
import path from 'path';
import * as winston from 'winston';
import { AwsS3Publish } from './awsS3';
import { PublisherBase, TechDocsMetadata } from './types';
// NOTE: /packages/techdocs-common/__mocks__ is being used to mock aws-sdk client library
const createMockEntity = (annotations = {}): Entity => {
return {
apiVersion: 'version',
@@ -41,13 +48,15 @@ const createMockEntityName = (): EntityName => ({
namespace: 'test-namespace',
});
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
const getEntityRootDir = (entity: Entity) => {
const {
kind,
metadata: { namespace, name },
} = entity;
const entityRootDir = path.join(namespace as string, kind, name);
return entityRootDir;
return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
};
const logger = winston.createLogger();
@@ -57,6 +66,7 @@ jest.spyOn(logger, 'error').mockReturnValue(logger);
let publisher: PublisherBase;
beforeEach(() => {
mockFs.restore();
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
@@ -78,7 +88,7 @@ beforeEach(() => {
describe('AwsS3Publish', () => {
describe('publish', () => {
it('should publish a directory', async () => {
beforeEach(() => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
@@ -91,6 +101,15 @@ describe('AwsS3Publish', () => {
},
},
});
});
afterEach(() => {
mockFs.restore();
});
it('should publish a directory', async () => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
expect(
await publisher.publish({
@@ -98,41 +117,43 @@ describe('AwsS3Publish', () => {
directory: entityRootDir,
}),
).toBeUndefined();
mockFs.restore();
});
it('should fail to publish a directory', async () => {
expect.assertions(3);
const wrongPathToGeneratedDirectory = path.join(
rootDir,
'wrong',
'path',
'to',
'generatedDirectory',
);
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': '',
'404.html': '',
assets: {
'main.css': '',
},
},
});
const entity = createMockEntity();
await expect(
publisher.publish({
entity,
directory: wrongPathToGeneratedDirectory,
}),
).rejects.toThrowError();
await publisher
.publish({
entity,
directory: wrongPathToGeneratedDirectory,
})
.catch(error =>
.catch(error => {
expect(error.message).toEqual(
// Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error
// Issue reported https://github.com/tschaub/mock-fs/issues/118
expect.stringContaining(
'Unable to upload file(s) to AWS S3. Error Failed to read template directory: ENOENT, no such file or directory',
`Unable to upload file(s) to AWS S3. Error: Failed to read template directory: ENOENT, no such file or directory`,
),
),
);
);
expect(error.message).toEqual(
expect.stringContaining(wrongPathToGeneratedDirectory),
);
});
mockFs.restore();
});
});
@@ -205,12 +226,19 @@ describe('AwsS3Publish', () => {
it('should return an error if the techdocs_metadata.json file is not present', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
await publisher
.fetchTechDocsMetadata(entityNameMock)
.catch(error =>
expect(error.message).toEqual(
expect.stringContaining('TechDocs metadata fetch'),
expect(error).toEqual(
new Error(
`TechDocs metadata fetch failed, The file ${path.join(
entityRootDir,
'techdocs_metadata.json',
)} does not exist !`,
),
),
);
});
packages/techdocs-common/src/stages/publish/awsS3.ts
+12 -3
View File
@@ -140,9 +140,18 @@ export class AwsS3Publish implements PublisherBase {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
const relativeFilePath = filePath.replace(`${directory}/`, '');
const relativeFilePath = path.relative(directory, filePath);
// Convert destination file path to a POSIX path for uploading.
// S3 expects / as path separator and relativeFilePath will contain \\ on Windows.
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
const relativeFilePathPosix = relativeFilePath
.split(path.sep)
.join(path.posix.sep);
// The / delimiter is intentional since it represents the cloud storage and not the local file system.
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const destination = `${entityRootDir}/${relativeFilePath}`; // S3 Bucket file relative path
const destination = `${entityRootDir}/${relativeFilePathPosix}`; // S3 Bucket file relative path
const fileContent = await fs.readFile(filePath, 'utf8');
@@ -164,7 +173,7 @@ export class AwsS3Publish implements PublisherBase {
);
return;
} catch (e) {
const errorMessage = `Unable to upload file(s) to AWS S3. Error ${e.message}`;
const errorMessage = `Unable to upload file(s) to AWS S3. ${e}`;
this.logger.error(errorMessage);
throw new Error(errorMessage);
}
packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
+60 -46
View File
@@ -13,13 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import mockFs from 'mock-fs';
import { ConfigReader } from '@backstage/config';
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import mockFs from 'mock-fs';
import os from 'os';
import path from 'path';
import { AzureBlobStoragePublish } from './azureBlobStorage';
import { PublisherBase } from './types';
import type { Entity } from '@backstage/catalog-model';
import type { Logger } from 'winston';
// NOTE: /packages/techdocs-common/__mocks__ is being used to mock Azure client library
const createMockEntity = (annotations = {}) => {
return {
@@ -35,13 +38,14 @@ const createMockEntity = (annotations = {}) => {
};
};
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
const getEntityRootDir = (entity: Entity) => {
const {
kind,
metadata: { namespace, name },
} = entity;
const entityRootDir = `${namespace}/${kind}/${name}`;
return entityRootDir;
return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
};
function createLogger() {
@@ -52,34 +56,33 @@ function createLogger() {
}
let publisher: PublisherBase;
describe('publishing with valid credentials', () => {
let logger: Logger;
beforeEach(async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'accountName',
accountKey: 'accountKey',
},
containerName: 'containerName',
beforeEach(async () => {
mockFs.restore();
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'accountName',
accountKey: 'accountKey',
},
containerName: 'containerName',
},
},
});
logger = createLogger();
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
},
});
publisher = await AzureBlobStoragePublish.fromConfig(
mockConfig,
createLogger(),
);
});
describe('publishing with valid credentials', () => {
describe('publish', () => {
it('should publish a directory', async () => {
beforeEach(() => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
@@ -92,6 +95,11 @@ describe('publishing with valid credentials', () => {
},
},
});
});
it('should publish a directory', async () => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
expect(
await publisher.publish({
@@ -103,30 +111,33 @@ describe('publishing with valid credentials', () => {
});
it('should fail to publish a directory', async () => {
const wrongPathToGeneratedDirectory = 'wrong/path/to/generatedDirectory';
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
expect.assertions(1);
const wrongPathToGeneratedDirectory = path.join(
rootDir,
'wrong',
'path',
'to',
'generatedDirectory',
);
mockFs({
[entityRootDir]: {
'index.html': '',
'404.html': '',
assets: {
'main.css': '',
},
},
});
const entity = createMockEntity();
await publisher
.publish({
entity,
directory: wrongPathToGeneratedDirectory,
})
.catch(error =>
expect(error.message).toContain(
'Unable to upload file(s) to Azure Blob Storage. Failed to read template directory: ENOENT, no such file or directory',
),
);
.catch(error => {
// Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error
// Issue reported https://github.com/tschaub/mock-fs/issues/118
expect.stringContaining(
`Unable to upload file(s) to Azure Blob Storage. Error: Failed to read template directory: ENOENT, no such file or directory`,
);
expect(error.message).toEqual(
expect.stringContaining(wrongPathToGeneratedDirectory),
);
});
mockFs.restore();
});
});
@@ -235,7 +246,10 @@ describe('error reporting', () => {
expect(logger.error).toHaveBeenCalledWith(
expect.stringContaining(
`Unable to upload file(s) to Azure Blob Storage. Upload failed for test-namespace/TestKind/test-component-name/index.html with status code 500`,
`Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join(
entityRootDir,
'index.html',
)} with status code 500`,
),
);
packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
+26 -18
View File
@@ -13,20 +13,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import platformPath from 'path';
import express from 'express';
import { DefaultAzureCredential } from '@azure/identity';
import {
BlobServiceClient,
StorageSharedKeyCredential,
} from '@azure/storage-blob';
import { DefaultAzureCredential } from '@azure/identity';
import { Logger } from 'winston';
import { Entity, EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
import limiterFactory from 'p-limit';
import express from 'express';
import JSON5 from 'json5';
import limiterFactory from 'p-limit';
import { default as path, default as platformPath } from 'path';
import { Logger } from 'winston';
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
// The number of batches that may be ongoing at the same time.
const BATCH_CONCURRENCY = 3;
@@ -126,27 +126,35 @@ export class AzureBlobStoragePublish implements PublisherBase {
// or start thrashing.
const limiter = limiterFactory(BATCH_CONCURRENCY);
const promises = allFilesToUpload.map(filePath => {
const promises = allFilesToUpload.map(sourceFilePath => {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
const relativeFilePath = filePath.replace(`${directory}/`, '');
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const destination = platformPath.normalize(
`${entityRootDir}/${relativeFilePath}`,
); // Azure Blob Storage Container file relative path
const relativeFilePath = path.normalize(
path.relative(directory, sourceFilePath),
);
// Convert destination file path to a POSIX path for uploading.
// Azure Blob Storage expects / as path separator and relativeFilePath will contain \\ on Windows.
// https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#blob-names
const relativeFilePathPosix = relativeFilePath
.split(path.sep)
.join(path.posix.sep);
// The / delimiter is intentional since it represents the cloud storage and not the local file system.
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Azure Blob Storage Container file relative path
return limiter(async () => {
const response = await this.storageClient
.getContainerClient(this.containerName)
.getBlockBlobClient(destination)
.uploadFile(filePath);
.uploadFile(sourceFilePath);
if (response._response.status >= 400) {
return {
...response,
error: new Error(
`Upload failed for ${filePath} with status code ${response._response.status}`,
`Upload failed for ${sourceFilePath} with status code ${response._response.status}`,
),
};
}
@@ -173,18 +181,18 @@ export class AzureBlobStoragePublish implements PublisherBase {
);
}
} catch (e) {
const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e.message}`;
const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e}`;
this.logger.error(errorMessage);
throw new Error(errorMessage);
}
}
private download(containerName: string, path: string): Promise<Buffer> {
private download(containerName: string, blobPath: string): Promise<Buffer> {
return new Promise((resolve, reject) => {
const fileStreamChunks: Array<any> = [];
this.storageClient
.getContainerClient(containerName)
.getBlockBlobClient(path)
.getBlockBlobClient(blobPath)
.download()
.then(res => {
const body = res.readableStreamBody;
packages/techdocs-common/src/stages/publish/googleStorage.test.ts
+191 -21
View File
@@ -13,18 +13,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import mockFs from 'mock-fs';
import * as winston from 'winston';
import { getVoidLogger } from '@backstage/backend-common';
import {
Entity,
EntityName,
ENTITY_DEFAULT_NAMESPACE,
} from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import mockFs from 'mock-fs';
import os from 'os';
import path from 'path';
import { GoogleGCSPublish } from './googleStorage';
import { PublisherBase } from './types';
import { PublisherBase, TechDocsMetadata } from './types';
const createMockEntity = (annotations = {}) => {
// NOTE: /packages/techdocs-common/__mocks__ is being used to mock Google Cloud Storage client library
const createMockEntity = (annotations = {}): Entity => {
return {
apiVersion: 'version',
kind: 'TestKind',
metadata: {
name: 'test-component-name',
namespace: 'test-namespace',
annotations: {
...annotations,
},
@@ -32,12 +42,30 @@ const createMockEntity = (annotations = {}) => {
};
};
const logger = winston.createLogger();
const createMockEntityName = (): EntityName => ({
kind: 'TestKind',
name: 'test-component-name',
namespace: 'test-namespace',
});
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
const getEntityRootDir = (entity: Entity) => {
const {
kind,
metadata: { namespace, name },
} = entity;
return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
};
const logger = getVoidLogger();
jest.spyOn(logger, 'info').mockReturnValue(logger);
let publisher: PublisherBase;
beforeEach(async () => {
mockFs.restore();
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
@@ -55,24 +83,166 @@ beforeEach(async () => {
});
describe('GoogleGCSPublish', () => {
it('should publish a directory', async () => {
mockFs({
'/path/to/generatedDirectory': {
'index.html': '',
'404.html': '',
assets: {
'main.css': '',
describe('publish', () => {
beforeEach(() => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': '',
'404.html': '',
assets: {
'main.css': '',
},
},
},
});
});
const entity = createMockEntity();
expect(
await publisher.publish({
entity,
directory: '/path/to/generatedDirectory',
}),
).toBeUndefined();
mockFs.restore();
afterEach(() => {
mockFs.restore();
});
it('should publish a directory', async () => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
expect(
await publisher.publish({
entity,
directory: entityRootDir,
}),
).toBeUndefined();
mockFs.restore();
});
it('should fail to publish a directory', async () => {
expect.assertions(3);
const wrongPathToGeneratedDirectory = path.join(
rootDir,
'wrong',
'path',
'to',
'generatedDirectory',
);
const entity = createMockEntity();
await expect(
publisher.publish({
entity,
directory: wrongPathToGeneratedDirectory,
}),
).rejects.toThrowError();
await publisher
.publish({
entity,
directory: wrongPathToGeneratedDirectory,
})
.catch(error => {
expect(error.message).toEqual(
// Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error
// Issue reported https://github.com/tschaub/mock-fs/issues/118
expect.stringContaining(
`Unable to upload file(s) to Google Cloud Storage. Error: Failed to read template directory: ENOENT, no such file or directory`,
),
);
expect(error.message).toEqual(
expect.stringContaining(wrongPathToGeneratedDirectory),
);
});
mockFs.restore();
});
});
describe('hasDocsBeenGenerated', () => {
it('should return true if docs has been generated', async () => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': 'file-content',
},
});
expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
mockFs.restore();
});
it('should return false if docs has not been generated', async () => {
const entity = createMockEntity();
expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false);
});
});
describe('fetchTechDocsMetadata', () => {
beforeEach(() => {
mockFs.restore();
});
it('should return tech docs metadata', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'techdocs_metadata.json':
'{"site_name": "backstage", "site_description": "site_content"}',
},
});
const expectedMetadata: TechDocsMetadata = {
site_name: 'backstage',
site_description: 'site_content',
};
expect(
await publisher.fetchTechDocsMetadata(entityNameMock),
).toStrictEqual(expectedMetadata);
});
it('should return tech docs metadata when json encoded with single quotes', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'techdocs_metadata.json':
"{'site_name': 'backstage', 'site_description': 'site_content'}",
},
});
const expectedMetadata: TechDocsMetadata = {
site_name: 'backstage',
site_description: 'site_content',
};
expect(
await publisher.fetchTechDocsMetadata(entityNameMock),
).toStrictEqual(expectedMetadata);
mockFs.restore();
});
it('should return an error if the techdocs_metadata.json file is not present', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
await publisher
.fetchTechDocsMetadata(entityNameMock)
.catch(errorMessage =>
expect(errorMessage).toEqual(
`The file ${path.join(
entityRootDir,
'techdocs_metadata.json',
)} does not exist !`,
),
);
});
});
});
packages/techdocs-common/src/stages/publish/googleStorage.ts
+40 -34
View File
@@ -13,20 +13,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import path from 'path';
import express from 'express';
import {
Storage,
UploadResponse,
FileExistsResponse,
} from '@google-cloud/storage';
import { Logger } from 'winston';
import { Entity, EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
import {
FileExistsResponse,
Storage,
UploadResponse,
} from '@google-cloud/storage';
import express from 'express';
import JSON5 from 'json5';
import createLimiter from 'p-limit';
import path from 'path';
import { Logger } from 'winston';
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
export class GoogleGCSPublish implements PublisherBase {
static async fromConfig(
@@ -97,44 +97,50 @@ export class GoogleGCSPublish implements PublisherBase {
* Upload all the files from the generated `directory` to the GCS bucket.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
*/
publish({ entity, directory }: PublishRequest): Promise<void> {
return new Promise(async (resolve, reject) => {
async publish({ entity, directory }: PublishRequest): Promise<void> {
try {
// Note: GCS manages creation of parent directories if they do not exist.
// So collecting path of only the files is good enough.
const allFilesToUpload = await getFileTreeRecursively(directory);
const limiter = createLimiter(10);
const uploadPromises: Array<Promise<UploadResponse>> = [];
allFilesToUpload.forEach(filePath => {
allFilesToUpload.forEach(sourceFilePath => {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
const relativeFilePath = filePath.replace(`${directory}/`, '');
const relativeFilePath = path.relative(directory, sourceFilePath);
// Convert destination file path to a POSIX path for uploading.
// GCS expects / as path separator and relativeFilePath will contain \\ on Windows.
// https://cloud.google.com/storage/docs/gsutil/addlhelp/HowSubdirectoriesWork
const relativeFilePathPosix = relativeFilePath
.split(path.sep)
.join(path.posix.sep);
// The / delimiter is intentional since it represents the cloud storage and not the local file system.
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const destination = `${entityRootDir}/${relativeFilePath}`; // GCS Bucket file relative path
const destination = `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path
// Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
const uploadFile = limiter(() =>
this.storageClient
.bucket(this.bucketName)
.upload(filePath, { destination }),
this.storageClient.bucket(this.bucketName).upload(sourceFilePath, {
destination,
}),
);
uploadPromises.push(uploadFile);
});
Promise.all(uploadPromises)
.then(() => {
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
);
resolve(undefined);
})
.catch((err: Error) => {
const errorMessage = `Unable to upload file(s) to Google Cloud Storage. Error ${err.message}`;
this.logger.error(errorMessage);
reject(errorMessage);
});
});
await Promise.all(uploadPromises);
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
);
} catch (e) {
const errorMessage = `Unable to upload file(s) to Google Cloud Storage. ${e}`;
this.logger.error(errorMessage);
throw new Error(errorMessage);
}
}
fetchTechDocsMetadata(entityName: EntityName): Promise<TechDocsMetadata> {
@@ -154,9 +160,9 @@ export class GoogleGCSPublish implements PublisherBase {
fileStreamChunks.push(chunk);
})
.on('end', () => {
const techdocsMetadataJson = Buffer.concat(
fileStreamChunks,
).toString();
const techdocsMetadataJson = Buffer.concat(fileStreamChunks).toString(
'utf-8',
);
resolve(JSON5.parse(techdocsMetadataJson));
});
});