diff --git a/.changeset/techdocs-wild-zoos-do.md b/.changeset/techdocs-wild-zoos-do.md new file mode 100644 index 0000000000..42a2a6f1c4 --- /dev/null +++ b/.changeset/techdocs-wild-zoos-do.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Fix AWS, GCS and Azure publisher to work on Windows. diff --git a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts index 6633a71b2c..4fc1e4c374 100644 --- a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts +++ b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts @@ -13,11 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import fs from 'fs'; import type { BlobUploadCommonResponse, ContainerGetPropertiesResponse, } from '@azure/storage-blob'; +import fs from 'fs-extra'; +import os from 'os'; +import path from 'path'; + +const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +/** + * @param sourceFile Relative path to entity root dir. Contains either / or \ as file separator + * depending upon the OS. + */ +const checkFileExists = async (sourceFile: string): Promise => { + // sourceFile will always have / as file separator irrespective of OS since Azure expects /. + // Normalize sourceFile to OS specific path before checking if file exists. + const relativeFilePath = sourceFile.split(path.posix.sep).join(path.sep); + const filePath = path.join(rootDir, sourceFile); + + try { + await fs.access(filePath, fs.constants.F_OK); + return true; + } catch (err) { + return false; + } +}; export class BlockBlobClient { private readonly blobName; @@ -39,7 +60,7 @@ export class BlockBlobClient { } exists() { - return Promise.resolve(fs.existsSync(this.blobName)); + return checkFileExists(this.blobName); } } diff --git a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts index b84018c089..264852d31c 100644 --- a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts +++ b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts @@ -13,10 +13,67 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { EventEmitter } from 'events'; +import fs from 'fs-extra'; +import os from 'os'; +import path from 'path'; + type storageOptions = { keyFilename?: string; }; +const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +/** + * @param sourceFile Absolute path. Contains either / or \ as file separator depending upon the OS. + */ +const checkFileExists = async (sourceFile: string): Promise => { + // sourceFile will always have / as file separator irrespective of OS since GCS expects /. + // Normalize sourceFile to OS specific path before checking if file exists. + const filePath = sourceFile.split(path.posix.sep).join(path.sep); + + try { + await fs.access(filePath, fs.constants.F_OK); + return true; + } catch (err) { + return false; + } +}; + +class GCSFile { + private readonly localFilePath: string; + + constructor(private readonly destinationFilePath: string) { + this.destinationFilePath = destinationFilePath; + this.localFilePath = path.join(rootDir, this.destinationFilePath); + } + + exists() { + return new Promise(async (resolve, reject) => { + if (await checkFileExists(this.localFilePath)) { + resolve([true]); + } else { + reject(); + } + }); + } + + createReadStream() { + const emitter = new EventEmitter(); + process.nextTick(() => { + if (fs.existsSync(this.localFilePath)) { + emitter.emit('data', Buffer.from(fs.readFileSync(this.localFilePath))); + emitter.emit('end'); + } else { + emitter.emit( + 'error', + new Error(`The file ${this.localFilePath} does not exist !`), + ); + } + }); + return emitter; + } +} + class Bucket { private readonly bucketName; @@ -31,10 +88,18 @@ class Bucket { } upload(source: string, { destination }) { - return new Promise(resolve => { - resolve({ source, destination }); + return new Promise(async (resolve, reject) => { + if (await checkFileExists(source)) { + resolve({ source, destination }); + } else { + reject(`Source file ${source} does not exist.`); + } }); } + + file(destinationFilePath: string) { + return new GCSFile(destinationFilePath); + } } export class Storage { diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts index f0fb13c642..4717feda9b 100644 --- a/packages/techdocs-common/__mocks__/aws-sdk.ts +++ b/packages/techdocs-common/__mocks__/aws-sdk.ts @@ -15,7 +15,29 @@ */ import type { S3 as S3Types } from 'aws-sdk'; import { EventEmitter } from 'events'; -import fs from 'fs'; +import fs from 'fs-extra'; +import os from 'os'; +import path from 'path'; + +const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; + +/** + * @param Key Relative path to entity root dir. Contains either / or \ as file separator + * depending upon the OS. + */ +const checkFileExists = async (Key: string): Promise => { + // Key will always have / as file separator irrespective of OS since S3 expects /. + // Normalize Key to OS specific path before checking if file exists. + const relativeFilePath = Key.split(path.posix.sep).join(path.sep); + const filePath = path.join(rootDir, Key); + + try { + await fs.access(filePath, fs.constants.F_OK); + return true; + } catch (err) { + return false; + } +}; export class S3 { private readonly options; @@ -26,39 +48,34 @@ export class S3 { headObject({ Key }: { Key: string }) { return { - promise: () => this.checkFileExists(Key), - }; - } - - getObject({ Key }: { Key: string }) { - return { - promise: () => this.checkFileExists(Key), - createReadStream: () => { - const emitter = new EventEmitter(); - process.nextTick(() => { - if (fs.existsSync(Key)) { - emitter.emit('data', Buffer.from(fs.readFileSync(Key))); - } else { - emitter.emit( - 'error', - new Error(`The file ${Key} doest not exist !`), - ); - } - emitter.emit('end'); - }); - return emitter; + promise: async () => { + if (!(await checkFileExists(Key))) { + throw new Error('File does not exist'); + } }, }; } - checkFileExists(Key: string) { - return new Promise((resolve, reject) => { - if (fs.existsSync(Key)) { - resolve(''); - } else { - reject({ message: 'The object doest not exist !' }); - } - }); + getObject({ Key }: { Key: string }) { + const filePath = path.join(rootDir, Key); + return { + promise: () => checkFileExists(filePath), + createReadStream: () => { + const emitter = new EventEmitter(); + process.nextTick(() => { + if (fs.existsSync(filePath)) { + emitter.emit('data', Buffer.from(fs.readFileSync(filePath))); + emitter.emit('end'); + } else { + emitter.emit( + 'error', + new Error(`The file ${filePath} does not exist !`), + ); + } + }); + return emitter; + }, + }; } headBucket() { @@ -70,8 +87,12 @@ export class S3 { upload({ Key }: { Key: string }) { return { promise: () => - new Promise((resolve, reject) => { - resolve(''); + new Promise(async (resolve, reject) => { + if (!(await checkFileExists(Key))) { + reject(`The file ${Key} does not exist`); + } else { + resolve(''); + } }), }; } diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index f8450a07a0..3d7bd337d2 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -13,14 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import type { Entity, EntityName } from '@backstage/catalog-model'; +import { + Entity, + EntityName, + ENTITY_DEFAULT_NAMESPACE, +} from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import mockFs from 'mock-fs'; +import os from 'os'; import path from 'path'; import * as winston from 'winston'; import { AwsS3Publish } from './awsS3'; import { PublisherBase, TechDocsMetadata } from './types'; +// NOTE: /packages/techdocs-common/__mocks__ is being used to mock aws-sdk client library + const createMockEntity = (annotations = {}): Entity => { return { apiVersion: 'version', @@ -41,13 +48,15 @@ const createMockEntityName = (): EntityName => ({ namespace: 'test-namespace', }); +const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; + const getEntityRootDir = (entity: Entity) => { const { kind, metadata: { namespace, name }, } = entity; - const entityRootDir = path.join(namespace as string, kind, name); - return entityRootDir; + + return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name); }; const logger = winston.createLogger(); @@ -57,6 +66,7 @@ jest.spyOn(logger, 'error').mockReturnValue(logger); let publisher: PublisherBase; beforeEach(() => { + mockFs.restore(); const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -78,7 +88,7 @@ beforeEach(() => { describe('AwsS3Publish', () => { describe('publish', () => { - it('should publish a directory', async () => { + beforeEach(() => { const entity = createMockEntity(); const entityRootDir = getEntityRootDir(entity); @@ -91,6 +101,15 @@ describe('AwsS3Publish', () => { }, }, }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('should publish a directory', async () => { + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); expect( await publisher.publish({ @@ -98,41 +117,43 @@ describe('AwsS3Publish', () => { directory: entityRootDir, }), ).toBeUndefined(); - mockFs.restore(); }); it('should fail to publish a directory', async () => { + expect.assertions(3); const wrongPathToGeneratedDirectory = path.join( + rootDir, 'wrong', 'path', 'to', 'generatedDirectory', ); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - mockFs({ - [entityRootDir]: { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', - }, - }, - }); + const entity = createMockEntity(); + await expect( + publisher.publish({ + entity, + directory: wrongPathToGeneratedDirectory, + }), + ).rejects.toThrowError(); await publisher .publish({ entity, directory: wrongPathToGeneratedDirectory, }) - .catch(error => + .catch(error => { expect(error.message).toEqual( + // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error + // Issue reported https://github.com/tschaub/mock-fs/issues/118 expect.stringContaining( - 'Unable to upload file(s) to AWS S3. Error Failed to read template directory: ENOENT, no such file or directory', + `Unable to upload file(s) to AWS S3. Error: Failed to read template directory: ENOENT, no such file or directory`, ), - ), - ); + ); + expect(error.message).toEqual( + expect.stringContaining(wrongPathToGeneratedDirectory), + ); + }); mockFs.restore(); }); }); @@ -205,12 +226,19 @@ describe('AwsS3Publish', () => { it('should return an error if the techdocs_metadata.json file is not present', async () => { const entityNameMock = createMockEntityName(); + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); await publisher .fetchTechDocsMetadata(entityNameMock) .catch(error => - expect(error.message).toEqual( - expect.stringContaining('TechDocs metadata fetch'), + expect(error).toEqual( + new Error( + `TechDocs metadata fetch failed, The file ${path.join( + entityRootDir, + 'techdocs_metadata.json', + )} does not exist !`, + ), ), ); }); diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index b0b29e8654..96cce331ec 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -140,9 +140,18 @@ export class AwsS3Publish implements PublisherBase { // Remove the absolute path prefix of the source directory // Path of all files to upload, relative to the root of the source directory // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] - const relativeFilePath = filePath.replace(`${directory}/`, ''); + const relativeFilePath = path.relative(directory, filePath); + + // Convert destination file path to a POSIX path for uploading. + // S3 expects / as path separator and relativeFilePath will contain \\ on Windows. + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + const relativeFilePathPosix = relativeFilePath + .split(path.sep) + .join(path.posix.sep); + + // The / delimiter is intentional since it represents the cloud storage and not the local file system. const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = `${entityRootDir}/${relativeFilePath}`; // S3 Bucket file relative path + const destination = `${entityRootDir}/${relativeFilePathPosix}`; // S3 Bucket file relative path const fileContent = await fs.readFile(filePath, 'utf8'); @@ -164,7 +173,7 @@ export class AwsS3Publish implements PublisherBase { ); return; } catch (e) { - const errorMessage = `Unable to upload file(s) to AWS S3. Error ${e.message}`; + const errorMessage = `Unable to upload file(s) to AWS S3. ${e}`; this.logger.error(errorMessage); throw new Error(errorMessage); } diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 62c81df5c4..cf1ea60592 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -13,13 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import mockFs from 'mock-fs'; -import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import mockFs from 'mock-fs'; +import os from 'os'; +import path from 'path'; import { AzureBlobStoragePublish } from './azureBlobStorage'; import { PublisherBase } from './types'; -import type { Entity } from '@backstage/catalog-model'; -import type { Logger } from 'winston'; + +// NOTE: /packages/techdocs-common/__mocks__ is being used to mock Azure client library const createMockEntity = (annotations = {}) => { return { @@ -35,13 +38,14 @@ const createMockEntity = (annotations = {}) => { }; }; +const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; const getEntityRootDir = (entity: Entity) => { const { kind, metadata: { namespace, name }, } = entity; - const entityRootDir = `${namespace}/${kind}/${name}`; - return entityRootDir; + + return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name); }; function createLogger() { @@ -52,34 +56,33 @@ function createLogger() { } let publisher: PublisherBase; - -describe('publishing with valid credentials', () => { - let logger: Logger; - - beforeEach(async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'azureBlobStorage', - azureBlobStorage: { - credentials: { - accountName: 'accountName', - accountKey: 'accountKey', - }, - containerName: 'containerName', +beforeEach(async () => { + mockFs.restore(); + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'azureBlobStorage', + azureBlobStorage: { + credentials: { + accountName: 'accountName', + accountKey: 'accountKey', }, + containerName: 'containerName', }, }, - }); - - logger = createLogger(); - - publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger); + }, }); + publisher = await AzureBlobStoragePublish.fromConfig( + mockConfig, + createLogger(), + ); +}); + +describe('publishing with valid credentials', () => { describe('publish', () => { - it('should publish a directory', async () => { + beforeEach(() => { const entity = createMockEntity(); const entityRootDir = getEntityRootDir(entity); @@ -92,6 +95,11 @@ describe('publishing with valid credentials', () => { }, }, }); + }); + + it('should publish a directory', async () => { + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); expect( await publisher.publish({ @@ -103,30 +111,33 @@ describe('publishing with valid credentials', () => { }); it('should fail to publish a directory', async () => { - const wrongPathToGeneratedDirectory = 'wrong/path/to/generatedDirectory'; - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + expect.assertions(1); + const wrongPathToGeneratedDirectory = path.join( + rootDir, + 'wrong', + 'path', + 'to', + 'generatedDirectory', + ); - mockFs({ - [entityRootDir]: { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', - }, - }, - }); + const entity = createMockEntity(); await publisher .publish({ entity, directory: wrongPathToGeneratedDirectory, }) - .catch(error => - expect(error.message).toContain( - 'Unable to upload file(s) to Azure Blob Storage. Failed to read template directory: ENOENT, no such file or directory', - ), - ); + .catch(error => { + // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error + // Issue reported https://github.com/tschaub/mock-fs/issues/118 + expect.stringContaining( + `Unable to upload file(s) to Azure Blob Storage. Error: Failed to read template directory: ENOENT, no such file or directory`, + ); + + expect(error.message).toEqual( + expect.stringContaining(wrongPathToGeneratedDirectory), + ); + }); mockFs.restore(); }); }); @@ -235,7 +246,10 @@ describe('error reporting', () => { expect(logger.error).toHaveBeenCalledWith( expect.stringContaining( - `Unable to upload file(s) to Azure Blob Storage. Upload failed for test-namespace/TestKind/test-component-name/index.html with status code 500`, + `Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join( + entityRootDir, + 'index.html', + )} with status code 500`, ), ); diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index 7000e9f28c..0d2e9871bf 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import platformPath from 'path'; -import express from 'express'; +import { DefaultAzureCredential } from '@azure/identity'; import { BlobServiceClient, StorageSharedKeyCredential, } from '@azure/storage-blob'; -import { DefaultAzureCredential } from '@azure/identity'; -import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; -import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; -import limiterFactory from 'p-limit'; +import express from 'express'; import JSON5 from 'json5'; +import limiterFactory from 'p-limit'; +import { default as path, default as platformPath } from 'path'; +import { Logger } from 'winston'; +import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; // The number of batches that may be ongoing at the same time. const BATCH_CONCURRENCY = 3; @@ -126,27 +126,35 @@ export class AzureBlobStoragePublish implements PublisherBase { // or start thrashing. const limiter = limiterFactory(BATCH_CONCURRENCY); - const promises = allFilesToUpload.map(filePath => { + const promises = allFilesToUpload.map(sourceFilePath => { // Remove the absolute path prefix of the source directory // Path of all files to upload, relative to the root of the source directory // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] - const relativeFilePath = filePath.replace(`${directory}/`, ''); - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = platformPath.normalize( - `${entityRootDir}/${relativeFilePath}`, - ); // Azure Blob Storage Container file relative path + const relativeFilePath = path.normalize( + path.relative(directory, sourceFilePath), + ); + // Convert destination file path to a POSIX path for uploading. + // Azure Blob Storage expects / as path separator and relativeFilePath will contain \\ on Windows. + // https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#blob-names + const relativeFilePathPosix = relativeFilePath + .split(path.sep) + .join(path.posix.sep); + + // The / delimiter is intentional since it represents the cloud storage and not the local file system. + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Azure Blob Storage Container file relative path return limiter(async () => { const response = await this.storageClient .getContainerClient(this.containerName) .getBlockBlobClient(destination) - .uploadFile(filePath); + .uploadFile(sourceFilePath); if (response._response.status >= 400) { return { ...response, error: new Error( - `Upload failed for ${filePath} with status code ${response._response.status}`, + `Upload failed for ${sourceFilePath} with status code ${response._response.status}`, ), }; } @@ -173,18 +181,18 @@ export class AzureBlobStoragePublish implements PublisherBase { ); } } catch (e) { - const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e.message}`; + const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e}`; this.logger.error(errorMessage); throw new Error(errorMessage); } } - private download(containerName: string, path: string): Promise { + private download(containerName: string, blobPath: string): Promise { return new Promise((resolve, reject) => { const fileStreamChunks: Array = []; this.storageClient .getContainerClient(containerName) - .getBlockBlobClient(path) + .getBlockBlobClient(blobPath) .download() .then(res => { const body = res.readableStreamBody; diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index f8fb00647c..10da0d476a 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -13,18 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import mockFs from 'mock-fs'; -import * as winston from 'winston'; +import { getVoidLogger } from '@backstage/backend-common'; +import { + Entity, + EntityName, + ENTITY_DEFAULT_NAMESPACE, +} from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; +import mockFs from 'mock-fs'; +import os from 'os'; +import path from 'path'; import { GoogleGCSPublish } from './googleStorage'; -import { PublisherBase } from './types'; +import { PublisherBase, TechDocsMetadata } from './types'; -const createMockEntity = (annotations = {}) => { +// NOTE: /packages/techdocs-common/__mocks__ is being used to mock Google Cloud Storage client library + +const createMockEntity = (annotations = {}): Entity => { return { apiVersion: 'version', kind: 'TestKind', metadata: { name: 'test-component-name', + namespace: 'test-namespace', annotations: { ...annotations, }, @@ -32,12 +42,30 @@ const createMockEntity = (annotations = {}) => { }; }; -const logger = winston.createLogger(); +const createMockEntityName = (): EntityName => ({ + kind: 'TestKind', + name: 'test-component-name', + namespace: 'test-namespace', +}); + +const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; + +const getEntityRootDir = (entity: Entity) => { + const { + kind, + metadata: { namespace, name }, + } = entity; + + return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name); +}; + +const logger = getVoidLogger(); jest.spyOn(logger, 'info').mockReturnValue(logger); let publisher: PublisherBase; beforeEach(async () => { + mockFs.restore(); const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -55,24 +83,166 @@ beforeEach(async () => { }); describe('GoogleGCSPublish', () => { - it('should publish a directory', async () => { - mockFs({ - '/path/to/generatedDirectory': { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', + describe('publish', () => { + beforeEach(() => { + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'index.html': '', + '404.html': '', + assets: { + 'main.css': '', + }, }, - }, + }); }); - const entity = createMockEntity(); - expect( - await publisher.publish({ - entity, - directory: '/path/to/generatedDirectory', - }), - ).toBeUndefined(); - mockFs.restore(); + afterEach(() => { + mockFs.restore(); + }); + + it('should publish a directory', async () => { + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + expect( + await publisher.publish({ + entity, + directory: entityRootDir, + }), + ).toBeUndefined(); + mockFs.restore(); + }); + + it('should fail to publish a directory', async () => { + expect.assertions(3); + + const wrongPathToGeneratedDirectory = path.join( + rootDir, + 'wrong', + 'path', + 'to', + 'generatedDirectory', + ); + + const entity = createMockEntity(); + + await expect( + publisher.publish({ + entity, + directory: wrongPathToGeneratedDirectory, + }), + ).rejects.toThrowError(); + + await publisher + .publish({ + entity, + directory: wrongPathToGeneratedDirectory, + }) + .catch(error => { + expect(error.message).toEqual( + // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error + // Issue reported https://github.com/tschaub/mock-fs/issues/118 + expect.stringContaining( + `Unable to upload file(s) to Google Cloud Storage. Error: Failed to read template directory: ENOENT, no such file or directory`, + ), + ); + expect(error.message).toEqual( + expect.stringContaining(wrongPathToGeneratedDirectory), + ); + }); + + mockFs.restore(); + }); + }); + + describe('hasDocsBeenGenerated', () => { + it('should return true if docs has been generated', async () => { + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'index.html': 'file-content', + }, + }); + + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + mockFs.restore(); + }); + + it('should return false if docs has not been generated', async () => { + const entity = createMockEntity(); + + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false); + }); + }); + + describe('fetchTechDocsMetadata', () => { + beforeEach(() => { + mockFs.restore(); + }); + + it('should return tech docs metadata', async () => { + const entityNameMock = createMockEntityName(); + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'techdocs_metadata.json': + '{"site_name": "backstage", "site_description": "site_content"}', + }, + }); + + const expectedMetadata: TechDocsMetadata = { + site_name: 'backstage', + site_description: 'site_content', + }; + expect( + await publisher.fetchTechDocsMetadata(entityNameMock), + ).toStrictEqual(expectedMetadata); + }); + + it('should return tech docs metadata when json encoded with single quotes', async () => { + const entityNameMock = createMockEntityName(); + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'techdocs_metadata.json': + "{'site_name': 'backstage', 'site_description': 'site_content'}", + }, + }); + + const expectedMetadata: TechDocsMetadata = { + site_name: 'backstage', + site_description: 'site_content', + }; + expect( + await publisher.fetchTechDocsMetadata(entityNameMock), + ).toStrictEqual(expectedMetadata); + mockFs.restore(); + }); + + it('should return an error if the techdocs_metadata.json file is not present', async () => { + const entityNameMock = createMockEntityName(); + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + await publisher + .fetchTechDocsMetadata(entityNameMock) + .catch(errorMessage => + expect(errorMessage).toEqual( + `The file ${path.join( + entityRootDir, + 'techdocs_metadata.json', + )} does not exist !`, + ), + ); + }); }); }); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 8876007504..12ecf45b2d 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import path from 'path'; -import express from 'express'; -import { - Storage, - UploadResponse, - FileExistsResponse, -} from '@google-cloud/storage'; -import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; -import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; +import { + FileExistsResponse, + Storage, + UploadResponse, +} from '@google-cloud/storage'; +import express from 'express'; import JSON5 from 'json5'; import createLimiter from 'p-limit'; +import path from 'path'; +import { Logger } from 'winston'; +import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; export class GoogleGCSPublish implements PublisherBase { static async fromConfig( @@ -97,44 +97,50 @@ export class GoogleGCSPublish implements PublisherBase { * Upload all the files from the generated `directory` to the GCS bucket. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ - publish({ entity, directory }: PublishRequest): Promise { - return new Promise(async (resolve, reject) => { + async publish({ entity, directory }: PublishRequest): Promise { + try { // Note: GCS manages creation of parent directories if they do not exist. // So collecting path of only the files is good enough. const allFilesToUpload = await getFileTreeRecursively(directory); const limiter = createLimiter(10); const uploadPromises: Array> = []; - allFilesToUpload.forEach(filePath => { + allFilesToUpload.forEach(sourceFilePath => { // Remove the absolute path prefix of the source directory // Path of all files to upload, relative to the root of the source directory // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] - const relativeFilePath = filePath.replace(`${directory}/`, ''); + const relativeFilePath = path.relative(directory, sourceFilePath); + + // Convert destination file path to a POSIX path for uploading. + // GCS expects / as path separator and relativeFilePath will contain \\ on Windows. + // https://cloud.google.com/storage/docs/gsutil/addlhelp/HowSubdirectoriesWork + const relativeFilePathPosix = relativeFilePath + .split(path.sep) + .join(path.posix.sep); + + // The / delimiter is intentional since it represents the cloud storage and not the local file system. const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = `${entityRootDir}/${relativeFilePath}`; // GCS Bucket file relative path + const destination = `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => - this.storageClient - .bucket(this.bucketName) - .upload(filePath, { destination }), + this.storageClient.bucket(this.bucketName).upload(sourceFilePath, { + destination, + }), ); uploadPromises.push(uploadFile); }); - Promise.all(uploadPromises) - .then(() => { - this.logger.info( - `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, - ); - resolve(undefined); - }) - .catch((err: Error) => { - const errorMessage = `Unable to upload file(s) to Google Cloud Storage. Error ${err.message}`; - this.logger.error(errorMessage); - reject(errorMessage); - }); - }); + await Promise.all(uploadPromises); + + this.logger.info( + `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, + ); + } catch (e) { + const errorMessage = `Unable to upload file(s) to Google Cloud Storage. ${e}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } } fetchTechDocsMetadata(entityName: EntityName): Promise { @@ -154,9 +160,9 @@ export class GoogleGCSPublish implements PublisherBase { fileStreamChunks.push(chunk); }) .on('end', () => { - const techdocsMetadataJson = Buffer.concat( - fileStreamChunks, - ).toString(); + const techdocsMetadataJson = Buffer.concat(fileStreamChunks).toString( + 'utf-8', + ); resolve(JSON5.parse(techdocsMetadataJson)); }); });