From 6c36689039d5a952c7c8d9da7e6c5b9dfb989426 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Thu, 18 Feb 2021 21:58:44 +0100 Subject: [PATCH] TechDocs/Azure: Fix publisher to work with Windows file paths --- .../stages/publish/azureBlobStorage.test.ts | 70 ++++++++----------- .../src/stages/publish/azureBlobStorage.ts | 44 +++++++----- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 62c81df5c4..e43f9112f5 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import mockFs from 'mock-fs'; -import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; +import type { Entity } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import mockFs from 'mock-fs'; import { AzureBlobStoragePublish } from './azureBlobStorage'; import { PublisherBase } from './types'; -import type { Entity } from '@backstage/catalog-model'; -import type { Logger } from 'winston'; const createMockEntity = (annotations = {}) => { return { @@ -52,34 +51,33 @@ function createLogger() { } let publisher: PublisherBase; - -describe('publishing with valid credentials', () => { - let logger: Logger; - - beforeEach(async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'azureBlobStorage', - azureBlobStorage: { - credentials: { - accountName: 'accountName', - accountKey: 'accountKey', - }, - containerName: 'containerName', +beforeEach(async () => { + mockFs.restore(); + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'azureBlobStorage', + azureBlobStorage: { + credentials: { + accountName: 'accountName', + accountKey: 'accountKey', }, + containerName: 'containerName', }, }, - }); - - logger = createLogger(); - - publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger); + }, }); + publisher = await AzureBlobStoragePublish.fromConfig( + mockConfig, + createLogger(), + ); +}); + +describe('publishing with valid credentials', () => { describe('publish', () => { - it('should publish a directory', async () => { + beforeEach(() => { const entity = createMockEntity(); const entityRootDir = getEntityRootDir(entity); @@ -92,6 +90,11 @@ describe('publishing with valid credentials', () => { }, }, }); + }); + + it('should publish a directory', async () => { + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); expect( await publisher.publish({ @@ -105,17 +108,6 @@ describe('publishing with valid credentials', () => { it('should fail to publish a directory', async () => { const wrongPathToGeneratedDirectory = 'wrong/path/to/generatedDirectory'; const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', - }, - }, - }); await publisher .publish({ @@ -124,7 +116,7 @@ describe('publishing with valid credentials', () => { }) .catch(error => expect(error.message).toContain( - 'Unable to upload file(s) to Azure Blob Storage. Failed to read template directory: ENOENT, no such file or directory', + 'Unable to upload file(s) to Azure Blob Storage. Error: Failed to read template directory: ENOENT, no such file or directory', ), ); mockFs.restore(); @@ -235,7 +227,7 @@ describe('error reporting', () => { expect(logger.error).toHaveBeenCalledWith( expect.stringContaining( - `Unable to upload file(s) to Azure Blob Storage. Upload failed for test-namespace/TestKind/test-component-name/index.html with status code 500`, + `Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for test-namespace/TestKind/test-component-name/index.html with status code 500`, ), ); diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index 7000e9f28c..0d2e9871bf 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import platformPath from 'path'; -import express from 'express'; +import { DefaultAzureCredential } from '@azure/identity'; import { BlobServiceClient, StorageSharedKeyCredential, } from '@azure/storage-blob'; -import { DefaultAzureCredential } from '@azure/identity'; -import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; -import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; -import limiterFactory from 'p-limit'; +import express from 'express'; import JSON5 from 'json5'; +import limiterFactory from 'p-limit'; +import { default as path, default as platformPath } from 'path'; +import { Logger } from 'winston'; +import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; // The number of batches that may be ongoing at the same time. const BATCH_CONCURRENCY = 3; @@ -126,27 +126,35 @@ export class AzureBlobStoragePublish implements PublisherBase { // or start thrashing. const limiter = limiterFactory(BATCH_CONCURRENCY); - const promises = allFilesToUpload.map(filePath => { + const promises = allFilesToUpload.map(sourceFilePath => { // Remove the absolute path prefix of the source directory // Path of all files to upload, relative to the root of the source directory // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] - const relativeFilePath = filePath.replace(`${directory}/`, ''); - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = platformPath.normalize( - `${entityRootDir}/${relativeFilePath}`, - ); // Azure Blob Storage Container file relative path + const relativeFilePath = path.normalize( + path.relative(directory, sourceFilePath), + ); + // Convert destination file path to a POSIX path for uploading. + // Azure Blob Storage expects / as path separator and relativeFilePath will contain \\ on Windows. + // https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#blob-names + const relativeFilePathPosix = relativeFilePath + .split(path.sep) + .join(path.posix.sep); + + // The / delimiter is intentional since it represents the cloud storage and not the local file system. + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Azure Blob Storage Container file relative path return limiter(async () => { const response = await this.storageClient .getContainerClient(this.containerName) .getBlockBlobClient(destination) - .uploadFile(filePath); + .uploadFile(sourceFilePath); if (response._response.status >= 400) { return { ...response, error: new Error( - `Upload failed for ${filePath} with status code ${response._response.status}`, + `Upload failed for ${sourceFilePath} with status code ${response._response.status}`, ), }; } @@ -173,18 +181,18 @@ export class AzureBlobStoragePublish implements PublisherBase { ); } } catch (e) { - const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e.message}`; + const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e}`; this.logger.error(errorMessage); throw new Error(errorMessage); } } - private download(containerName: string, path: string): Promise { + private download(containerName: string, blobPath: string): Promise { return new Promise((resolve, reject) => { const fileStreamChunks: Array = []; this.storageClient .getContainerClient(containerName) - .getBlockBlobClient(path) + .getBlockBlobClient(blobPath) .download() .then(res => { const body = res.readableStreamBody;