paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 42 Packages Projects Releases Wiki Activity

TechDocs/Azure: Fix publisher to work with Windows file paths

Browse Source
This commit is contained in:
Himanshu Mishra
2021-02-18 21:58:44 +01:00
parent 9ff646b6a8
commit 6c36689039
2 changed files with 57 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
+31 -39
View File
@@ -13,13 +13,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import mockFs from 'mock-fs';
import { ConfigReader } from '@backstage/config';
import { getVoidLogger } from '@backstage/backend-common';
import type { Entity } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import mockFs from 'mock-fs';
import { AzureBlobStoragePublish } from './azureBlobStorage';
import { PublisherBase } from './types';
import type { Entity } from '@backstage/catalog-model';
import type { Logger } from 'winston';
const createMockEntity = (annotations = {}) => {
return {
@@ -52,34 +51,33 @@ function createLogger() {
}
let publisher: PublisherBase;
describe('publishing with valid credentials', () => {
let logger: Logger;
beforeEach(async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'accountName',
accountKey: 'accountKey',
},
containerName: 'containerName',
beforeEach(async () => {
mockFs.restore();
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'accountName',
accountKey: 'accountKey',
},
containerName: 'containerName',
},
},
});
logger = createLogger();
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
},
});
publisher = await AzureBlobStoragePublish.fromConfig(
mockConfig,
createLogger(),
);
});
describe('publishing with valid credentials', () => {
describe('publish', () => {
it('should publish a directory', async () => {
beforeEach(() => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
@@ -92,6 +90,11 @@ describe('publishing with valid credentials', () => {
},
},
});
});
it('should publish a directory', async () => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
expect(
await publisher.publish({
@@ -105,17 +108,6 @@ describe('publishing with valid credentials', () => {
it('should fail to publish a directory', async () => {
const wrongPathToGeneratedDirectory = 'wrong/path/to/generatedDirectory';
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': '',
'404.html': '',
assets: {
'main.css': '',
},
},
});
await publisher
.publish({
@@ -124,7 +116,7 @@ describe('publishing with valid credentials', () => {
})
.catch(error =>
expect(error.message).toContain(
'Unable to upload file(s) to Azure Blob Storage. Failed to read template directory: ENOENT, no such file or directory',
'Unable to upload file(s) to Azure Blob Storage. Error: Failed to read template directory: ENOENT, no such file or directory',
),
);
mockFs.restore();
@@ -235,7 +227,7 @@ describe('error reporting', () => {
expect(logger.error).toHaveBeenCalledWith(
expect.stringContaining(
`Unable to upload file(s) to Azure Blob Storage. Upload failed for test-namespace/TestKind/test-component-name/index.html with status code 500`,
`Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for test-namespace/TestKind/test-component-name/index.html with status code 500`,
),
);
packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
+26 -18
View File
@@ -13,20 +13,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import platformPath from 'path';
import express from 'express';
import { DefaultAzureCredential } from '@azure/identity';
import {
BlobServiceClient,
StorageSharedKeyCredential,
} from '@azure/storage-blob';
import { DefaultAzureCredential } from '@azure/identity';
import { Logger } from 'winston';
import { Entity, EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
import limiterFactory from 'p-limit';
import express from 'express';
import JSON5 from 'json5';
import limiterFactory from 'p-limit';
import { default as path, default as platformPath } from 'path';
import { Logger } from 'winston';
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
// The number of batches that may be ongoing at the same time.
const BATCH_CONCURRENCY = 3;
@@ -126,27 +126,35 @@ export class AzureBlobStoragePublish implements PublisherBase {
// or start thrashing.
const limiter = limiterFactory(BATCH_CONCURRENCY);
const promises = allFilesToUpload.map(filePath => {
const promises = allFilesToUpload.map(sourceFilePath => {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
const relativeFilePath = filePath.replace(`${directory}/`, '');
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const destination = platformPath.normalize(
`${entityRootDir}/${relativeFilePath}`,
); // Azure Blob Storage Container file relative path
const relativeFilePath = path.normalize(
path.relative(directory, sourceFilePath),
);
// Convert destination file path to a POSIX path for uploading.
// Azure Blob Storage expects / as path separator and relativeFilePath will contain \\ on Windows.
// https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#blob-names
const relativeFilePathPosix = relativeFilePath
.split(path.sep)
.join(path.posix.sep);
// The / delimiter is intentional since it represents the cloud storage and not the local file system.
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Azure Blob Storage Container file relative path
return limiter(async () => {
const response = await this.storageClient
.getContainerClient(this.containerName)
.getBlockBlobClient(destination)
.uploadFile(filePath);
.uploadFile(sourceFilePath);
if (response._response.status >= 400) {
return {
...response,
error: new Error(
`Upload failed for ${filePath} with status code ${response._response.status}`,
`Upload failed for ${sourceFilePath} with status code ${response._response.status}`,
),
};
}
@@ -173,18 +181,18 @@ export class AzureBlobStoragePublish implements PublisherBase {
);
}
} catch (e) {
const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e.message}`;
const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e}`;
this.logger.error(errorMessage);
throw new Error(errorMessage);
}
}
private download(containerName: string, path: string): Promise<Buffer> {
private download(containerName: string, blobPath: string): Promise<Buffer> {
return new Promise((resolve, reject) => {
const fileStreamChunks: Array<any> = [];
this.storageClient
.getContainerClient(containerName)
.getBlockBlobClient(path)
.getBlockBlobClient(blobPath)
.download()
.then(res => {
const body = res.readableStreamBody;