test(techdocs-common): fix failing tests for all providers

Co-authored-by: Eric Peterson <ericperterson@spotify.com>
Signed-off-by: Camila Belo <camilaibs@gmail.com>
This commit is contained in:
Camila Belo
2021-08-04 22:23:06 +02:00
parent a72470c193
commit a7a55632d3
8 changed files with 360 additions and 411 deletions
@@ -42,12 +42,6 @@ const checkFileExists = async (Key: string): Promise<boolean> => {
};
export class S3 {
private readonly options;
constructor(options: S3Types.ClientConfiguration) {
this.options = options;
}
headObject({ Key }: { Key: string }) {
return {
promise: async () => {
@@ -61,7 +55,7 @@ export class S3 {
getObject({ Key }: { Key: string }) {
const filePath = path.join(rootDir, Key);
return {
promise: () => checkFileExists(filePath),
promise: async () => await checkFileExists(filePath),
createReadStream: () => {
const emitter = new EventEmitter();
process.nextTick(() => {
@@ -86,7 +80,6 @@ export class S3 {
if (Bucket === 'errorBucket') {
throw new Error('Bucket does not exist');
}
return {};
},
};
@@ -15,11 +15,7 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import {
Entity,
EntityName,
ENTITY_DEFAULT_NAMESPACE,
} from '@backstage/catalog-model';
import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import express from 'express';
import request from 'supertest';
@@ -27,30 +23,9 @@ import mockFs from 'mock-fs';
import os from 'os';
import path from 'path';
import { AwsS3Publish } from './awsS3';
import { PublisherBase, TechDocsMetadata } from './types';
// NOTE: /packages/techdocs-common/__mocks__ is being used to mock aws-sdk client library
const createMockEntity = (annotations = {}): Entity => {
return {
apiVersion: 'version',
kind: 'TestKind',
metadata: {
name: 'test-component-name',
namespace: 'test-namespace',
annotations: {
...annotations,
},
},
};
};
const createMockEntityName = (): EntityName => ({
kind: 'TestKind',
name: 'test-component-name',
namespace: 'test-namespace',
});
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
const getEntityRootDir = (entity: Entity) => {
@@ -64,10 +39,7 @@ const getEntityRootDir = (entity: Entity) => {
const logger = getVoidLogger();
let publisher: PublisherBase;
beforeEach(() => {
mockFs.restore();
const createPublisherFromConfig = (bucketName: string = 'bucketName') => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
@@ -78,80 +50,119 @@ beforeEach(() => {
accessKeyId: 'accessKeyId',
secretAccessKey: 'secretAccessKey',
},
bucketName: 'bucketName',
bucketName,
},
},
},
});
publisher = AwsS3Publish.fromConfig(mockConfig, logger);
});
return AwsS3Publish.fromConfig(mockConfig, logger);
};
describe('AwsS3Publish', () => {
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'backstage',
namespace: 'default',
annotations: {},
},
};
const entityName = {
kind: 'Component',
name: 'backstage',
namespace: 'default',
};
const techdocsMetadata = {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
};
const localRootDir = getEntityRootDir(entity);
const storageRootDir = getEntityRootDir({
...entity,
kind: entity.kind.toLowerCase(),
});
const storageSingleQuoteDir = getEntityRootDir({
metadata: {
namespace: 'storage',
name: 'quote',
},
kind: 'single',
} as Entity);
beforeAll(() => {
mockFs({
[localRootDir]: {
'index.html': '',
'404.html': '',
assets: {
'main.css': '',
},
attachments: {
'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]),
},
'techdocs_metadata.json': JSON.stringify(techdocsMetadata),
},
[storageRootDir]: {
'index.html': '',
'404.html': '',
'techdocs_metadata.json': JSON.stringify(techdocsMetadata),
assets: {
'main.css': '',
},
attachments: {
'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]),
},
html: {
'unsafe.html': '<html></html>',
},
img: {
'with spaces.png': 'found it',
'unsafe.svg': '<svg></svg>',
},
'some folder': {
'also with spaces.js': 'found it too',
},
},
[storageSingleQuoteDir]: {
'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`,
},
});
});
afterAll(() => {
mockFs.restore();
});
describe('getReadiness', () => {
it('should validate correct config', async () => {
const publisher = createPublisherFromConfig();
expect(await publisher.getReadiness()).toEqual({
isAvailable: true,
});
});
it('should reject incorrect config', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'awsS3',
awsS3: {
credentials: {
accessKeyId: 'accessKeyId',
secretAccessKey: 'secretAccessKey',
},
// this bucket name will throw an error
bucketName: 'errorBucket',
},
},
},
});
const errorPublisher = AwsS3Publish.fromConfig(mockConfig, logger);
expect(await errorPublisher.getReadiness()).toEqual({
const publisher = createPublisherFromConfig('errorBucket');
expect(await publisher.getReadiness()).toEqual({
isAvailable: false,
});
});
});
describe('publish', () => {
beforeEach(() => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': '',
'404.html': '',
assets: {
'main.css': '',
},
attachments: {
'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]),
},
},
});
});
afterEach(() => {
mockFs.restore();
});
it('should publish a directory', async () => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
const publisher = createPublisherFromConfig();
expect(
await publisher.publish({
entity,
directory: entityRootDir,
directory: localRootDir,
}),
).toBeUndefined();
});
@@ -165,172 +176,114 @@ describe('AwsS3Publish', () => {
'generatedDirectory',
);
const entity = createMockEntity();
await expect(
publisher.publish({
entity,
directory: wrongPathToGeneratedDirectory,
}),
).rejects.toThrowError();
const publisher = createPublisherFromConfig();
const fails = publisher.publish({
entity,
directory: wrongPathToGeneratedDirectory,
});
// Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error
// Issue reported https://github.com/tschaub/mock-fs/issues/118
await expect(fails).rejects.toMatchObject({
message: expect.stringContaining(
'Unable to upload file(s) to AWS S3. Error: Failed to read template directory: ENOENT, no such file or directory',
),
});
await expect(fails).rejects.toMatchObject({
message: expect.stringContaining(wrongPathToGeneratedDirectory),
});
mockFs.restore();
});
});
describe('hasDocsBeenGenerated', () => {
it('should return true if docs has been generated', async () => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': 'file-content',
},
});
const publisher = createPublisherFromConfig();
expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
mockFs.restore();
});
it('should return false if docs has not been generated', async () => {
const entity = createMockEntity();
expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false);
const publisher = createPublisherFromConfig();
expect(
await publisher.hasDocsBeenGenerated({
kind: 'entity',
metadata: {
namespace: 'invalid',
name: 'triplet',
},
} as Entity),
).toBe(false);
});
});
describe('fetchTechDocsMetadata', () => {
it('should return tech docs metadata', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'techdocs_metadata.json':
'{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}',
},
});
const expectedMetadata: TechDocsMetadata = {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
};
expect(
await publisher.fetchTechDocsMetadata(entityNameMock),
).toStrictEqual(expectedMetadata);
mockFs.restore();
const publisher = createPublisherFromConfig();
expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
techdocsMetadata,
);
});
it('should return tech docs metadata when json encoded with single quotes', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`,
},
});
const expectedMetadata: TechDocsMetadata = {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
};
const publisher = createPublisherFromConfig();
expect(
await publisher.fetchTechDocsMetadata(entityNameMock),
).toStrictEqual(expectedMetadata);
mockFs.restore();
await publisher.fetchTechDocsMetadata({
namespace: 'storage',
kind: 'single',
name: 'quote',
}),
).toStrictEqual(techdocsMetadata);
});
it('should return an error if the techdocs_metadata.json file is not present', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
const publisher = createPublisherFromConfig();
const fails = publisher.fetchTechDocsMetadata(entityNameMock);
const invalidEntityName = {
namespace: 'invalid',
kind: 'triplet',
name: 'path',
};
const techDocsMetadaFilePath = path.join(
rootDir,
...Object.values(invalidEntityName),
'techdocs_metadata.json',
);
const fails = publisher.fetchTechDocsMetadata(invalidEntityName);
const errorPath = path.join(entityRootDir, 'techdocs_metadata.json');
await expect(fails).rejects.toMatchObject({
message: `TechDocs metadata fetch failed, The file ${errorPath} does not exist !`,
message: `TechDocs metadata fetch failed, The file ${techDocsMetadaFilePath} does not exist !`,
});
});
});
describe('docsRouter', () => {
const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
let app: express.Express;
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
beforeEach(() => {
const publisher = createPublisherFromConfig();
app = express().use(publisher.docsRouter());
mockFs.restore();
mockFs({
[entityRootDir]: {
html: {
'unsafe.html': '<html></html>',
},
img: {
'with spaces.png': 'found it',
'unsafe.svg': '<svg></svg>',
},
'some folder': {
'also with spaces.js': 'found it too',
},
},
});
});
afterEach(() => {
mockFs.restore();
});
it('should pass expected object path to bucket', async () => {
const {
kind,
metadata: { namespace, name },
} = entity;
// Ensures leading slash is trimmed and encoded path is decoded.
const pngResponse = await request(app).get(
`/${namespace}/${kind}/${name}/img/with%20spaces.png`,
`/${entityTripletPath}/img/with%20spaces.png`,
);
expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
'found it',
);
const jsResponse = await request(app).get(
`/${namespace}/${kind}/${name}/some%20folder/also%20with%20spaces.js`,
`/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
);
expect(jsResponse.text).toEqual('found it too');
});
it('should pass text/plain content-type for html', async () => {
const {
kind,
metadata: { namespace, name },
} = entity;
const htmlResponse = await request(app).get(
`/${namespace}/${kind}/${name}/html/unsafe.html`,
`/${entityTripletPath}/html/unsafe.html`,
);
expect(htmlResponse.text).toEqual('<html></html>');
expect(htmlResponse.header).toMatchObject({
@@ -338,7 +291,7 @@ describe('AwsS3Publish', () => {
});
const svgResponse = await request(app).get(
`/${namespace}/${kind}/${name}/img/unsafe.svg`,
`/${entityTripletPath}/img/unsafe.svg`,
);
expect(svgResponse.text).toEqual('<svg></svg>');
expect(svgResponse.header).toMatchObject({
@@ -32,6 +32,7 @@ import { Logger } from 'winston';
import {
getFileTreeRecursively,
getHeadersForFileExtension,
lowerCaseEntityTriplet,
lowerCaseEntityTripletInStoragePath,
} from './helpers';
import {
@@ -238,7 +239,9 @@ export class AwsS3Publish implements PublisherBase {
): Promise<TechDocsMetadata> {
try {
return await new Promise<TechDocsMetadata>(async (resolve, reject) => {
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
const entityRootDir = lowerCaseEntityTriplet(
`${entityName.namespace}/${entityName.kind}/${entityName.name}`,
);
const stream = this.storageClient
.getObject({
@@ -310,7 +313,9 @@ export class AwsS3Publish implements PublisherBase {
*/
async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
try {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const entityRootDir = lowerCaseEntityTriplet(
`${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`,
);
await this.storageClient
.headObject({
Bucket: this.bucketName,
@@ -15,11 +15,7 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import {
Entity,
EntityName,
ENTITY_DEFAULT_NAMESPACE,
} from '@backstage/catalog-model';
import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import express from 'express';
import request from 'supertest';
@@ -27,30 +23,9 @@ import mockFs from 'mock-fs';
import os from 'os';
import path from 'path';
import { AzureBlobStoragePublish } from './azureBlobStorage';
import { PublisherBase, TechDocsMetadata } from './types';
// NOTE: /packages/techdocs-common/__mocks__ is being used to mock Azure client library
const createMockEntity = (annotations = {}) => {
return {
apiVersion: 'version',
kind: 'TestKind',
metadata: {
name: 'test-component-name',
namespace: 'test-namespace',
annotations: {
...annotations,
},
},
};
};
const createMockEntityName = (): EntityName => ({
kind: 'TestKind',
name: 'test-component-name',
namespace: 'test-namespace',
});
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
const getEntityRootDir = (entity: Entity) => {
@@ -66,9 +41,10 @@ const logger = getVoidLogger();
jest.spyOn(logger, 'info').mockReturnValue(logger);
jest.spyOn(logger, 'error').mockReturnValue(logger);
let publisher: PublisherBase;
beforeEach(async () => {
mockFs.restore();
const createPublisherFromConfig = ({
accountName = 'accountName',
containerName = 'containerName',
}: undefined | { accountName?: string; containerName?: string } = {}) => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
@@ -76,49 +52,107 @@ beforeEach(async () => {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'accountName',
accountName,
accountKey: 'accountKey',
},
containerName: 'containerName',
containerName,
},
},
},
});
publisher = AzureBlobStoragePublish.fromConfig(mockConfig, logger);
});
return AzureBlobStoragePublish.fromConfig(mockConfig, logger);
};
describe('publishing with valid credentials', () => {
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'backstage',
namespace: 'default',
annotations: {},
},
};
const entityName = {
kind: 'Component',
name: 'backstage',
namespace: 'default',
};
const techdocsMetadata = {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
};
const localRootDir = getEntityRootDir(entity);
const storageRootDir = getEntityRootDir({
...entity,
kind: entity.kind.toLowerCase(),
});
const storageSingleQuoteDir = getEntityRootDir({
metadata: {
namespace: 'storage',
name: 'quote',
},
kind: 'single',
} as Entity);
beforeEach(() => {
(logger.info as jest.Mock).mockClear();
(logger.error as jest.Mock).mockClear();
});
beforeAll(async () => {
mockFs({
[localRootDir]: {
'index.html': 'file-content',
'404.html': '',
assets: {
'main.css': '',
},
'techdocs_metadata.json': JSON.stringify(techdocsMetadata),
},
[storageRootDir]: {
'index.html': '',
'techdocs_metadata.json': JSON.stringify(techdocsMetadata),
html: {
'unsafe.html': '<html></html>',
},
img: {
'with spaces.png': 'found it',
'unsafe.svg': '<svg></svg>',
},
'some folder': {
'also with spaces.js': 'found it too',
},
},
[storageSingleQuoteDir]: {
'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`,
},
});
});
afterAll(() => {
mockFs.restore();
});
describe('getReadiness', () => {
it('should validate correct config', async () => {
const publisher = createPublisherFromConfig();
expect(await publisher.getReadiness()).toEqual({
isAvailable: true,
});
});
it('should reject incorrect config', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'accountName',
accountKey: 'accountKey',
},
containerName: 'bad_container',
},
},
},
const publisher = createPublisherFromConfig({
containerName: 'bad_container',
});
const errorPublisher = await AzureBlobStoragePublish.fromConfig(
mockConfig,
logger,
);
expect(await errorPublisher.getReadiness()).toEqual({
expect(await publisher.getReadiness()).toEqual({
isAvailable: false,
});
@@ -131,32 +165,14 @@ describe('publishing with valid credentials', () => {
});
describe('publish', () => {
beforeEach(() => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': '',
'404.html': '',
assets: {
'main.css': '',
},
},
});
});
it('should publish a directory', async () => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
const publisher = createPublisherFromConfig();
expect(
await publisher.publish({
entity,
directory: entityRootDir,
directory: localRootDir,
}),
).toBeUndefined();
mockFs.restore();
});
it('should fail to publish a directory', async () => {
@@ -168,7 +184,9 @@ describe('publishing with valid credentials', () => {
'generatedDirectory',
);
const entity = createMockEntity();
const publisher = createPublisherFromConfig({
containerName: 'bad_container',
});
const fails = publisher.publish({
entity,
@@ -180,46 +198,22 @@ describe('publishing with valid credentials', () => {
`Unable to upload file(s) to Azure Blob Storage. Error: Failed to read template directory: ENOENT, no such file or directory`,
),
});
await expect(fails).rejects.toMatchObject({
message: expect.stringContaining(wrongPathToGeneratedDirectory),
});
mockFs.restore();
});
it('reports an error when bad account credentials', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'failupload',
accountKey: 'accountKey',
},
containerName: 'containerName',
},
},
},
});
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': '',
},
const publisher = createPublisherFromConfig({
accountName: 'failupload',
});
let error;
try {
await publisher.publish({
entity,
directory: entityRootDir,
directory: localRootDir,
});
} catch (e) {
error = e;
@@ -232,91 +226,64 @@ describe('publishing with valid credentials', () => {
expect(logger.error).toHaveBeenCalledWith(
expect.stringContaining(
`Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join(
entityRootDir,
'index.html',
localRootDir,
'404.html',
)} with status code 500`,
),
);
mockFs.restore();
});
});
describe('hasDocsBeenGenerated', () => {
it('should return true if docs has been generated', async () => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': 'file-content',
},
});
const publisher = createPublisherFromConfig();
expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
mockFs.restore();
});
it('should return false if docs has not been generated', async () => {
const entity = createMockEntity();
expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false);
const publisher = createPublisherFromConfig();
expect(
await publisher.hasDocsBeenGenerated({
kind: 'triplet',
metadata: {
namespace: 'invalid',
name: 'path',
},
} as Entity),
).toBe(false);
});
});
describe('fetchTechDocsMetadata', () => {
it('should return tech docs metadata', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'techdocs_metadata.json':
'{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}',
},
});
const expectedMetadata: TechDocsMetadata = {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
};
expect(
await publisher.fetchTechDocsMetadata(entityNameMock),
).toStrictEqual(expectedMetadata);
mockFs.restore();
const publisher = createPublisherFromConfig();
expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
techdocsMetadata,
);
});
it('should return tech docs metadata when json encoded with single quotes', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`,
},
});
const expectedMetadata: TechDocsMetadata = {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
};
const publisher = createPublisherFromConfig();
expect(
await publisher.fetchTechDocsMetadata(entityNameMock),
).toStrictEqual(expectedMetadata);
mockFs.restore();
await publisher.fetchTechDocsMetadata({
namespace: 'storage',
kind: 'single',
name: 'quote',
}),
).toStrictEqual(techdocsMetadata);
});
it('should return an error if the techdocs_metadata.json file is not present', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
const publisher = createPublisherFromConfig();
const invalidEntityName = {
namespace: 'invalid',
kind: 'triplet',
name: 'path',
};
let error;
try {
await publisher.fetchTechDocsMetadata(entityNameMock);
await publisher.fetchTechDocsMetadata(invalidEntityName);
} catch (e) {
error = e;
}
@@ -324,7 +291,8 @@ describe('publishing with valid credentials', () => {
expect(error).toEqual(
new Error(
`TechDocs metadata fetch failed, The file ${path.join(
entityRootDir,
rootDir,
...Object.values(invalidEntityName),
'techdocs_metadata.json',
)} does not exist !`,
),
@@ -333,61 +301,32 @@ describe('publishing with valid credentials', () => {
});
describe('docsRouter', () => {
const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
let app: express.Express;
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
beforeEach(() => {
const publisher = createPublisherFromConfig();
app = express().use(publisher.docsRouter());
mockFs.restore();
mockFs({
[entityRootDir]: {
html: {
'unsafe.html': '<html></html>',
},
img: {
'with spaces.png': 'found it',
'unsafe.svg': '<svg></svg>',
},
'some folder': {
'also with spaces.js': 'found it too',
},
},
});
});
afterEach(() => {
mockFs.restore();
});
it('should pass expected object path to bucket', async () => {
const {
kind,
metadata: { namespace, name },
} = entity;
// Ensures leading slash is trimmed and encoded path is decoded.
const pngResponse = await request(app).get(
`/${namespace}/${kind}/${name}/img/with%20spaces.png`,
`/${entityTripletPath}/img/with%20spaces.png`,
);
expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
'found it',
);
const jsResponse = await request(app).get(
`/${namespace}/${kind}/${name}/some%20folder/also%20with%20spaces.js`,
`/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
);
expect(jsResponse.text).toEqual('found it too');
});
it('should pass text/plain content-type for html', async () => {
const {
kind,
metadata: { namespace, name },
} = entity;
const htmlResponse = await request(app).get(
`/${namespace}/${kind}/${name}/html/unsafe.html`,
`/${entityTripletPath}/html/unsafe.html`,
);
expect(htmlResponse.text).toEqual('<html></html>');
expect(htmlResponse.header).toMatchObject({
@@ -395,7 +334,7 @@ describe('publishing with valid credentials', () => {
});
const svgResponse = await request(app).get(
`/${namespace}/${kind}/${name}/img/unsafe.svg`,
`/${entityTripletPath}/img/unsafe.svg`,
);
expect(svgResponse.text).toEqual('<svg></svg>');
expect(svgResponse.header).toMatchObject({
@@ -32,6 +32,7 @@ import { Logger } from 'winston';
import {
getFileTreeRecursively,
getHeadersForFileExtension,
lowerCaseEntityTriplet,
lowerCaseEntityTripletInStoragePath,
} from './helpers';
import {
@@ -241,7 +242,9 @@ export class AzureBlobStoragePublish implements PublisherBase {
async fetchTechDocsMetadata(
entityName: EntityName,
): Promise<TechDocsMetadata> {
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
const entityRootDir = lowerCaseEntityTriplet(
`${entityName.namespace}/${entityName.kind}/${entityName.name}`,
);
try {
const techdocsMetadataJson = await this.download(
this.containerName,
@@ -298,7 +301,9 @@ export class AzureBlobStoragePublish implements PublisherBase {
* can be used to verify if there are any pre-generated docs available to serve.
*/
hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const entityRootDir = lowerCaseEntityTriplet(
`${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`,
);
return this.storageClient
.getContainerClient(this.containerName)
.getBlockBlobClient(`${entityRootDir}/index.html`)
@@ -189,7 +189,10 @@ describe('GoogleGCSPublish', () => {
describe('hasDocsBeenGenerated', () => {
it('should return true if docs has been generated', async () => {
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
const entityRootDir = getEntityRootDir({
...entity,
kind: entity.kind.toLowerCase(),
});
mockFs({
[entityRootDir]: {
@@ -216,7 +219,10 @@ describe('GoogleGCSPublish', () => {
it('should return tech docs metadata', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
const entityRootDir = getEntityRootDir({
...entity,
kind: entity.kind.toLowerCase(),
});
mockFs({
[entityRootDir]: {
@@ -238,7 +244,10 @@ describe('GoogleGCSPublish', () => {
it('should return tech docs metadata when json encoded with single quotes', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
const entityRootDir = getEntityRootDir({
...entity,
kind: entity.kind.toLowerCase(),
});
mockFs({
[entityRootDir]: {
@@ -261,7 +270,10 @@ describe('GoogleGCSPublish', () => {
it('should return an error if the techdocs_metadata.json file is not present', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
const entityRootDir = getEntityRootDir({
...entity,
kind: entity.kind.toLowerCase(),
});
const fails = publisher.fetchTechDocsMetadata(entityNameMock);
@@ -277,7 +289,10 @@ describe('GoogleGCSPublish', () => {
describe('docsRouter', () => {
let app: express.Express;
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
const entityRootDir = getEntityRootDir({
...entity,
kind: entity.kind.toLowerCase(),
});
beforeEach(() => {
app = express().use(publisher.docsRouter());
@@ -33,6 +33,7 @@ import { Logger } from 'winston';
import {
getFileTreeRecursively,
getHeadersForFileExtension,
lowerCaseEntityTriplet,
lowerCaseEntityTripletInStoragePath,
} from './helpers';
import { MigrateWriteStream } from './migrations';
@@ -77,17 +78,29 @@ export class GoogleGCSPublish implements PublisherBase {
}),
});
return new GoogleGCSPublish(storageClient, bucketName, logger);
const legacyPathCasing =
config.getOptionalBoolean(
'techdocs.legacyUseCaseSensitiveTripletPaths',
) || false;
return new GoogleGCSPublish(
storageClient,
bucketName,
legacyPathCasing,
logger,
);
}
constructor(
private readonly storageClient: Storage,
private readonly bucketName: string,
private readonly legacyPathCasing: boolean,
private readonly logger: Logger,
) {
this.storageClient = storageClient;
this.bucketName = bucketName;
this.logger = logger;
this.legacyPathCasing = legacyPathCasing;
}
/**
@@ -147,9 +160,11 @@ export class GoogleGCSPublish implements PublisherBase {
entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE
}/${entity.kind}/${entity.metadata.name}`;
const destination = lowerCaseEntityTripletInStoragePath(
`${entityRootDir}/${relativeFilePathPosix}`,
); // GCS Bucket file relative path
const destination = this.legacyPathCasing
? lowerCaseEntityTripletInStoragePath(
`${entityRootDir}/${relativeFilePathPosix}`,
)
: `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path
// Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
const uploadFile = limiter(() =>
@@ -174,7 +189,10 @@ export class GoogleGCSPublish implements PublisherBase {
fetchTechDocsMetadata(entityName: EntityName): Promise<TechDocsMetadata> {
return new Promise((resolve, reject) => {
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
const entityRootDir = this.legacyPathCasing
? entityTriplet
: lowerCaseEntityTriplet(entityTriplet);
const fileStreamChunks: Array<any> = [];
this.storageClient
@@ -206,7 +224,9 @@ export class GoogleGCSPublish implements PublisherBase {
const decodedUri = decodeURI(req.path.replace(/^\//, ''));
// filePath example - /default/component/documented-component/index.html
const filePath = lowerCaseEntityTripletInStoragePath(decodedUri);
const filePath = this.legacyPathCasing
? decodedUri
: lowerCaseEntityTripletInStoragePath(decodedUri);
// Files with different extensions (CSS, HTML) need to be served with different headers
const fileExtension = path.extname(filePath);
@@ -239,7 +259,11 @@ export class GoogleGCSPublish implements PublisherBase {
*/
async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
return new Promise(resolve => {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const entityRootDir = this.legacyPathCasing
? entityTriplet
: lowerCaseEntityTriplet(entityTriplet);
this.storageClient
.bucket(this.bucketName)
.file(`${entityRootDir}/index.html`)
@@ -83,6 +83,23 @@ export const getFileTreeRecursively = async (
return fileList;
};
/**
* Returns a lower-cased version of entity's triplet in the form of a path.
*
* Path must not include a starting slash.
*
* @example
* lowerCaseEntityTriplet('default/Component/backstage')
* // return default/component/backstage
*/
export const lowerCaseEntityTriplet = (originalPath: string): string => {
const [namespace, kind, name, ...parts] = originalPath.split('/');
const lowerNamespace = namespace.toLowerCase();
const lowerKind = kind.toLowerCase();
const lowerName = name.toLowerCase();
return [lowerNamespace, lowerKind, lowerName, ...parts].join('/');
};
/**
* Returns the version of an object's storage path where the first three parts
* of the path (the entity triplet of namespace, kind, and name) are
@@ -90,9 +107,11 @@ export const getFileTreeRecursively = async (
*
* Path must not include a starting slash.
*
* Throws an error if the path does not appear to be an entity triplet.
*
* @example
* lowerCaseEntityTripletInStoragePath('default/Component/backstage')
* // return default/component/backstage
* lowerCaseEntityTripletInStoragePath('default/Component/backstage/file.txt')
* // return default/component/backstage/file.txt
*/
export const lowerCaseEntityTripletInStoragePath = (
originalPath: string,
@@ -105,9 +124,5 @@ export const lowerCaseEntityTripletInStoragePath = (
`Encountered file unmanaged by TechDocs ${originalPath}. Skipping.`,
);
}
const [namespace, kind, name, ...parts] = originalPath.split('/');
const lowerNamespace = namespace.toLowerCase();
const lowerKind = kind.toLowerCase();
const lowerName = name.toLowerCase();
return [lowerNamespace, lowerKind, lowerName, ...parts].join('/');
return lowerCaseEntityTriplet(originalPath);
};