From af8e1d1b6d098a60e0c95b95110427a257ae6701 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 4 Aug 2021 11:44:24 +0200 Subject: [PATCH 01/15] feat(techdocs-common): lowercase entity triplet path for GCS Signed-off-by: Camila Belo --- .../src/stages/publish/googleStorage.ts | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 9def24722a..3b8ed36edd 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -13,7 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { + Entity, + EntityName, + ENTITY_DEFAULT_NAMESPACE, +} from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { FileExistsResponse, @@ -26,7 +30,11 @@ import createLimiter from 'p-limit'; import path from 'path'; import { Readable } from 'stream'; import { Logger } from 'winston'; -import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { + getFileTreeRecursively, + getHeadersForFileExtension, + lowerCaseEntityTripletInStoragePath, +} from './helpers'; import { MigrateWriteStream } from './migrations'; import { PublisherBase, @@ -135,8 +143,13 @@ export class GoogleGCSPublish implements PublisherBase { .join(path.posix.sep); // The / delimiter is intentional since it represents the cloud storage and not the local file system. - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path + const entityRootDir = `${ + entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE + }/${entity.kind}/${entity.metadata.name}`; + + const destination = lowerCaseEntityTripletInStoragePath( + `${entityRootDir}/${relativeFilePathPosix}`, + ); // GCS Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => @@ -190,8 +203,10 @@ export class GoogleGCSPublish implements PublisherBase { docsRouter(): express.Handler { return (req, res) => { // Decode and trim the leading forward slash - // filePath example - /default/Component/documented-component/index.html - const filePath = decodeURI(req.path.replace(/^\//, '')); + const decodedUri = decodeURI(req.path.replace(/^\//, '')); + + // filePath example - /default/component/documented-component/index.html + const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); From 1fb4b4eb44311bd4375bbeb78a7c252c50e1ed6f Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 4 Aug 2021 12:05:10 +0200 Subject: [PATCH 02/15] feat(techdocs-common): lowercase entity triplet path for AWS Signed-off-by: Camila Belo --- .../src/stages/publish/awsS3.ts | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 7bdc91ffd4..d08a7d55e7 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -13,7 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { + Entity, + EntityName, + ENTITY_DEFAULT_NAMESPACE, +} from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import aws, { Credentials } from 'aws-sdk'; import { ListObjectsV2Output, ManagedUpload } from 'aws-sdk/clients/s3'; @@ -196,8 +200,12 @@ export class AwsS3Publish implements PublisherBase { .join(path.posix.sep); // The / delimiter is intentional since it represents the cloud storage and not the local file system. - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = `${entityRootDir}/${relativeFilePathPosix}`; // S3 Bucket file relative path + const entityRootDir = `${ + entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE + }/${entity.kind}/${entity.metadata.name}`; + const destination = lowerCaseEntityTripletInStoragePath( + `${entityRootDir}/${relativeFilePathPosix}`, + ); // S3 Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => { @@ -268,8 +276,10 @@ export class AwsS3Publish implements PublisherBase { docsRouter(): express.Handler { return async (req, res) => { // Decode and trim the leading forward slash - // filePath example - /default/Component/documented-component/index.html - const filePath = decodeURI(req.path.replace(/^\//, '')); + const decodedUri = decodeURI(req.path.replace(/^\//, '')); + + // filePath example - /default/component/documented-component/index.html + const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); From a72470c1932067928336943c9ef307aff4ef296f Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 4 Aug 2021 12:11:19 +0200 Subject: [PATCH 03/15] feat(techdocs-common): lowercase entity triplet path for Azure Signed-off-by: Camila Belo --- .../src/stages/publish/azureBlobStorage.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index fe8fe9c2ce..8af6137e0c 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -18,7 +18,11 @@ import { BlobServiceClient, StorageSharedKeyCredential, } from '@azure/storage-blob'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { + Entity, + EntityName, + ENTITY_DEFAULT_NAMESPACE, +} from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import express from 'express'; import JSON5 from 'json5'; @@ -159,8 +163,12 @@ export class AzureBlobStoragePublish implements PublisherBase { .join(path.posix.sep); // The / delimiter is intentional since it represents the cloud storage and not the local file system. - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Azure Blob Storage Container file relative path + const entityRootDir = `${ + entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE + }/${entity.kind}/${entity.metadata.name}`; + const destination = lowerCaseEntityTripletInStoragePath( + `${entityRootDir}/${relativeFilePathPosix}`, + ); // Azure Blob Storage Container file relative path return limiter(async () => { const response = await this.storageClient .getContainerClient(this.containerName) @@ -259,8 +267,11 @@ export class AzureBlobStoragePublish implements PublisherBase { docsRouter(): express.Handler { return (req, res) => { // Decode and trim the leading forward slash + const decodedUri = decodeURI(req.path.replace(/^\//, '')); + // filePath example - /default/Component/documented-component/index.html - const filePath = decodeURI(req.path.replace(/^\//, '')); + const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); + // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = platformPath.extname(filePath); const responseHeaders = getHeadersForFileExtension(fileExtension); From a7a55632d398e6ef2beafee9076d45b7f3d9ebd9 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 4 Aug 2021 22:23:06 +0200 Subject: [PATCH 04/15] test(techdocs-common): fix failing tests for all providers Co-authored-by: Eric Peterson Signed-off-by: Camila Belo --- packages/techdocs-common/__mocks__/aws-sdk.ts | 9 +- .../src/stages/publish/awsS3.test.ts | 319 +++++++---------- .../src/stages/publish/awsS3.ts | 9 +- .../stages/publish/azureBlobStorage.test.ts | 333 +++++++----------- .../src/stages/publish/azureBlobStorage.ts | 9 +- .../src/stages/publish/googleStorage.test.ts | 25 +- .../src/stages/publish/googleStorage.ts | 38 +- .../src/stages/publish/helpers.ts | 29 +- 8 files changed, 360 insertions(+), 411 deletions(-) diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts index 2826a8d8bb..763fad0bda 100644 --- a/packages/techdocs-common/__mocks__/aws-sdk.ts +++ b/packages/techdocs-common/__mocks__/aws-sdk.ts @@ -42,12 +42,6 @@ const checkFileExists = async (Key: string): Promise => { }; export class S3 { - private readonly options; - - constructor(options: S3Types.ClientConfiguration) { - this.options = options; - } - headObject({ Key }: { Key: string }) { return { promise: async () => { @@ -61,7 +55,7 @@ export class S3 { getObject({ Key }: { Key: string }) { const filePath = path.join(rootDir, Key); return { - promise: () => checkFileExists(filePath), + promise: async () => await checkFileExists(filePath), createReadStream: () => { const emitter = new EventEmitter(); process.nextTick(() => { @@ -86,7 +80,6 @@ export class S3 { if (Bucket === 'errorBucket') { throw new Error('Bucket does not exist'); } - return {}; }, }; diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index 81f06716b1..d25b54c529 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -15,11 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { - Entity, - EntityName, - ENTITY_DEFAULT_NAMESPACE, -} from '@backstage/catalog-model'; +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; @@ -27,30 +23,9 @@ import mockFs from 'mock-fs'; import os from 'os'; import path from 'path'; import { AwsS3Publish } from './awsS3'; -import { PublisherBase, TechDocsMetadata } from './types'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock aws-sdk client library -const createMockEntity = (annotations = {}): Entity => { - return { - apiVersion: 'version', - kind: 'TestKind', - metadata: { - name: 'test-component-name', - namespace: 'test-namespace', - annotations: { - ...annotations, - }, - }, - }; -}; - -const createMockEntityName = (): EntityName => ({ - kind: 'TestKind', - name: 'test-component-name', - namespace: 'test-namespace', -}); - const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; const getEntityRootDir = (entity: Entity) => { @@ -64,10 +39,7 @@ const getEntityRootDir = (entity: Entity) => { const logger = getVoidLogger(); -let publisher: PublisherBase; - -beforeEach(() => { - mockFs.restore(); +const createPublisherFromConfig = (bucketName: string = 'bucketName') => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -78,80 +50,119 @@ beforeEach(() => { accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', }, - bucketName: 'bucketName', + bucketName, }, }, }, }); - publisher = AwsS3Publish.fromConfig(mockConfig, logger); -}); + return AwsS3Publish.fromConfig(mockConfig, logger); +}; describe('AwsS3Publish', () => { + const entity = { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'backstage', + + namespace: 'default', + annotations: {}, + }, + }; + + const entityName = { + kind: 'Component', + name: 'backstage', + namespace: 'default', + }; + + const techdocsMetadata = { + site_name: 'backstage', + site_description: 'site_content', + etag: 'etag', + }; + + const localRootDir = getEntityRootDir(entity); + const storageRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); + const storageSingleQuoteDir = getEntityRootDir({ + metadata: { + namespace: 'storage', + name: 'quote', + }, + kind: 'single', + } as Entity); + + beforeAll(() => { + mockFs({ + [localRootDir]: { + 'index.html': '', + '404.html': '', + assets: { + 'main.css': '', + }, + attachments: { + 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), + }, + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + }, + [storageRootDir]: { + 'index.html': '', + '404.html': '', + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + assets: { + 'main.css': '', + }, + attachments: { + 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), + }, + html: { + 'unsafe.html': '', + }, + img: { + 'with spaces.png': 'found it', + 'unsafe.svg': '', + }, + 'some folder': { + 'also with spaces.js': 'found it too', + }, + }, + [storageSingleQuoteDir]: { + 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, + }, + }); + }); + + afterAll(() => { + mockFs.restore(); + }); + describe('getReadiness', () => { it('should validate correct config', async () => { + const publisher = createPublisherFromConfig(); expect(await publisher.getReadiness()).toEqual({ isAvailable: true, }); }); it('should reject incorrect config', async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'awsS3', - awsS3: { - credentials: { - accessKeyId: 'accessKeyId', - secretAccessKey: 'secretAccessKey', - }, - // this bucket name will throw an error - bucketName: 'errorBucket', - }, - }, - }, - }); - - const errorPublisher = AwsS3Publish.fromConfig(mockConfig, logger); - - expect(await errorPublisher.getReadiness()).toEqual({ + const publisher = createPublisherFromConfig('errorBucket'); + expect(await publisher.getReadiness()).toEqual({ isAvailable: false, }); }); }); describe('publish', () => { - beforeEach(() => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', - }, - attachments: { - 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), - }, - }, - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - it('should publish a directory', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - + const publisher = createPublisherFromConfig(); expect( await publisher.publish({ entity, - directory: entityRootDir, + directory: localRootDir, }), ).toBeUndefined(); }); @@ -165,172 +176,114 @@ describe('AwsS3Publish', () => { 'generatedDirectory', ); - const entity = createMockEntity(); - await expect( - publisher.publish({ - entity, - directory: wrongPathToGeneratedDirectory, - }), - ).rejects.toThrowError(); + const publisher = createPublisherFromConfig(); const fails = publisher.publish({ entity, directory: wrongPathToGeneratedDirectory, }); - // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error - // Issue reported https://github.com/tschaub/mock-fs/issues/118 await expect(fails).rejects.toMatchObject({ message: expect.stringContaining( 'Unable to upload file(s) to AWS S3. Error: Failed to read template directory: ENOENT, no such file or directory', ), }); + await expect(fails).rejects.toMatchObject({ message: expect.stringContaining(wrongPathToGeneratedDirectory), }); - - mockFs.restore(); }); }); describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': 'file-content', - }, - }); - + const publisher = createPublisherFromConfig(); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); - mockFs.restore(); }); it('should return false if docs has not been generated', async () => { - const entity = createMockEntity(); - - expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false); + const publisher = createPublisherFromConfig(); + expect( + await publisher.hasDocsBeenGenerated({ + kind: 'entity', + metadata: { + namespace: 'invalid', + name: 'triplet', + }, + } as Entity), + ).toBe(false); }); }); describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': - '{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}', - }, - }); - - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; - expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); + const publisher = createPublisherFromConfig(); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, - }, - }); - - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; + const publisher = createPublisherFromConfig(); expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); + await publisher.fetchTechDocsMetadata({ + namespace: 'storage', + kind: 'single', + name: 'quote', + }), + ).toStrictEqual(techdocsMetadata); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const publisher = createPublisherFromConfig(); - const fails = publisher.fetchTechDocsMetadata(entityNameMock); + const invalidEntityName = { + namespace: 'invalid', + kind: 'triplet', + name: 'path', + }; + + const techDocsMetadaFilePath = path.join( + rootDir, + ...Object.values(invalidEntityName), + 'techdocs_metadata.json', + ); + + const fails = publisher.fetchTechDocsMetadata(invalidEntityName); - const errorPath = path.join(entityRootDir, 'techdocs_metadata.json'); await expect(fails).rejects.toMatchObject({ - message: `TechDocs metadata fetch failed, The file ${errorPath} does not exist !`, + message: `TechDocs metadata fetch failed, The file ${techDocsMetadaFilePath} does not exist !`, }); }); }); describe('docsRouter', () => { + const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + let app: express.Express; - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); beforeEach(() => { + const publisher = createPublisherFromConfig(); app = express().use(publisher.docsRouter()); - - mockFs.restore(); - mockFs({ - [entityRootDir]: { - html: { - 'unsafe.html': '', - }, - img: { - 'with spaces.png': 'found it', - 'unsafe.svg': '', - }, - 'some folder': { - 'also with spaces.js': 'found it too', - }, - }, - }); - }); - - afterEach(() => { - mockFs.restore(); }); it('should pass expected object path to bucket', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/with%20spaces.png`, + `/${entityTripletPath}/img/with%20spaces.png`, ); expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( 'found it', ); const jsResponse = await request(app).get( - `/${namespace}/${kind}/${name}/some%20folder/also%20with%20spaces.js`, + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, ); expect(jsResponse.text).toEqual('found it too'); }); it('should pass text/plain content-type for html', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - const htmlResponse = await request(app).get( - `/${namespace}/${kind}/${name}/html/unsafe.html`, + `/${entityTripletPath}/html/unsafe.html`, ); expect(htmlResponse.text).toEqual(''); expect(htmlResponse.header).toMatchObject({ @@ -338,7 +291,7 @@ describe('AwsS3Publish', () => { }); const svgResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/unsafe.svg`, + `/${entityTripletPath}/img/unsafe.svg`, ); expect(svgResponse.text).toEqual(''); expect(svgResponse.header).toMatchObject({ diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index d08a7d55e7..26a896cbac 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -32,6 +32,7 @@ import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension, + lowerCaseEntityTriplet, lowerCaseEntityTripletInStoragePath, } from './helpers'; import { @@ -238,7 +239,9 @@ export class AwsS3Publish implements PublisherBase { ): Promise { try { return await new Promise(async (resolve, reject) => { - const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityRootDir = lowerCaseEntityTriplet( + `${entityName.namespace}/${entityName.kind}/${entityName.name}`, + ); const stream = this.storageClient .getObject({ @@ -310,7 +313,9 @@ export class AwsS3Publish implements PublisherBase { */ async hasDocsBeenGenerated(entity: Entity): Promise { try { - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityRootDir = lowerCaseEntityTriplet( + `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`, + ); await this.storageClient .headObject({ Bucket: this.bucketName, diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index e55314a71b..60a70a2e0d 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -15,11 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { - Entity, - EntityName, - ENTITY_DEFAULT_NAMESPACE, -} from '@backstage/catalog-model'; +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; @@ -27,30 +23,9 @@ import mockFs from 'mock-fs'; import os from 'os'; import path from 'path'; import { AzureBlobStoragePublish } from './azureBlobStorage'; -import { PublisherBase, TechDocsMetadata } from './types'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Azure client library -const createMockEntity = (annotations = {}) => { - return { - apiVersion: 'version', - kind: 'TestKind', - metadata: { - name: 'test-component-name', - namespace: 'test-namespace', - annotations: { - ...annotations, - }, - }, - }; -}; - -const createMockEntityName = (): EntityName => ({ - kind: 'TestKind', - name: 'test-component-name', - namespace: 'test-namespace', -}); - const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; const getEntityRootDir = (entity: Entity) => { @@ -66,9 +41,10 @@ const logger = getVoidLogger(); jest.spyOn(logger, 'info').mockReturnValue(logger); jest.spyOn(logger, 'error').mockReturnValue(logger); -let publisher: PublisherBase; -beforeEach(async () => { - mockFs.restore(); +const createPublisherFromConfig = ({ + accountName = 'accountName', + containerName = 'containerName', +}: undefined | { accountName?: string; containerName?: string } = {}) => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -76,49 +52,107 @@ beforeEach(async () => { type: 'azureBlobStorage', azureBlobStorage: { credentials: { - accountName: 'accountName', + accountName, accountKey: 'accountKey', }, - containerName: 'containerName', + containerName, }, }, }, }); - publisher = AzureBlobStoragePublish.fromConfig(mockConfig, logger); -}); + return AzureBlobStoragePublish.fromConfig(mockConfig, logger); +}; describe('publishing with valid credentials', () => { + const entity = { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'backstage', + namespace: 'default', + annotations: {}, + }, + }; + + const entityName = { + kind: 'Component', + name: 'backstage', + namespace: 'default', + }; + + const techdocsMetadata = { + site_name: 'backstage', + site_description: 'site_content', + etag: 'etag', + }; + + const localRootDir = getEntityRootDir(entity); + const storageRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); + const storageSingleQuoteDir = getEntityRootDir({ + metadata: { + namespace: 'storage', + name: 'quote', + }, + kind: 'single', + } as Entity); + + beforeEach(() => { + (logger.info as jest.Mock).mockClear(); + (logger.error as jest.Mock).mockClear(); + }); + + beforeAll(async () => { + mockFs({ + [localRootDir]: { + 'index.html': 'file-content', + '404.html': '', + assets: { + 'main.css': '', + }, + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + }, + [storageRootDir]: { + 'index.html': '', + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + html: { + 'unsafe.html': '', + }, + img: { + 'with spaces.png': 'found it', + 'unsafe.svg': '', + }, + 'some folder': { + 'also with spaces.js': 'found it too', + }, + }, + [storageSingleQuoteDir]: { + 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, + }, + }); + }); + + afterAll(() => { + mockFs.restore(); + }); + describe('getReadiness', () => { it('should validate correct config', async () => { + const publisher = createPublisherFromConfig(); expect(await publisher.getReadiness()).toEqual({ isAvailable: true, }); }); it('should reject incorrect config', async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'azureBlobStorage', - azureBlobStorage: { - credentials: { - accountName: 'accountName', - accountKey: 'accountKey', - }, - containerName: 'bad_container', - }, - }, - }, + const publisher = createPublisherFromConfig({ + containerName: 'bad_container', }); - const errorPublisher = await AzureBlobStoragePublish.fromConfig( - mockConfig, - logger, - ); - - expect(await errorPublisher.getReadiness()).toEqual({ + expect(await publisher.getReadiness()).toEqual({ isAvailable: false, }); @@ -131,32 +165,14 @@ describe('publishing with valid credentials', () => { }); describe('publish', () => { - beforeEach(() => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', - }, - }, - }); - }); - it('should publish a directory', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - + const publisher = createPublisherFromConfig(); expect( await publisher.publish({ entity, - directory: entityRootDir, + directory: localRootDir, }), ).toBeUndefined(); - mockFs.restore(); }); it('should fail to publish a directory', async () => { @@ -168,7 +184,9 @@ describe('publishing with valid credentials', () => { 'generatedDirectory', ); - const entity = createMockEntity(); + const publisher = createPublisherFromConfig({ + containerName: 'bad_container', + }); const fails = publisher.publish({ entity, @@ -180,46 +198,22 @@ describe('publishing with valid credentials', () => { `Unable to upload file(s) to Azure Blob Storage. Error: Failed to read template directory: ENOENT, no such file or directory`, ), }); + await expect(fails).rejects.toMatchObject({ message: expect.stringContaining(wrongPathToGeneratedDirectory), }); - - mockFs.restore(); }); it('reports an error when bad account credentials', async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'azureBlobStorage', - azureBlobStorage: { - credentials: { - accountName: 'failupload', - accountKey: 'accountKey', - }, - containerName: 'containerName', - }, - }, - }, - }); - - publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger); - - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': '', - }, + const publisher = createPublisherFromConfig({ + accountName: 'failupload', }); let error; try { await publisher.publish({ entity, - directory: entityRootDir, + directory: localRootDir, }); } catch (e) { error = e; @@ -232,91 +226,64 @@ describe('publishing with valid credentials', () => { expect(logger.error).toHaveBeenCalledWith( expect.stringContaining( `Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join( - entityRootDir, - 'index.html', + localRootDir, + '404.html', )} with status code 500`, ), ); - - mockFs.restore(); }); }); describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': 'file-content', - }, - }); - + const publisher = createPublisherFromConfig(); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); - mockFs.restore(); }); it('should return false if docs has not been generated', async () => { - const entity = createMockEntity(); - - expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false); + const publisher = createPublisherFromConfig(); + expect( + await publisher.hasDocsBeenGenerated({ + kind: 'triplet', + metadata: { + namespace: 'invalid', + name: 'path', + }, + } as Entity), + ).toBe(false); }); }); describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': - '{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}', - }, - }); - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; - expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); + const publisher = createPublisherFromConfig(); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, - }, - }); - - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; + const publisher = createPublisherFromConfig(); expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); + await publisher.fetchTechDocsMetadata({ + namespace: 'storage', + kind: 'single', + name: 'quote', + }), + ).toStrictEqual(techdocsMetadata); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const publisher = createPublisherFromConfig(); + const invalidEntityName = { + namespace: 'invalid', + kind: 'triplet', + name: 'path', + }; let error; try { - await publisher.fetchTechDocsMetadata(entityNameMock); + await publisher.fetchTechDocsMetadata(invalidEntityName); } catch (e) { error = e; } @@ -324,7 +291,8 @@ describe('publishing with valid credentials', () => { expect(error).toEqual( new Error( `TechDocs metadata fetch failed, The file ${path.join( - entityRootDir, + rootDir, + ...Object.values(invalidEntityName), 'techdocs_metadata.json', )} does not exist !`, ), @@ -333,61 +301,32 @@ describe('publishing with valid credentials', () => { }); describe('docsRouter', () => { + const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + let app: express.Express; - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); beforeEach(() => { + const publisher = createPublisherFromConfig(); app = express().use(publisher.docsRouter()); - - mockFs.restore(); - mockFs({ - [entityRootDir]: { - html: { - 'unsafe.html': '', - }, - img: { - 'with spaces.png': 'found it', - 'unsafe.svg': '', - }, - 'some folder': { - 'also with spaces.js': 'found it too', - }, - }, - }); - }); - - afterEach(() => { - mockFs.restore(); }); it('should pass expected object path to bucket', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/with%20spaces.png`, + `/${entityTripletPath}/img/with%20spaces.png`, ); expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( 'found it', ); const jsResponse = await request(app).get( - `/${namespace}/${kind}/${name}/some%20folder/also%20with%20spaces.js`, + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, ); expect(jsResponse.text).toEqual('found it too'); }); it('should pass text/plain content-type for html', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - const htmlResponse = await request(app).get( - `/${namespace}/${kind}/${name}/html/unsafe.html`, + `/${entityTripletPath}/html/unsafe.html`, ); expect(htmlResponse.text).toEqual(''); expect(htmlResponse.header).toMatchObject({ @@ -395,7 +334,7 @@ describe('publishing with valid credentials', () => { }); const svgResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/unsafe.svg`, + `/${entityTripletPath}/img/unsafe.svg`, ); expect(svgResponse.text).toEqual(''); expect(svgResponse.header).toMatchObject({ diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index 8af6137e0c..76f91cff66 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -32,6 +32,7 @@ import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension, + lowerCaseEntityTriplet, lowerCaseEntityTripletInStoragePath, } from './helpers'; import { @@ -241,7 +242,9 @@ export class AzureBlobStoragePublish implements PublisherBase { async fetchTechDocsMetadata( entityName: EntityName, ): Promise { - const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityRootDir = lowerCaseEntityTriplet( + `${entityName.namespace}/${entityName.kind}/${entityName.name}`, + ); try { const techdocsMetadataJson = await this.download( this.containerName, @@ -298,7 +301,9 @@ export class AzureBlobStoragePublish implements PublisherBase { * can be used to verify if there are any pre-generated docs available to serve. */ hasDocsBeenGenerated(entity: Entity): Promise { - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityRootDir = lowerCaseEntityTriplet( + `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`, + ); return this.storageClient .getContainerClient(this.containerName) .getBlockBlobClient(`${entityRootDir}/index.html`) diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index 8a00fc9708..a1c9d70fd3 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -189,7 +189,10 @@ describe('GoogleGCSPublish', () => { describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const entityRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); mockFs({ [entityRootDir]: { @@ -216,7 +219,10 @@ describe('GoogleGCSPublish', () => { it('should return tech docs metadata', async () => { const entityNameMock = createMockEntityName(); const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const entityRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); mockFs({ [entityRootDir]: { @@ -238,7 +244,10 @@ describe('GoogleGCSPublish', () => { it('should return tech docs metadata when json encoded with single quotes', async () => { const entityNameMock = createMockEntityName(); const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const entityRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); mockFs({ [entityRootDir]: { @@ -261,7 +270,10 @@ describe('GoogleGCSPublish', () => { it('should return an error if the techdocs_metadata.json file is not present', async () => { const entityNameMock = createMockEntityName(); const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const entityRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); const fails = publisher.fetchTechDocsMetadata(entityNameMock); @@ -277,7 +289,10 @@ describe('GoogleGCSPublish', () => { describe('docsRouter', () => { let app: express.Express; const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const entityRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); beforeEach(() => { app = express().use(publisher.docsRouter()); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 3b8ed36edd..3aa7b09b74 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -33,6 +33,7 @@ import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension, + lowerCaseEntityTriplet, lowerCaseEntityTripletInStoragePath, } from './helpers'; import { MigrateWriteStream } from './migrations'; @@ -77,17 +78,29 @@ export class GoogleGCSPublish implements PublisherBase { }), }); - return new GoogleGCSPublish(storageClient, bucketName, logger); + const legacyPathCasing = + config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + + return new GoogleGCSPublish( + storageClient, + bucketName, + legacyPathCasing, + logger, + ); } constructor( private readonly storageClient: Storage, private readonly bucketName: string, + private readonly legacyPathCasing: boolean, private readonly logger: Logger, ) { this.storageClient = storageClient; this.bucketName = bucketName; this.logger = logger; + this.legacyPathCasing = legacyPathCasing; } /** @@ -147,9 +160,11 @@ export class GoogleGCSPublish implements PublisherBase { entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE }/${entity.kind}/${entity.metadata.name}`; - const destination = lowerCaseEntityTripletInStoragePath( - `${entityRootDir}/${relativeFilePathPosix}`, - ); // GCS Bucket file relative path + const destination = this.legacyPathCasing + ? lowerCaseEntityTripletInStoragePath( + `${entityRootDir}/${relativeFilePathPosix}`, + ) + : `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => @@ -174,7 +189,10 @@ export class GoogleGCSPublish implements PublisherBase { fetchTechDocsMetadata(entityName: EntityName): Promise { return new Promise((resolve, reject) => { - const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); const fileStreamChunks: Array = []; this.storageClient @@ -206,7 +224,9 @@ export class GoogleGCSPublish implements PublisherBase { const decodedUri = decodeURI(req.path.replace(/^\//, '')); // filePath example - /default/component/documented-component/index.html - const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); + const filePath = this.legacyPathCasing + ? decodedUri + : lowerCaseEntityTripletInStoragePath(decodedUri); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); @@ -239,7 +259,11 @@ export class GoogleGCSPublish implements PublisherBase { */ async hasDocsBeenGenerated(entity: Entity): Promise { return new Promise(resolve => { - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); + this.storageClient .bucket(this.bucketName) .file(`${entityRootDir}/index.html`) diff --git a/packages/techdocs-common/src/stages/publish/helpers.ts b/packages/techdocs-common/src/stages/publish/helpers.ts index ed681eb8ba..dd62865468 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.ts @@ -83,6 +83,23 @@ export const getFileTreeRecursively = async ( return fileList; }; +/** + * Returns a lower-cased version of entity's triplet in the form of a path. + * + * Path must not include a starting slash. + * + * @example + * lowerCaseEntityTriplet('default/Component/backstage') + * // return default/component/backstage + */ +export const lowerCaseEntityTriplet = (originalPath: string): string => { + const [namespace, kind, name, ...parts] = originalPath.split('/'); + const lowerNamespace = namespace.toLowerCase(); + const lowerKind = kind.toLowerCase(); + const lowerName = name.toLowerCase(); + return [lowerNamespace, lowerKind, lowerName, ...parts].join('/'); +}; + /** * Returns the version of an object's storage path where the first three parts * of the path (the entity triplet of namespace, kind, and name) are @@ -90,9 +107,11 @@ export const getFileTreeRecursively = async ( * * Path must not include a starting slash. * + * Throws an error if the path does not appear to be an entity triplet. + * * @example - * lowerCaseEntityTripletInStoragePath('default/Component/backstage') - * // return default/component/backstage + * lowerCaseEntityTripletInStoragePath('default/Component/backstage/file.txt') + * // return default/component/backstage/file.txt */ export const lowerCaseEntityTripletInStoragePath = ( originalPath: string, @@ -105,9 +124,5 @@ export const lowerCaseEntityTripletInStoragePath = ( `Encountered file unmanaged by TechDocs ${originalPath}. Skipping.`, ); } - const [namespace, kind, name, ...parts] = originalPath.split('/'); - const lowerNamespace = namespace.toLowerCase(); - const lowerKind = kind.toLowerCase(); - const lowerName = name.toLowerCase(); - return [lowerNamespace, lowerKind, lowerName, ...parts].join('/'); + return lowerCaseEntityTriplet(originalPath); }; From bf4ab567502c0bb29a425e775e6f211a4d1d6d84 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Thu, 5 Aug 2021 15:55:41 +0200 Subject: [PATCH 05/15] feat(techdocs-common): add legacy casing config for all providers Co-authored-by: Eric Peterson Signed-off-by: Camila Belo --- .../src/stages/publish/awsS3.test.ts | 120 ++++-- .../src/stages/publish/awsS3.ts | 41 +- .../stages/publish/azureBlobStorage.test.ts | 105 +++-- .../src/stages/publish/azureBlobStorage.ts | 43 +- .../src/stages/publish/googleStorage.test.ts | 373 +++++++++--------- .../src/stages/publish/googleStorage.ts | 9 +- plugins/techdocs-backend/config.d.ts | 5 + 7 files changed, 416 insertions(+), 280 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index d25b54c529..715786e3f9 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -39,7 +39,13 @@ const getEntityRootDir = (entity: Entity) => { const logger = getVoidLogger(); -const createPublisherFromConfig = (bucketName: string = 'bucketName') => { +const createPublisherFromConfig = ({ + bucketName = 'bucketName', + legacyUseCaseSensitiveTripletPaths = false, +}: { + bucketName?: string; + legacyUseCaseSensitiveTripletPaths?: boolean; +} = {}) => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -53,6 +59,7 @@ const createPublisherFromConfig = (bucketName: string = 'bucketName') => { bucketName, }, }, + legacyUseCaseSensitiveTripletPaths, }, }); @@ -96,42 +103,37 @@ describe('AwsS3Publish', () => { kind: 'single', } as Entity); + const files = { + 'index.html': '', + '404.html': '', + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + assets: { + 'main.css': '', + }, + attachments: { + 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), + }, + html: { + 'unsafe.html': '', + }, + img: { + 'with spaces.png': 'found it', + 'unsafe.svg': '', + }, + 'some folder': { + 'also with spaces.js': 'found it too', + }, + }; + beforeAll(() => { mockFs({ - [localRootDir]: { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', - }, - attachments: { - 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), - }, - 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), - }, - [storageRootDir]: { - 'index.html': '', - '404.html': '', - 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), - assets: { - 'main.css': '', - }, - attachments: { - 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), - }, - html: { - 'unsafe.html': '', - }, - img: { - 'with spaces.png': 'found it', - 'unsafe.svg': '', - }, - 'some folder': { - 'also with spaces.js': 'found it too', - }, - }, + [localRootDir]: files, + [storageRootDir]: files, [storageSingleQuoteDir]: { - 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, + 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( + /"/g, + "'", + ), }, }); }); @@ -149,7 +151,9 @@ describe('AwsS3Publish', () => { }); it('should reject incorrect config', async () => { - const publisher = createPublisherFromConfig('errorBucket'); + const publisher = createPublisherFromConfig({ + bucketName: 'errorBucket', + }); expect(await publisher.getReadiness()).toEqual({ isAvailable: false, }); @@ -167,6 +171,18 @@ describe('AwsS3Publish', () => { ).toBeUndefined(); }); + it('should publish a directory as well when legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect( + await publisher.publish({ + entity, + directory: localRootDir, + }), + ).toBeUndefined(); + }); + it('should fail to publish a directory', async () => { const wrongPathToGeneratedDirectory = path.join( rootDir, @@ -201,6 +217,13 @@ describe('AwsS3Publish', () => { expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); + it('should return true if docs has been generated even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + }); + it('should return false if docs has not been generated', async () => { const publisher = createPublisherFromConfig(); expect( @@ -223,6 +246,15 @@ describe('AwsS3Publish', () => { ); }); + it('should return tech docs metadata even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + }); + it('should return tech docs metadata when json encoded with single quotes', async () => { const publisher = createPublisherFromConfig(); expect( @@ -281,6 +313,24 @@ describe('AwsS3Publish', () => { expect(jsResponse.text).toEqual('found it too'); }); + it('should pass expected object path to bucket even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + app = express().use(publisher.docsRouter()); + // Ensures leading slash is trimmed and encoded path is decoded. + const pngResponse = await request(app).get( + `/${entityTripletPath}/img/with%20spaces.png`, + ); + expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( + 'found it', + ); + const jsResponse = await request(app).get( + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, + ); + expect(jsResponse.text).toEqual('found it too'); + }); + it('should pass text/plain content-type for html', async () => { const htmlResponse = await request(app).get( `/${entityTripletPath}/html/unsafe.html`, diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 26a896cbac..f0e2d9cb6e 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -102,7 +102,17 @@ export class AwsS3Publish implements PublisherBase { ...(s3ForcePathStyle && { s3ForcePathStyle }), }); - return new AwsS3Publish(storageClient, bucketName, logger); + const legacyPathCasing = + config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + + return new AwsS3Publish( + storageClient, + bucketName, + legacyPathCasing, + logger, + ); } private static buildCredentials( @@ -139,10 +149,12 @@ export class AwsS3Publish implements PublisherBase { constructor( private readonly storageClient: aws.S3, private readonly bucketName: string, + private readonly legacyPathCasing: boolean, private readonly logger: Logger, ) { this.storageClient = storageClient; this.bucketName = bucketName; + this.legacyPathCasing = legacyPathCasing; this.logger = logger; } @@ -204,9 +216,11 @@ export class AwsS3Publish implements PublisherBase { const entityRootDir = `${ entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE }/${entity.kind}/${entity.metadata.name}`; - const destination = lowerCaseEntityTripletInStoragePath( - `${entityRootDir}/${relativeFilePathPosix}`, - ); // S3 Bucket file relative path + + const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`; + const destination = this.legacyPathCasing + ? relativeFilePathTriplet + : lowerCaseEntityTripletInStoragePath(relativeFilePathTriplet); // S3 Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => { @@ -239,9 +253,10 @@ export class AwsS3Publish implements PublisherBase { ): Promise { try { return await new Promise(async (resolve, reject) => { - const entityRootDir = lowerCaseEntityTriplet( - `${entityName.namespace}/${entityName.kind}/${entityName.name}`, - ); + const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); const stream = this.storageClient .getObject({ @@ -282,7 +297,9 @@ export class AwsS3Publish implements PublisherBase { const decodedUri = decodeURI(req.path.replace(/^\//, '')); // filePath example - /default/component/documented-component/index.html - const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); + const filePath = this.legacyPathCasing + ? decodedUri + : lowerCaseEntityTripletInStoragePath(decodedUri); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); @@ -313,9 +330,11 @@ export class AwsS3Publish implements PublisherBase { */ async hasDocsBeenGenerated(entity: Entity): Promise { try { - const entityRootDir = lowerCaseEntityTriplet( - `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`, - ); + const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); + await this.storageClient .headObject({ Bucket: this.bucketName, diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 60a70a2e0d..78cc7c0c79 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -44,7 +44,12 @@ jest.spyOn(logger, 'error').mockReturnValue(logger); const createPublisherFromConfig = ({ accountName = 'accountName', containerName = 'containerName', -}: undefined | { accountName?: string; containerName?: string } = {}) => { + legacyUseCaseSensitiveTripletPaths = false, +}: { + accountName?: string; + containerName?: string; + legacyUseCaseSensitiveTripletPaths?: boolean; +} = {}) => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -58,6 +63,7 @@ const createPublisherFromConfig = ({ containerName, }, }, + legacyUseCaseSensitiveTripletPaths, }, }); @@ -105,32 +111,37 @@ describe('publishing with valid credentials', () => { (logger.error as jest.Mock).mockClear(); }); + const files = { + 'index.html': '', + '404.html': '', + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + assets: { + 'main.css': '', + }, + attachments: { + 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), + }, + html: { + 'unsafe.html': '', + }, + img: { + 'with spaces.png': 'found it', + 'unsafe.svg': '', + }, + 'some folder': { + 'also with spaces.js': 'found it too', + }, + }; + beforeAll(async () => { mockFs({ - [localRootDir]: { - 'index.html': 'file-content', - '404.html': '', - assets: { - 'main.css': '', - }, - 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), - }, - [storageRootDir]: { - 'index.html': '', - 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), - html: { - 'unsafe.html': '', - }, - img: { - 'with spaces.png': 'found it', - 'unsafe.svg': '', - }, - 'some folder': { - 'also with spaces.js': 'found it too', - }, - }, + [localRootDir]: files, + [storageRootDir]: files, [storageSingleQuoteDir]: { - 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, + 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( + /"/g, + "'", + ), }, }); }); @@ -175,6 +186,18 @@ describe('publishing with valid credentials', () => { ).toBeUndefined(); }); + it('should publish a directory as well when legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect( + await publisher.publish({ + entity, + directory: localRootDir, + }), + ).toBeUndefined(); + }); + it('should fail to publish a directory', async () => { const wrongPathToGeneratedDirectory = path.join( rootDir, @@ -240,6 +263,13 @@ describe('publishing with valid credentials', () => { expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); + it('should return true if docs has been generated even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + }); + it('should return false if docs has not been generated', async () => { const publisher = createPublisherFromConfig(); expect( @@ -262,6 +292,15 @@ describe('publishing with valid credentials', () => { ); }); + it('should return tech docs metadata even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + }); + it('should return tech docs metadata when json encoded with single quotes', async () => { const publisher = createPublisherFromConfig(); expect( @@ -324,6 +363,24 @@ describe('publishing with valid credentials', () => { expect(jsResponse.text).toEqual('found it too'); }); + it('should pass expected object path to bucket even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + app = express().use(publisher.docsRouter()); + // Ensures leading slash is trimmed and encoded path is decoded. + const pngResponse = await request(app).get( + `/${entityTripletPath}/img/with%20spaces.png`, + ); + expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( + 'found it', + ); + const jsResponse = await request(app).get( + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, + ); + expect(jsResponse.text).toEqual('found it too'); + }); + it('should pass text/plain content-type for html', async () => { const htmlResponse = await request(app).get( `/${entityTripletPath}/html/unsafe.html`, diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index 76f91cff66..cb8367ed9d 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -89,16 +89,28 @@ export class AzureBlobStoragePublish implements PublisherBase { credential, ); - return new AzureBlobStoragePublish(storageClient, containerName, logger); + const legacyPathCasing = + config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + + return new AzureBlobStoragePublish( + storageClient, + containerName, + legacyPathCasing, + logger, + ); } constructor( private readonly storageClient: BlobServiceClient, private readonly containerName: string, + private readonly legacyPathCasing: boolean, private readonly logger: Logger, ) { this.storageClient = storageClient; this.containerName = containerName; + this.legacyPathCasing = legacyPathCasing; this.logger = logger; } @@ -167,9 +179,12 @@ export class AzureBlobStoragePublish implements PublisherBase { const entityRootDir = `${ entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE }/${entity.kind}/${entity.metadata.name}`; - const destination = lowerCaseEntityTripletInStoragePath( - `${entityRootDir}/${relativeFilePathPosix}`, - ); // Azure Blob Storage Container file relative path + + const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`; + const destination = this.legacyPathCasing + ? relativeFilePathTriplet + : lowerCaseEntityTripletInStoragePath(relativeFilePathTriplet); // Azure Blob Storage Container file relative path + return limiter(async () => { const response = await this.storageClient .getContainerClient(this.containerName) @@ -242,9 +257,11 @@ export class AzureBlobStoragePublish implements PublisherBase { async fetchTechDocsMetadata( entityName: EntityName, ): Promise { - const entityRootDir = lowerCaseEntityTriplet( - `${entityName.namespace}/${entityName.kind}/${entityName.name}`, - ); + const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); + try { const techdocsMetadataJson = await this.download( this.containerName, @@ -273,7 +290,9 @@ export class AzureBlobStoragePublish implements PublisherBase { const decodedUri = decodeURI(req.path.replace(/^\//, '')); // filePath example - /default/Component/documented-component/index.html - const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); + const filePath = this.legacyPathCasing + ? decodedUri + : lowerCaseEntityTripletInStoragePath(decodedUri); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = platformPath.extname(filePath); @@ -301,9 +320,11 @@ export class AzureBlobStoragePublish implements PublisherBase { * can be used to verify if there are any pre-generated docs available to serve. */ hasDocsBeenGenerated(entity: Entity): Promise { - const entityRootDir = lowerCaseEntityTriplet( - `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`, - ); + const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); + return this.storageClient .getContainerClient(this.containerName) .getBlockBlobClient(`${entityRootDir}/index.html`) diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index a1c9d70fd3..ce08e2f2ce 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -15,11 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { - Entity, - ENTITY_DEFAULT_NAMESPACE, - EntityName, -} from '@backstage/catalog-model'; +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; @@ -27,30 +23,9 @@ import mockFs from 'mock-fs'; import os from 'os'; import path from 'path'; import { GoogleGCSPublish } from './googleStorage'; -import { PublisherBase, TechDocsMetadata } from './types'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Google Cloud Storage client library -const createMockEntity = (annotations = {}): Entity => { - return { - apiVersion: 'version', - kind: 'TestKind', - metadata: { - name: 'test-component-name', - namespace: 'test-namespace', - annotations: { - ...annotations, - }, - }, - }; -}; - -const createMockEntityName = (): EntityName => ({ - kind: 'TestKind', - name: 'test-component-name', - namespace: 'test-namespace', -}); - const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; const getEntityRootDir = (entity: Entity) => { @@ -65,10 +40,13 @@ const getEntityRootDir = (entity: Entity) => { const logger = getVoidLogger(); jest.spyOn(logger, 'info').mockReturnValue(logger); -let publisher: PublisherBase; - -beforeEach(async () => { - mockFs.restore(); +const createPublisherFromConfig = ({ + bucketName = 'bucketName', + legacyUseCaseSensitiveTripletPaths = false, +}: { + bucketName?: string; + legacyUseCaseSensitiveTripletPaths?: boolean; +} = {}) => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -76,76 +54,127 @@ beforeEach(async () => { type: 'googleGcs', googleGcs: { credentials: '{}', - bucketName: 'bucketName', + bucketName, }, }, + legacyUseCaseSensitiveTripletPaths, }, }); - publisher = await GoogleGCSPublish.fromConfig(mockConfig, logger); -}); + return GoogleGCSPublish.fromConfig(mockConfig, logger); +}; describe('GoogleGCSPublish', () => { + const entity = { + apiVersion: 'version', + kind: 'Component', + metadata: { + name: 'backstage', + namespace: 'default', + annotations: {}, + }, + }; + + const entityName = { + kind: 'Component', + name: 'backstage', + namespace: 'default', + }; + + const techdocsMetadata = { + site_name: 'backstage', + site_description: 'site_content', + etag: 'etag', + }; + + const localRootDir = getEntityRootDir(entity); + const storageRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); + const storageSingleQuoteDir = getEntityRootDir({ + metadata: { + namespace: 'storage', + name: 'quote', + }, + kind: 'single', + } as Entity); + + const files = { + 'index.html': '', + '404.html': '', + assets: { + 'main.css': '', + }, + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + html: { + 'unsafe.html': '', + }, + img: { + 'with spaces.png': 'found it', + 'unsafe.svg': '', + }, + 'some folder': { + 'also with spaces.js': 'found it too', + }, + }; + + beforeAll(() => { + mockFs({ + [localRootDir]: files, + [storageRootDir]: files, + [storageSingleQuoteDir]: { + 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( + /"/g, + "'", + ), + }, + }); + }); + + afterAll(() => { + mockFs.restore(); + }); + describe('getReadiness', () => { it('should validate correct config', async () => { + const publisher = createPublisherFromConfig(); expect(await publisher.getReadiness()).toEqual({ isAvailable: true, }); }); it('should reject incorrect config', async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'googleGcs', - googleGcs: { - credentials: '{}', - bucketName: 'errorBucket', - }, - }, - }, + const publisher = createPublisherFromConfig({ + bucketName: 'errorBucket', }); - - const errorPublisher = GoogleGCSPublish.fromConfig(mockConfig, logger); - - expect(await errorPublisher.getReadiness()).toEqual({ + expect(await publisher.getReadiness()).toEqual({ isAvailable: false, }); }); }); describe('publish', () => { - beforeEach(() => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', - }, - }, - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - it('should publish a directory', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - + const publisher = createPublisherFromConfig(); expect( await publisher.publish({ entity, - directory: entityRootDir, + directory: localRootDir, + }), + ).toBeUndefined(); + }); + + it('should publish a directory as well when legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect( + await publisher.publish({ + entity, + directory: localRootDir, }), ).toBeUndefined(); - mockFs.restore(); }); it('should fail to publish a directory', async () => { @@ -157,14 +186,7 @@ describe('GoogleGCSPublish', () => { 'generatedDirectory', ); - const entity = createMockEntity(); - - await expect( - publisher.publish({ - entity, - directory: wrongPathToGeneratedDirectory, - }), - ).rejects.toThrowError(); + const publisher = createPublisherFromConfig(); const fails = publisher.publish({ entity, @@ -178,173 +200,136 @@ describe('GoogleGCSPublish', () => { `Unable to upload file(s) to Google Cloud Storage. Error: Failed to read template directory: ENOENT, no such file or directory`, ), }); + await expect(fails).rejects.toMatchObject({ message: expect.stringContaining(wrongPathToGeneratedDirectory), }); - - mockFs.restore(); }); }); describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); - - mockFs({ - [entityRootDir]: { - 'index.html': 'file-content', - }, - }); - + const publisher = createPublisherFromConfig(); + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + }); + + it('should return true if docs has been generated even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); - mockFs.restore(); }); it('should return false if docs has not been generated', async () => { - const entity = createMockEntity(); - - expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false); + const publisher = createPublisherFromConfig(); + expect( + await publisher.hasDocsBeenGenerated({ + kind: 'entity', + metadata: { + namespace: 'invalid', + name: 'triplet', + }, + } as Entity), + ).toBe(false); }); }); describe('fetchTechDocsMetadata', () => { - beforeEach(() => { - mockFs.restore(); + it('should return tech docs metadata', async () => { + const publisher = createPublisherFromConfig(); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); }); - it('should return tech docs metadata', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), + it('should return tech docs metadata even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, }); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': - '{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}', - }, - }); - - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; - expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': - "{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}", - }, - }); - - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; + const publisher = createPublisherFromConfig(); expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); + await publisher.fetchTechDocsMetadata({ + namespace: 'storage', + kind: 'single', + name: 'quote', + }), + ).toStrictEqual(techdocsMetadata); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); + const publisher = createPublisherFromConfig(); - const fails = publisher.fetchTechDocsMetadata(entityNameMock); + const invalidEntityName = { + namespace: 'invalid', + kind: 'triplet', + name: 'path', + }; + + const techDocsMetadaFilePath = path.join( + rootDir, + ...Object.values(invalidEntityName), + 'techdocs_metadata.json', + ); + + const fails = publisher.fetchTechDocsMetadata(invalidEntityName); await expect(fails).rejects.toMatchObject({ - message: `The file ${path.join( - entityRootDir, - 'techdocs_metadata.json', - )} does not exist !`, + message: `The file ${techDocsMetadaFilePath} does not exist !`, }); }); }); describe('docsRouter', () => { + const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + let app: express.Express; - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); beforeEach(() => { + const publisher = createPublisherFromConfig(); app = express().use(publisher.docsRouter()); - - mockFs.restore(); - mockFs({ - [entityRootDir]: { - html: { - 'unsafe.html': '', - }, - img: { - 'with spaces.png': 'found it', - 'unsafe.svg': '', - }, - 'some folder': { - 'also with spaces.js': 'found it too', - }, - }, - }); - }); - - afterEach(() => { - mockFs.restore(); }); it('should pass expected object path to bucket', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/with%20spaces.png`, + `/${entityTripletPath}/img/with%20spaces.png`, ); expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( 'found it', ); const jsResponse = await request(app).get( - `/${namespace}/${kind}/${name}/some%20folder/also%20with%20spaces.js`, + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, + ); + expect(jsResponse.text).toEqual('found it too'); + }); + + it('should pass expected object path to bucket even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + app = express().use(publisher.docsRouter()); + // Ensures leading slash is trimmed and encoded path is decoded. + const pngResponse = await request(app).get( + `/${entityTripletPath}/img/with%20spaces.png`, + ); + expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( + 'found it', + ); + const jsResponse = await request(app).get( + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, ); expect(jsResponse.text).toEqual('found it too'); }); it('should pass text/plain content-type for html', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - const htmlResponse = await request(app).get( - `/${namespace}/${kind}/${name}/html/unsafe.html`, + `/${entityTripletPath}/html/unsafe.html`, ); expect(htmlResponse.text).toEqual(''); expect(htmlResponse.header).toMatchObject({ @@ -352,7 +337,7 @@ describe('GoogleGCSPublish', () => { }); const svgResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/unsafe.svg`, + `/${entityTripletPath}/img/unsafe.svg`, ); expect(svgResponse.text).toEqual(''); expect(svgResponse.header).toMatchObject({ diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 3aa7b09b74..9db9b6fb49 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -99,8 +99,8 @@ export class GoogleGCSPublish implements PublisherBase { ) { this.storageClient = storageClient; this.bucketName = bucketName; - this.logger = logger; this.legacyPathCasing = legacyPathCasing; + this.logger = logger; } /** @@ -160,11 +160,10 @@ export class GoogleGCSPublish implements PublisherBase { entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE }/${entity.kind}/${entity.metadata.name}`; + const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`; const destination = this.legacyPathCasing - ? lowerCaseEntityTripletInStoragePath( - `${entityRootDir}/${relativeFilePathPosix}`, - ) - : `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path + ? relativeFilePathTriplet + : lowerCaseEntityTripletInStoragePath(relativeFilePathTriplet); // GCS Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index 7f2fdf8b26..537daa815f 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -246,5 +246,10 @@ export interface Config { * @deprecated */ storageUrl?: string; + + /** + * (Optional) + */ + legacyUseCaseSensitiveTripletPaths?: boolean; }; } From 759f15a14a7db08f048679cd42d7a1e2b502c7d1 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Sat, 7 Aug 2021 21:53:09 +0200 Subject: [PATCH 06/15] test(techdocs-common): refactor storage files mock Signed-off-by: Camila Belo --- .../__mocks__/@azure/storage-blob.ts | 53 ++------ .../__mocks__/@google-cloud/storage.ts | 58 +++------ packages/techdocs-common/__mocks__/aws-sdk.ts | 60 +++------ packages/techdocs-common/src/setupTests.ts | 61 +++++++++ .../src/stages/publish/awsS3.test.ts | 84 +++++------- .../stages/publish/azureBlobStorage.test.ts | 122 +++++++----------- .../src/stages/publish/googleStorage.test.ts | 86 ++++++------ 7 files changed, 229 insertions(+), 295 deletions(-) create mode 100644 packages/techdocs-common/src/setupTests.ts diff --git a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts index 73c5873832..80c60acce7 100644 --- a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts +++ b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts @@ -18,29 +18,8 @@ import type { ContainerGetPropertiesResponse, } from '@azure/storage-blob'; import { EventEmitter } from 'events'; -import fs from 'fs-extra'; -import os from 'os'; -import path from 'path'; -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; - -/** - * @param sourceFile Relative path to entity root dir. Contains either / or \ as file separator - * depending upon the OS. - */ -const checkFileExists = async (sourceFile: string): Promise => { - // sourceFile will always have / as file separator irrespective of OS since Azure expects /. - // Normalize sourceFile to OS specific path before checking if file exists. - const relativeFilePath = sourceFile.split(path.posix.sep).join(path.sep); - const filePath = path.join(rootDir, sourceFile); - - try { - await fs.access(filePath, fs.constants.F_OK); - return true; - } catch (err) { - return false; - } -}; +const storage = new (global as any).StorageFilesMock(); export class BlockBlobClient { private readonly blobName; @@ -50,9 +29,7 @@ export class BlockBlobClient { } uploadFile(source: string): Promise { - if (!fs.existsSync(source)) { - return Promise.reject(new Error(`The file ${source} does not exist`)); - } + storage.writeFile(this.blobName, source); return Promise.resolve({ _response: { request: { @@ -65,20 +42,19 @@ export class BlockBlobClient { } exists() { - return checkFileExists(this.blobName); + return storage.fileExists(this.blobName); } download() { - const filePath = path.join(rootDir, this.blobName); const emitter = new EventEmitter(); setTimeout(() => { - if (fs.existsSync(filePath)) { - emitter.emit('data', fs.readFileSync(filePath)); + if (storage.fileExists(this.blobName)) { + emitter.emit('data', storage.readFile(this.blobName)); emitter.emit('end'); } else { emitter.emit( 'error', - new Error(`The file ${filePath} does not exist !`), + new Error(`The file ${this.blobName} does not exist!`), ); } }, 0); @@ -89,7 +65,7 @@ export class BlockBlobClient { } class BlockBlobClientFailUpload extends BlockBlobClient { - uploadFile(source: string): Promise { + uploadFile(): Promise { return Promise.resolve({ _response: { request: { @@ -103,12 +79,6 @@ class BlockBlobClientFailUpload extends BlockBlobClient { } export class ContainerClient { - private readonly containerName; - - constructor(containerName: string) { - this.containerName = containerName; - } - getProperties(): Promise { return Promise.resolve({ _response: { @@ -153,18 +123,19 @@ export class BlobServiceClient { private readonly credential; constructor(url: string, credential?: StorageSharedKeyCredential) { + storage.emptyFiles(); this.url = url; this.credential = credential; } getContainerClient(containerName: string) { if (containerName === 'bad_container') { - return new ContainerClientFailGetProperties(containerName); + return new ContainerClientFailGetProperties(); } - if (this.credential.accountName === 'failupload') { - return new ContainerClientFailUpload(containerName); + if (this.credential.accountName === 'bad_account_credentials') { + return new ContainerClientFailUpload(); } - return new ContainerClient(containerName); + return new ContainerClient(); } } diff --git a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts index 18fe62af14..efc2454d08 100644 --- a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts +++ b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts @@ -14,42 +14,19 @@ * limitations under the License. */ import { Readable } from 'stream'; -import fs from 'fs-extra'; -import os from 'os'; -import path from 'path'; -type storageOptions = { - keyFilename?: string; -}; - -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; -/** - * @param sourceFile Absolute path. Contains either / or \ as file separator depending upon the OS. - */ -const checkFileExists = async (sourceFile: string): Promise => { - // sourceFile will always have / as file separator irrespective of OS since GCS expects /. - // Normalize sourceFile to OS specific path before checking if file exists. - const filePath = sourceFile.split(path.posix.sep).join(path.sep); - - try { - await fs.access(filePath, fs.constants.F_OK); - return true; - } catch (err) { - return false; - } -}; +const storage = new (global as any).StorageFilesMock(); class GCSFile { - private readonly localFilePath: string; + private readonly path: string; - constructor(private readonly destinationFilePath: string) { - this.destinationFilePath = destinationFilePath; - this.localFilePath = path.join(rootDir, this.destinationFilePath); + constructor(path: string) { + this.path = path; } exists() { return new Promise(async (resolve, reject) => { - if (await checkFileExists(this.localFilePath)) { + if (storage.fileExists(this.path)) { resolve([true]); } else { reject(); @@ -62,19 +39,20 @@ class GCSFile { readable._read = () => {}; process.nextTick(() => { - if (fs.existsSync(this.localFilePath)) { + if (storage.fileExists(this.path)) { if (readable.eventNames().includes('pipe')) { readable.emit('pipe'); } - readable.emit('data', fs.readFileSync(this.localFilePath)); + readable.emit('data', storage.readFile(this.path)); readable.emit('end'); } else { readable.emit( 'error', - new Error(`The file ${this.localFilePath} does not exist !`), + new Error(`The file ${this.path} does not exist!`), ); } }); + return readable; } } @@ -87,20 +65,16 @@ class Bucket { } async getMetadata() { - if (this.bucketName === 'errorBucket') { + if (this.bucketName === 'bad_bucket_name') { throw Error('Bucket does not exist'); } - return ''; } upload(source: string, { destination }) { - return new Promise(async (resolve, reject) => { - if (await checkFileExists(source)) { - resolve({ source, destination }); - } else { - reject(`Source file ${source} does not exist.`); - } + return new Promise(async resolve => { + storage.writeFile(destination, source); + resolve(null); }); } @@ -110,10 +84,8 @@ class Bucket { } export class Storage { - private readonly keyFilename; - - constructor(options: storageOptions) { - this.keyFilename = options.keyFilename; + constructor() { + storage.emptyFiles(); } bucket(bucketName) { diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts index 763fad0bda..36c864bfe7 100644 --- a/packages/techdocs-common/__mocks__/aws-sdk.ts +++ b/packages/techdocs-common/__mocks__/aws-sdk.ts @@ -13,39 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import type { S3 as S3Types } from 'aws-sdk'; import { EventEmitter } from 'events'; -import fs from 'fs-extra'; -import os from 'os'; -import path from 'path'; +import { ReadStream } from 'fs'; export { Credentials } from 'aws-sdk'; -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; - -/** - * @param Key Relative path to entity root dir. Contains either / or \ as file separator - * depending upon the OS. - */ -const checkFileExists = async (Key: string): Promise => { - // Key will always have / as file separator irrespective of OS since S3 expects /. - // Normalize Key to OS specific path before checking if file exists. - const relativeFilePath = Key.split(path.posix.sep).join(path.sep); - const filePath = path.join(rootDir, Key); - - try { - await fs.access(filePath, fs.constants.F_OK); - return true; - } catch (err) { - return false; - } -}; +const storage = new (global as any).StorageFilesMock(); export class S3 { + constructor() { + storage.emptyFiles(); + } + headObject({ Key }: { Key: string }) { return { promise: async () => { - if (!(await checkFileExists(Key))) { + if (!storage.fileExists(Key)) { throw new Error('File does not exist'); } }, @@ -53,20 +36,16 @@ export class S3 { } getObject({ Key }: { Key: string }) { - const filePath = path.join(rootDir, Key); return { - promise: async () => await checkFileExists(filePath), + promise: async () => storage.fileExists(Key), createReadStream: () => { const emitter = new EventEmitter(); process.nextTick(() => { - if (fs.existsSync(filePath)) { - emitter.emit('data', Buffer.from(fs.readFileSync(filePath))); + if (storage.fileExists(Key)) { + emitter.emit('data', Buffer.from(storage.readFile(Key))); emitter.emit('end'); } else { - emitter.emit( - 'error', - new Error(`The file ${filePath} does not exist !`), - ); + emitter.emit('error', new Error(`The file ${Key} does not exist!`)); } }); return emitter; @@ -85,15 +64,18 @@ export class S3 { }; } - upload({ Key }: { Key: string }) { + upload({ Key, Body }: { Key: string; Body: ReadStream }) { return { promise: () => - new Promise(async (resolve, reject) => { - if (!(await checkFileExists(Key))) { - reject(`The file ${Key} does not exist`); - } else { - resolve(''); - } + new Promise(async resolve => { + const chunks = []; + Body.on('data', chunk => { + chunks.push(chunk); + }); + Body.once('end', () => { + storage.writeFile(Key, Buffer.concat(chunks)); + resolve(null); + }); }), }; } diff --git a/packages/techdocs-common/src/setupTests.ts b/packages/techdocs-common/src/setupTests.ts new file mode 100644 index 0000000000..996a8d35dd --- /dev/null +++ b/packages/techdocs-common/src/setupTests.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import path from 'path'; +import fs from 'fs-extra'; + +const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; + +const encoding = 'utf8'; + +export class StorageFilesMock { + private files: Record; + + constructor() { + this.files = {}; + } + + public emptyFiles() { + this.files = {}; + } + + public fileExists(targetPath: string): boolean { + const filePath = path.join(rootDir, targetPath); + const posixPath = filePath.split(path.posix.sep).join(path.sep); + return this.files[posixPath] !== undefined; + } + + public readFile(targetPath: string): Buffer { + const filePath = path.join(rootDir, targetPath); + return Buffer.from(this.files[filePath] ?? '', encoding); + } + + public writeFile(targetPath: string, sourcePath: string): void; + public writeFile(targetPath: string, sourceBuffer: Buffer): void; + public writeFile(targetPath: string, source: string | Buffer): void { + const filePath = path.join(rootDir, targetPath); + if (typeof source === 'string') { + this.files[filePath] = fs.readFileSync(source).toString(encoding); + } else { + this.files[filePath] = source.toString(encoding); + } + } +} + +const _global = global as any; +_global.rootDir = rootDir; +_global.StorageFilesMock = StorageFilesMock; diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index 715786e3f9..d77d643291 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -20,13 +20,13 @@ import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import mockFs from 'mock-fs'; -import os from 'os'; import path from 'path'; +import fs from 'fs-extra'; import { AwsS3Publish } from './awsS3'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock aws-sdk client library -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +const rootDir = (global as any).rootDir; const getEntityRootDir = (entity: Entity) => { const { @@ -46,7 +46,7 @@ const createPublisherFromConfig = ({ bucketName?: string; legacyUseCaseSensitiveTripletPaths?: boolean; } = {}) => { - const mockConfig = new ConfigReader({ + const config = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', publisher: { @@ -63,7 +63,7 @@ const createPublisherFromConfig = ({ }, }); - return AwsS3Publish.fromConfig(mockConfig, logger); + return AwsS3Publish.fromConfig(config, logger); }; describe('AwsS3Publish', () => { @@ -72,7 +72,6 @@ describe('AwsS3Publish', () => { kind: 'Component', metadata: { name: 'backstage', - namespace: 'default', annotations: {}, }, @@ -90,18 +89,7 @@ describe('AwsS3Publish', () => { etag: 'etag', }; - const localRootDir = getEntityRootDir(entity); - const storageRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); - const storageSingleQuoteDir = getEntityRootDir({ - metadata: { - namespace: 'storage', - name: 'quote', - }, - kind: 'single', - } as Entity); + const directory = getEntityRootDir(entity); const files = { 'index.html': '', @@ -110,9 +98,6 @@ describe('AwsS3Publish', () => { assets: { 'main.css': '', }, - attachments: { - 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), - }, html: { 'unsafe.html': '', }, @@ -127,14 +112,7 @@ describe('AwsS3Publish', () => { beforeAll(() => { mockFs({ - [localRootDir]: files, - [storageRootDir]: files, - [storageSingleQuoteDir]: { - 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( - /"/g, - "'", - ), - }, + [directory]: files, }); }); @@ -163,24 +141,14 @@ describe('AwsS3Publish', () => { describe('publish', () => { it('should publish a directory', async () => { const publisher = createPublisherFromConfig(); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should publish a directory as well when legacy casing is used', async () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should fail to publish a directory', async () => { @@ -214,6 +182,7 @@ describe('AwsS3Publish', () => { describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); @@ -221,6 +190,7 @@ describe('AwsS3Publish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); @@ -241,6 +211,7 @@ describe('AwsS3Publish', () => { describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); @@ -250,20 +221,32 @@ describe('AwsS3Publish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { + const techdocsMetadataPath = path.join( + directory, + 'techdocs_metadata.json', + ); + const techdocsMetadataContent = files['techdocs_metadata.json']; + + fs.writeFileSync( + techdocsMetadataPath, + techdocsMetadataContent.replace(/"/g, "'"), + ); + const publisher = createPublisherFromConfig(); - expect( - await publisher.fetchTechDocsMetadata({ - namespace: 'storage', - kind: 'single', - name: 'quote', - }), - ).toStrictEqual(techdocsMetadata); + await publisher.publish({ entity, directory }); + + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + + fs.writeFileSync(techdocsMetadataPath, techdocsMetadataContent); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { @@ -276,7 +259,6 @@ describe('AwsS3Publish', () => { }; const techDocsMetadaFilePath = path.join( - rootDir, ...Object.values(invalidEntityName), 'techdocs_metadata.json', ); @@ -284,7 +266,7 @@ describe('AwsS3Publish', () => { const fails = publisher.fetchTechDocsMetadata(invalidEntityName); await expect(fails).rejects.toMatchObject({ - message: `TechDocs metadata fetch failed, The file ${techDocsMetadaFilePath} does not exist !`, + message: `TechDocs metadata fetch failed, The file ${techDocsMetadaFilePath} does not exist!`, }); }); }); @@ -294,8 +276,9 @@ describe('AwsS3Publish', () => { let app: express.Express; - beforeEach(() => { + beforeEach(async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); }); @@ -317,6 +300,7 @@ describe('AwsS3Publish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 78cc7c0c79..dd658440d6 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -20,13 +20,13 @@ import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import mockFs from 'mock-fs'; -import os from 'os'; import path from 'path'; +import fs from 'fs-extra'; import { AzureBlobStoragePublish } from './azureBlobStorage'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Azure client library -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +const rootDir = (global as any).rootDir; const getEntityRootDir = (entity: Entity) => { const { @@ -38,7 +38,6 @@ const getEntityRootDir = (entity: Entity) => { }; const logger = getVoidLogger(); -jest.spyOn(logger, 'info').mockReturnValue(logger); jest.spyOn(logger, 'error').mockReturnValue(logger); const createPublisherFromConfig = ({ @@ -50,7 +49,7 @@ const createPublisherFromConfig = ({ containerName?: string; legacyUseCaseSensitiveTripletPaths?: boolean; } = {}) => { - const mockConfig = new ConfigReader({ + const config = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', publisher: { @@ -67,10 +66,10 @@ const createPublisherFromConfig = ({ }, }); - return AzureBlobStoragePublish.fromConfig(mockConfig, logger); + return AzureBlobStoragePublish.fromConfig(config, logger); }; -describe('publishing with valid credentials', () => { +describe('AzureBlobStoragePublish', () => { const entity = { apiVersion: '1', kind: 'Component', @@ -93,21 +92,9 @@ describe('publishing with valid credentials', () => { etag: 'etag', }; - const localRootDir = getEntityRootDir(entity); - const storageRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); - const storageSingleQuoteDir = getEntityRootDir({ - metadata: { - namespace: 'storage', - name: 'quote', - }, - kind: 'single', - } as Entity); + const directory = getEntityRootDir(entity); beforeEach(() => { - (logger.info as jest.Mock).mockClear(); (logger.error as jest.Mock).mockClear(); }); @@ -118,9 +105,6 @@ describe('publishing with valid credentials', () => { assets: { 'main.css': '', }, - attachments: { - 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), - }, html: { 'unsafe.html': '', }, @@ -135,14 +119,7 @@ describe('publishing with valid credentials', () => { beforeAll(async () => { mockFs({ - [localRootDir]: files, - [storageRootDir]: files, - [storageSingleQuoteDir]: { - 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( - /"/g, - "'", - ), - }, + [directory]: files, }); }); @@ -178,24 +155,14 @@ describe('publishing with valid credentials', () => { describe('publish', () => { it('should publish a directory', async () => { const publisher = createPublisherFromConfig(); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should publish a directory as well when legacy casing is used', async () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should fail to publish a directory', async () => { @@ -229,15 +196,12 @@ describe('publishing with valid credentials', () => { it('reports an error when bad account credentials', async () => { const publisher = createPublisherFromConfig({ - accountName: 'failupload', + accountName: 'bad_account_credentials', }); let error; try { - await publisher.publish({ - entity, - directory: localRootDir, - }); + await publisher.publish({ entity, directory }); } catch (e) { error = e; } @@ -249,7 +213,7 @@ describe('publishing with valid credentials', () => { expect(logger.error).toHaveBeenCalledWith( expect.stringContaining( `Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join( - localRootDir, + directory, '404.html', )} with status code 500`, ), @@ -258,15 +222,17 @@ describe('publishing with valid credentials', () => { }); describe('hasDocsBeenGenerated', () => { - it('should return true if docs has been generated', async () => { + it('should check expected file', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); - it('should return true if docs has been generated even if the legacy case is enabled', async () => { + it('should check expected file when legacy case flag is passed', async () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); @@ -287,6 +253,7 @@ describe('publishing with valid credentials', () => { describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); @@ -296,46 +263,53 @@ describe('publishing with valid credentials', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { + const techdocsMetadataPath = path.join( + directory, + 'techdocs_metadata.json', + ); + const techdocsMetadataContent = files['techdocs_metadata.json']; + + fs.writeFileSync( + techdocsMetadataPath, + techdocsMetadataContent.replace(/"/g, "'"), + ); + const publisher = createPublisherFromConfig(); - expect( - await publisher.fetchTechDocsMetadata({ - namespace: 'storage', - kind: 'single', - name: 'quote', - }), - ).toStrictEqual(techdocsMetadata); + await publisher.publish({ entity, directory }); + + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + + fs.writeFileSync(techdocsMetadataPath, techdocsMetadataContent); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { const publisher = createPublisherFromConfig(); + const invalidEntityName = { namespace: 'invalid', kind: 'triplet', name: 'path', }; - let error; - try { - await publisher.fetchTechDocsMetadata(invalidEntityName); - } catch (e) { - error = e; - } - - expect(error).toEqual( - new Error( - `TechDocs metadata fetch failed, The file ${path.join( - rootDir, - ...Object.values(invalidEntityName), - 'techdocs_metadata.json', - )} does not exist !`, - ), + const techDocsMetadaFilePath = path.join( + ...Object.values(invalidEntityName), + 'techdocs_metadata.json', ); + + const fails = publisher.fetchTechDocsMetadata(invalidEntityName); + + await expect(fails).rejects.toMatchObject({ + message: `TechDocs metadata fetch failed, The file ${techDocsMetadaFilePath} does not exist!`, + }); }); }); @@ -344,8 +318,9 @@ describe('publishing with valid credentials', () => { let app: express.Express; - beforeEach(() => { + beforeEach(async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); }); @@ -367,6 +342,7 @@ describe('publishing with valid credentials', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index ce08e2f2ce..57fde767ba 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -20,13 +20,13 @@ import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import mockFs from 'mock-fs'; -import os from 'os'; import path from 'path'; +import fs from 'fs-extra'; import { GoogleGCSPublish } from './googleStorage'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Google Cloud Storage client library -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +const rootDir = (global as any).rootDir; const getEntityRootDir = (entity: Entity) => { const { @@ -38,7 +38,6 @@ const getEntityRootDir = (entity: Entity) => { }; const logger = getVoidLogger(); -jest.spyOn(logger, 'info').mockReturnValue(logger); const createPublisherFromConfig = ({ bucketName = 'bucketName', @@ -47,7 +46,7 @@ const createPublisherFromConfig = ({ bucketName?: string; legacyUseCaseSensitiveTripletPaths?: boolean; } = {}) => { - const mockConfig = new ConfigReader({ + const config = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', publisher: { @@ -61,7 +60,7 @@ const createPublisherFromConfig = ({ }, }); - return GoogleGCSPublish.fromConfig(mockConfig, logger); + return GoogleGCSPublish.fromConfig(config, logger); }; describe('GoogleGCSPublish', () => { @@ -87,18 +86,7 @@ describe('GoogleGCSPublish', () => { etag: 'etag', }; - const localRootDir = getEntityRootDir(entity); - const storageRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); - const storageSingleQuoteDir = getEntityRootDir({ - metadata: { - namespace: 'storage', - name: 'quote', - }, - kind: 'single', - } as Entity); + const directory = getEntityRootDir(entity); const files = { 'index.html': '', @@ -121,14 +109,7 @@ describe('GoogleGCSPublish', () => { beforeAll(() => { mockFs({ - [localRootDir]: files, - [storageRootDir]: files, - [storageSingleQuoteDir]: { - 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( - /"/g, - "'", - ), - }, + [directory]: files, }); }); @@ -146,7 +127,7 @@ describe('GoogleGCSPublish', () => { it('should reject incorrect config', async () => { const publisher = createPublisherFromConfig({ - bucketName: 'errorBucket', + bucketName: 'bad_bucket_name', }); expect(await publisher.getReadiness()).toEqual({ isAvailable: false, @@ -157,24 +138,14 @@ describe('GoogleGCSPublish', () => { describe('publish', () => { it('should publish a directory', async () => { const publisher = createPublisherFromConfig(); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should publish a directory as well when legacy casing is used', async () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should fail to publish a directory', async () => { @@ -210,6 +181,7 @@ describe('GoogleGCSPublish', () => { describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); @@ -217,6 +189,7 @@ describe('GoogleGCSPublish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); @@ -237,6 +210,7 @@ describe('GoogleGCSPublish', () => { describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); @@ -246,20 +220,32 @@ describe('GoogleGCSPublish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { + const techdocsMetadataPath = path.join( + directory, + 'techdocs_metadata.json', + ); + const techdocsMetadataContent = files['techdocs_metadata.json']; + + fs.writeFileSync( + techdocsMetadataPath, + techdocsMetadataContent.replace(/"/g, "'"), + ); + const publisher = createPublisherFromConfig(); - expect( - await publisher.fetchTechDocsMetadata({ - namespace: 'storage', - kind: 'single', - name: 'quote', - }), - ).toStrictEqual(techdocsMetadata); + await publisher.publish({ entity, directory }); + + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + + fs.writeFileSync(techdocsMetadataPath, techdocsMetadataContent); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { @@ -272,7 +258,6 @@ describe('GoogleGCSPublish', () => { }; const techDocsMetadaFilePath = path.join( - rootDir, ...Object.values(invalidEntityName), 'techdocs_metadata.json', ); @@ -280,7 +265,7 @@ describe('GoogleGCSPublish', () => { const fails = publisher.fetchTechDocsMetadata(invalidEntityName); await expect(fails).rejects.toMatchObject({ - message: `The file ${techDocsMetadaFilePath} does not exist !`, + message: `The file ${techDocsMetadaFilePath} does not exist!`, }); }); }); @@ -288,10 +273,11 @@ describe('GoogleGCSPublish', () => { describe('docsRouter', () => { const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - let app: express.Express; + let app: Express.Application; - beforeEach(() => { + beforeEach(async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); }); @@ -313,7 +299,9 @@ describe('GoogleGCSPublish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); + // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( `/${entityTripletPath}/img/with%20spaces.png`, From 9181e8c6e029af12619eeeb97b6451f87730ef8d Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Mon, 9 Aug 2021 09:27:34 +0200 Subject: [PATCH 07/15] tests(techdocs-common): add globals and types for mocked files Signed-off-by: Camila Belo --- .../__mocks__/@azure/storage-blob.ts | 4 ++-- .../__mocks__/@google-cloud/storage.ts | 2 +- packages/techdocs-common/__mocks__/aws-sdk.ts | 2 +- .../techdocs-common/__mocks__/globals.d.ts | 22 +++++++++++++++++ packages/techdocs-common/__mocks__/types.d.ts | 24 +++++++++++++++++++ packages/techdocs-common/src/globals.d.ts | 22 +++++++++++++++++ packages/techdocs-common/src/setupTests.ts | 11 ++++----- .../src/stages/publish/awsS3.test.ts | 2 +- .../stages/publish/azureBlobStorage.test.ts | 2 +- .../src/stages/publish/googleStorage.test.ts | 2 +- packages/techdocs-common/src/types.d.ts | 24 +++++++++++++++++++ 11 files changed, 104 insertions(+), 13 deletions(-) create mode 100644 packages/techdocs-common/__mocks__/globals.d.ts create mode 100644 packages/techdocs-common/__mocks__/types.d.ts create mode 100644 packages/techdocs-common/src/globals.d.ts create mode 100644 packages/techdocs-common/src/types.d.ts diff --git a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts index 80c60acce7..b901f5f5a7 100644 --- a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts +++ b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import type { +import { BlobUploadCommonResponse, ContainerGetPropertiesResponse, } from '@azure/storage-blob'; import { EventEmitter } from 'events'; -const storage = new (global as any).StorageFilesMock(); +const storage = global.storageFilesMock; export class BlockBlobClient { private readonly blobName; diff --git a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts index efc2454d08..856f09c4df 100644 --- a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts +++ b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts @@ -15,7 +15,7 @@ */ import { Readable } from 'stream'; -const storage = new (global as any).StorageFilesMock(); +const storage = global.storageFilesMock; class GCSFile { private readonly path: string; diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts index 36c864bfe7..6251fe8357 100644 --- a/packages/techdocs-common/__mocks__/aws-sdk.ts +++ b/packages/techdocs-common/__mocks__/aws-sdk.ts @@ -18,7 +18,7 @@ import { ReadStream } from 'fs'; export { Credentials } from 'aws-sdk'; -const storage = new (global as any).StorageFilesMock(); +const storage = global.storageFilesMock; export class S3 { constructor() { diff --git a/packages/techdocs-common/__mocks__/globals.d.ts b/packages/techdocs-common/__mocks__/globals.d.ts new file mode 100644 index 0000000000..ad293d2ebe --- /dev/null +++ b/packages/techdocs-common/__mocks__/globals.d.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +declare module NodeJS { + interface Global { + rootDir: string; + storageFilesMock: IStorageFilesMock; + } +} diff --git a/packages/techdocs-common/__mocks__/types.d.ts b/packages/techdocs-common/__mocks__/types.d.ts new file mode 100644 index 0000000000..e085b95851 --- /dev/null +++ b/packages/techdocs-common/__mocks__/types.d.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +interface IStorageFilesMock { + emptyFiles(): void; + fileExists(targetPath: string): boolean; + readFile(targetPath: string): Buffer; + writeFile(targetPath: string, sourcePath: string): void; + writeFile(targetPath: string, sourceBuffer: Buffer): void; + writeFile(targetPath: string, source: string | Buffer): void; +} diff --git a/packages/techdocs-common/src/globals.d.ts b/packages/techdocs-common/src/globals.d.ts new file mode 100644 index 0000000000..ad293d2ebe --- /dev/null +++ b/packages/techdocs-common/src/globals.d.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +declare module NodeJS { + interface Global { + rootDir: string; + storageFilesMock: IStorageFilesMock; + } +} diff --git a/packages/techdocs-common/src/setupTests.ts b/packages/techdocs-common/src/setupTests.ts index 996a8d35dd..9a13e57633 100644 --- a/packages/techdocs-common/src/setupTests.ts +++ b/packages/techdocs-common/src/setupTests.ts @@ -18,18 +18,18 @@ import os from 'os'; import path from 'path'; import fs from 'fs-extra'; -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +const rootDir: string = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; const encoding = 'utf8'; -export class StorageFilesMock { +class StorageFilesMock implements IStorageFilesMock { private files: Record; constructor() { this.files = {}; } - public emptyFiles() { + public emptyFiles(): void { this.files = {}; } @@ -56,6 +56,5 @@ export class StorageFilesMock { } } -const _global = global as any; -_global.rootDir = rootDir; -_global.StorageFilesMock = StorageFilesMock; +global.rootDir = rootDir; +global.storageFilesMock = new StorageFilesMock(); diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index d77d643291..62ec80d634 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -26,7 +26,7 @@ import { AwsS3Publish } from './awsS3'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock aws-sdk client library -const rootDir = (global as any).rootDir; +const rootDir = global.rootDir; const getEntityRootDir = (entity: Entity) => { const { diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index dd658440d6..c32ec310d0 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -26,7 +26,7 @@ import { AzureBlobStoragePublish } from './azureBlobStorage'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Azure client library -const rootDir = (global as any).rootDir; +const rootDir = global.rootDir; const getEntityRootDir = (entity: Entity) => { const { diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index 57fde767ba..fe93ef909b 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -26,7 +26,7 @@ import { GoogleGCSPublish } from './googleStorage'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Google Cloud Storage client library -const rootDir = (global as any).rootDir; +const rootDir = global.rootDir; const getEntityRootDir = (entity: Entity) => { const { diff --git a/packages/techdocs-common/src/types.d.ts b/packages/techdocs-common/src/types.d.ts new file mode 100644 index 0000000000..e085b95851 --- /dev/null +++ b/packages/techdocs-common/src/types.d.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +interface IStorageFilesMock { + emptyFiles(): void; + fileExists(targetPath: string): boolean; + readFile(targetPath: string): Buffer; + writeFile(targetPath: string, sourcePath: string): void; + writeFile(targetPath: string, sourceBuffer: Buffer): void; + writeFile(targetPath: string, source: string | Buffer): void; +} From c772d9a84eab20ebb6553b15a07fb9af4f7f3bff Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Mon, 9 Aug 2021 10:23:18 +0200 Subject: [PATCH 08/15] docs(techdocs-common): Add changeset file for next release Signed-off-by: Camila Belo --- .changeset/techdocs-all-done.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/techdocs-all-done.md diff --git a/.changeset/techdocs-all-done.md b/.changeset/techdocs-all-done.md new file mode 100644 index 0000000000..2696b40df7 --- /dev/null +++ b/.changeset/techdocs-all-done.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +This is the third and final step of migrating cloud storage entities to lowercase ([see](https://github.com/backstage/backstage/issues/4367#issuecomment-876461002)). + +Updates GCS, AWS S3 and Azure Blob Storage Provider to lowercase entity triplet paths before reading and writing to it cloud storage. From 2f4f15846d6fe91a2a1dcbaaad0907929d074e36 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Mon, 9 Aug 2021 11:38:33 +0200 Subject: [PATCH 09/15] docs(techdocs-common): add case sentitive migration steps Co-authored-by: Eric Peterson Signed-off-by: Camila Belo --- .changeset/techdocs-all-done.md | 18 +++++-- docs/features/techdocs/configuration.md | 8 +++ docs/features/techdocs/how-to-guides.md | 70 +++++++++++++++++++++++++ plugins/techdocs-backend/config.d.ts | 7 ++- 4 files changed, 99 insertions(+), 4 deletions(-) diff --git a/.changeset/techdocs-all-done.md b/.changeset/techdocs-all-done.md index 2696b40df7..6d2146e2a5 100644 --- a/.changeset/techdocs-all-done.md +++ b/.changeset/techdocs-all-done.md @@ -1,7 +1,19 @@ --- -'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs': minor +'@backstage/plugin-techdocs-backend': minor +'@backstage/techdocs-common': minor --- +TechDocs sites can now be accessed using paths containing entity triplets of +any case (e.g. `/docs/namespace/KIND/name` or `/docs/namespace/kind/name`). + +If you do not use an external storage provider for serving TechDocs, this is a +transparent change and no action is required from you. + +If you _do_ use an external storage provider for serving TechDocs (e.g. Google +Cloud Storage, AWS S3, or Azure Blob Storage), you must run a migration command +against your storage provider before updating to this version. + +[A migration guide is available here](#todo). + This is the third and final step of migrating cloud storage entities to lowercase ([see](https://github.com/backstage/backstage/issues/4367#issuecomment-876461002)). - -Updates GCS, AWS S3 and Azure Blob Storage Provider to lowercase entity triplet paths before reading and writing to it cloud storage. diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 4bb6857106..46cbc2a8a2 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -113,6 +113,14 @@ techdocs: # https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY} + # (Optional and not recommended) Prior to version [0.x.y] of TechDocs, docs + # sites could only be accessed over paths with case-sensitive entity triplets + # e.g. (namespace/Kind/name). If you are upgrading from an older version of + # TechDocs and are unable to perform the necessary migration of files in your + # external storage, you can set this value to `true` to temporarily revert to + # the old, case-sensitive entity triplet behavior. + legacyUseCaseSensitiveTripletPaths: false + # (Optional and Legacy) TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc. # You don't have to specify this anymore. diff --git a/docs/features/techdocs/how-to-guides.md b/docs/features/techdocs/how-to-guides.md index e1533498d7..4962b9b107 100644 --- a/docs/features/techdocs/how-to-guides.md +++ b/docs/features/techdocs/how-to-guides.md @@ -184,3 +184,73 @@ annotation ) as an 'entities' attribute. For a reference to the React structure of the default home page, please refer to `plugins/techdocs/src/home/components/TechDocsCustomHome.tsx`. + +## How to migrate from TechDocs Alpha to Beta + +> This guide only applies to the "recommended" TechDocs deployment method (where +> an external storage provider and external CI/CD is used). If you use the +> "basic" or "out-of-the-box" setup, you can stop here! No action needed. + +The beta version of TechDocs (v0.x.y) made a breaking change to the way TechDocs +content was accessed and stored, allowing pages to be accessed with +case-insensitive entity triplet paths (e.g. `/docs/namespace/kind/name` whereas +in prior versions, they could only be accessed at `/docs/namespace/Kind/name`). +In order to enable this change, documentation has be stored in an external +storage provider using an object key whose entity triplet is lower-cased. + +New installations of TechDocs since the beta version will work fine with no +action, but for those who were running TechDocs prior to this version, a +migration will need to be performed so that all existing content in your storage +bucket matches this lower-case entity triplet expectation. + +1. **Ensure you have the right permissions on your storage provider**: In order + to migrate files in your storage provider, the `techdocs-cli` needs to be + able to read/copy/rename/move/delete files. The exact instructions vary by + storage provider, but check the [using cloud storage][using-cloud-storage] + page for details. + +2. **Run a non-destructive migration of files**: Ensure you have the latest + version of `techdocs-cli` installed. Then run the following command, using + the details relevant for your provider / configuration. This will copy all + files from, e.g. `namespace/Kind/name/index.html` to + `namespace/kind/name/index.html`, without removing the original files. + +```sh +techdocs-cli migrate --publisher-type --storage-name --verbose +``` + +3. **Deploy the updated versions of the TechDocs plugins**: Once the migration + above has been run, you can deploy the beta versions of the TechDocs backend + and frontend plugins to your Backstage instance. + +4. **Verify that your TechDocs sites are still loading/accessible**: Try + accessing a TechDocs site using different entity-triplet case variants, e.g. + `/docs/namespace/KIND/name` or `/docs/namespace/kind/name`. Your TechDocs + site should load regardless of the URL path casing you use. + +5. **Clean up the old objects from storage**: Once you've verified that your + TechDocs site is accessible, you can clean up your storage bucket by + re-running the `migrate` command on the TechDocs CLI, but with an additional + `removeOriginal` flag passed: + +```sh +techdocs-cli migrate --publisher-type --storage-name --removeOriginal --verbose +``` + +6. **Update your CI/CD pipelines to use the beta version of the TechDocs CLI**: + Finally, you can update all of your CI/CD pipelines to use at least v0.x.y of + the TechDocs CLI, ensuring that all sites are published to the new, + lower-cased entity triplet paths going forward. + +If you encounter problems running this migration, please [report the +issue][beta-migrate-bug]. You can temporarily revert to pre-beta storage +expectations with a configuration change: + +```yaml +techdocs: + legacyUseCaseSensitiveTripletPaths: true +``` + +[beta-migrate-bug]: +https://github.com/backstage/backstage/issues/new?assignees=&labels=bug&template=bug_template.md&title=[TechDocs]%20Unable%20to%20run%20beta%20migration +[using-cloud-storage]: ./using-cloud-storage.md diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index 537daa815f..2f1efa0037 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -248,7 +248,12 @@ export interface Config { storageUrl?: string; /** - * (Optional) + * (Optional and not recommended) Prior to version [0.x.y] of TechDocs, docs + * sites could only be accessed over paths with case-sensitive entity triplets + * e.g. (namespace/Kind/name). If you are upgrading from an older version of + * TechDocs and are unable to perform the necessary migration of files in your + * external storage, you can set this value to `true` to temporarily revert to + * the old, case-sensitive entity triplet behavior. */ legacyUseCaseSensitiveTripletPaths?: boolean; }; From 3067f0a0c2dcd208f150be59710c20e4aa0840bd Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 9 Aug 2021 18:37:31 +0200 Subject: [PATCH 10/15] Note OpenStack Swift caveat. Signed-off-by: Eric Peterson --- .changeset/techdocs-all-done.md | 11 ++++++----- docs/features/techdocs/README.md | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.changeset/techdocs-all-done.md b/.changeset/techdocs-all-done.md index 6d2146e2a5..c4c1b5ed83 100644 --- a/.changeset/techdocs-all-done.md +++ b/.changeset/techdocs-all-done.md @@ -10,10 +10,11 @@ any case (e.g. `/docs/namespace/KIND/name` or `/docs/namespace/kind/name`). If you do not use an external storage provider for serving TechDocs, this is a transparent change and no action is required from you. -If you _do_ use an external storage provider for serving TechDocs (e.g. Google -Cloud Storage, AWS S3, or Azure Blob Storage), you must run a migration command -against your storage provider before updating to this version. +If you _do_ use an external storage provider for serving TechDocs (one of\* GCS, +AWS S3, or Azure Blob Storage), you must run a migration command against your +storage provider before updating. -[A migration guide is available here](#todo). +[A migration guide is available here](https://backstage.io/docs/features/techdocs/how-to-guides#how-to-migrate-from-techdocs-alpha-to-beta). -This is the third and final step of migrating cloud storage entities to lowercase ([see](https://github.com/backstage/backstage/issues/4367#issuecomment-876461002)). +- (\*) We're seeking help from the community to bring OpenStack Swift support + [to feature parity](https://github.com/backstage/backstage/issues/6763) with the above. diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index 6732b3ab18..b845691ac8 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -54,7 +54,7 @@ providers are used. | Google Cloud Storage (GCS) | Yes ✅ | | Amazon Web Services (AWS) S3 | Yes ✅ | | Azure Blob Storage | Yes ✅ | -| OpenStack Swift | Yes ✅ | +| OpenStack Swift | Community ✅ | [Reach out to us](#feedback) if you want to request more platforms. From add5ee5d97ac949929d4790665bdeef5ddf40bb3 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 10 Aug 2021 12:30:34 +0200 Subject: [PATCH 11/15] Use lower-case links within techdocs plugin; allow legacy behavior with config. Signed-off-by: Eric Peterson --- plugins/techdocs/config.d.ts | 7 ++ .../src/home/components/DocsCardGrid.test.tsx | 65 +++++++++++++++- .../src/home/components/DocsCardGrid.tsx | 16 +++- .../src/home/components/DocsTable.test.tsx | 76 ++++++++++++++++++- .../src/home/components/DocsTable.tsx | 13 +++- 5 files changed, 168 insertions(+), 9 deletions(-) diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index ebc2aa2fd7..d6bf21b10b 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -26,6 +26,13 @@ export interface Config { */ builder: 'local' | 'external'; + /** + * Allows fallback to case-sensitive triplets in case of migration issues. + * @visibility frontend + * @see https://backstage.io/docs/features/techdocs/how-to-guides#how-to-migrate-from-techdocs-alpha-to-beta + */ + legacyUseCaseSensitiveTripletPaths?: boolean; + /** * @example http://localhost:7000/api/techdocs * @visibility frontend diff --git a/plugins/techdocs/src/home/components/DocsCardGrid.test.tsx b/plugins/techdocs/src/home/components/DocsCardGrid.test.tsx index f9831e2fa4..4adada48fa 100644 --- a/plugins/techdocs/src/home/components/DocsCardGrid.test.tsx +++ b/plugins/techdocs/src/home/components/DocsCardGrid.test.tsx @@ -17,11 +17,35 @@ import { wrapInTestApp } from '@backstage/test-utils'; import { render } from '@testing-library/react'; import React from 'react'; +import { configApiRef } from '@backstage/core-plugin-api'; import { DocsCardGrid } from './DocsCardGrid'; +// Hacky way to mock a specific boolean config value. +const getOptionalBooleanMock = jest.fn().mockReturnValue(false); +jest.mock('@backstage/core-plugin-api', () => ({ + ...jest.requireActual('@backstage/core-plugin-api'), + useApi: (apiRef: any) => { + const actualUseApi = jest.requireActual( + '@backstage/core-plugin-api', + ).useApi; + const actualApi = actualUseApi(apiRef); + if (apiRef === configApiRef) { + const configReader = actualApi; + configReader.getOptionalBoolean = getOptionalBooleanMock; + return configReader; + } + + return actualApi; + }, +})); + describe('Entity Docs Card Grid', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + it('should render all entities passed ot it', async () => { - const { findByText } = render( + const { findByText, findAllByRole } = render( wrapInTestApp( { ); expect(await findByText('testName')).toBeInTheDocument(); expect(await findByText('testName2')).toBeInTheDocument(); + const [button1, button2] = await findAllByRole('button'); + expect(button1.getAttribute('href')).toContain( + '/default/testkind/testname', + ); + expect(button2.getAttribute('href')).toContain( + '/default/testkind2/testname2', + ); + }); + + it('should fall back to case-sensitive links when configured', async () => { + getOptionalBooleanMock.mockReturnValue(true); + + const { findByRole } = render( + wrapInTestApp( + , + ), + ); + + const button = await findByRole('button'); + expect(getOptionalBooleanMock).toHaveBeenCalledWith( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ); + expect(button.getAttribute('href')).toContain( + '/SomeNamespace/TestKind/testName', + ); }); }); diff --git a/plugins/techdocs/src/home/components/DocsCardGrid.tsx b/plugins/techdocs/src/home/components/DocsCardGrid.tsx index 500debaf2d..8b7e114b09 100644 --- a/plugins/techdocs/src/home/components/DocsCardGrid.tsx +++ b/plugins/techdocs/src/home/components/DocsCardGrid.tsx @@ -18,6 +18,7 @@ import React from 'react'; import { generatePath } from 'react-router-dom'; import { Entity } from '@backstage/catalog-model'; +import { useApi, configApiRef } from '@backstage/core-plugin-api'; import { Card, CardActions, CardContent, CardMedia } from '@material-ui/core'; import { rootDocsRouteRef } from '../../routes'; @@ -32,6 +33,13 @@ export const DocsCardGrid = ({ }: { entities: Entity[] | undefined; }) => { + // Lower-case entity triplets by default, but allow override. + const toLowerMaybe = useApi(configApiRef).getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) + ? (str: string) => str + : (str: string) => str.toLocaleLowerCase(); + if (!entities) return null; return ( @@ -46,9 +54,11 @@ export const DocsCardGrid = ({