diff --git a/.changeset/chilly-dodos-drop.md b/.changeset/chilly-dodos-drop.md index 0bc099a353..4a03bb8ae1 100644 --- a/.changeset/chilly-dodos-drop.md +++ b/.changeset/chilly-dodos-drop.md @@ -1,7 +1,6 @@ --- -'@backstage/techdocs-common': minor -'@backstage/plugin-techdocs': minor -'@backstage/plugin-techdocs-backend': minor +'@backstage/techdocs-common': patch +'@backstage/plugin-techdocs-backend': patch --- 1. Added option to use AWS S3 as a choice to store the static generated files for TechDocs. diff --git a/.changeset/fast-colts-cross.md b/.changeset/fast-colts-cross.md deleted file mode 100644 index 0bc099a353..0000000000 --- a/.changeset/fast-colts-cross.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@backstage/techdocs-common': minor -'@backstage/plugin-techdocs': minor -'@backstage/plugin-techdocs-backend': minor ---- - -1. Added option to use AWS S3 as a choice to store the static generated files for TechDocs. diff --git a/app-config.yaml b/app-config.yaml index c6857a3327..4b8f66f6ce 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -76,7 +76,7 @@ techdocs: generators: techdocs: 'docker' # Alternatives - 'local' publisher: - type: 'local' # Alternatives - 'googleGcs' - 'awsS3'. Read documentation for using alternatives. + type: 'local' # Alternatives - 'googleGcs' or 'awsS3'. Read documentation for using alternatives. sentry: organization: my-company diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 7ab04c5a81..869febdf5c 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -52,7 +52,8 @@ techdocs: # techdocs.publisher.type can be - 'local' or 'googleGcs' (awsS3, azureStorage, etc. to be available as well). # When set to 'local', techdocs-backend will create a 'static' directory at its root to store generated documentation files. - # When set to 'googleGcs' or 'awsS3', techdocs-backend will use a Google Cloud Storage Bucket to store generated documentation files. + # When set to 'googleGcs', techdocs-backend will use a Google Cloud Storage Bucket to store generated documentation files. + # When set to 'awsS3', techdocs-backend will use an Amazon Web Service (AWS) S3 bucket to store generated documentation files. type: 'local' diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 8c4f45f07e..c1cea9c91b 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -134,8 +134,8 @@ and the **user** policy. **2.1.1 Create an Admin user** (if you don't have one yet) -Create an **administrator user** `ADMIN_USER` and grant him administrator -privileges by attaching a user policy giving him **full access**. +Create an **administrator user** account `ADMIN_USER` and grant it administrator +privileges by attaching a user policy giving the account **full access**. Note down the Admin User credentials and IAM User Sign-In URL as you will need to use this information in the next step. diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts index 55cc953b82..966021fea7 100644 --- a/packages/techdocs-common/__mocks__/aws-sdk.ts +++ b/packages/techdocs-common/__mocks__/aws-sdk.ts @@ -26,11 +26,18 @@ export class S3 { headObject({ Key }: { Key: string }) { return { - promise: this.promise, + promise: () => this.checkFileExists(Key), createReadStream: () => { const emitter = new EventEmitter(); process.nextTick(() => { - emitter.emit('data', Buffer.from(fs.readFileSync(Key))); + if (fs.existsSync(Key)) { + emitter.emit('data', Buffer.from(fs.readFileSync(Key))); + } else { + emitter.emit( + 'error', + new Error(`The file ${Key} doest not exist !`), + ); + } emitter.emit('end'); }); return emitter; @@ -40,22 +47,39 @@ export class S3 { getObject({ Key }: { Key: string }) { return { - promise: () => - new Promise(resolve => { - resolve(fs.existsSync(Key)); - }), + promise: () => this.checkFileExists(Key) }; } - promise() { - return new Promise(resolve => { - resolve(''); - }); + checkFileExists(Key: string) { + return new Promise((resolve, reject) => { + if (fs.existsSync(Key)) { + resolve(''); + } else { + reject({ message: 'The object doest not exist !'}); + } + }) } - upload() { + headBucket() { + return new Promise((resolve) => { + resolve('') + }) + } + + upload({ Key }: { Key: string }) { return { - promise: this.promise, + promise: () => + new Promise((resolve, reject) => { + if ( + fs.existsSync(Key) && + fs.readFileSync(Key).toString() === 'mock-error' + ) { + reject(''); + } else { + resolve(''); + } + }), }; } } diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index ca069ea858..1e7ed024b2 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -34,6 +34,7 @@ const createMockEntity = (annotations = {}) => { const logger = winston.createLogger(); jest.spyOn(logger, 'info').mockReturnValue(logger); +jest.spyOn(logger, 'error').mockReturnValue(logger); let publisher: PublisherBase; @@ -55,63 +56,117 @@ beforeEach(() => { }); describe('AwsS3Publish', () => { - it('should publish a directory', async () => { - mockFs({ - '/path/to/generatedDirectory': { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', + describe('publish', () => { + it('should publish a directory', async () => { + mockFs({ + '/path/to/generatedDirectory': { + 'index.html': '', + '404.html': '', + assets: { + 'main.css': '', + }, }, - }, + }); + + const entity = createMockEntity(); + expect( + await publisher.publish({ + entity, + directory: '/path/to/generatedDirectory', + }), + ).toBeUndefined(); + mockFs.restore(); }); - const entity = createMockEntity(); - expect( + it('should fail to publish a directory', async () => { + mockFs({ + '/path/to/generatedDirectory': { + 'index.html': 'mock-error', + '404.html': '', + assets: { + 'main.css': '', + }, + }, + }); + + const entity = createMockEntity(); await publisher.publish({ entity, directory: '/path/to/generatedDirectory', - }), - ).toBeUndefined(); - mockFs.restore(); + }).catch(error => expect(error).toBe(`Unable to upload file(s) to AWS S3. Error`)) + mockFs.restore(); + }); }); - it('should return true if docs has been generated', async () => { - const entityMock = { - apiVersion: 'apiVersion', - kind: 'kind', - metadata: { - namespace: '/namespace', + describe('hasDocsBeenGenerated', () => { + it('should return true if docs has been generated', async () => { + const entityMock = { + apiVersion: 'apiVersion', + kind: 'kind', + metadata: { + namespace: '/namespace', + name: 'name', + }, + }; + const entityRootDir = `${entityMock.metadata.namespace}/${entityMock.kind}/${entityMock.metadata.name}`; + mockFs({ + [entityRootDir]: { + 'index.html': 'file-content', + }, + }); + + expect(await publisher.hasDocsBeenGenerated(entityMock)).toBe(true); + mockFs.restore(); + }); + + it('should return false if docs has not been generated', async () => { + const entityMock = { + apiVersion: 'apiVersion', + kind: 'kind', + metadata: { + namespace: 'namespace', + name: 'name', + }, + }; + + expect(await publisher.hasDocsBeenGenerated(entityMock)).toBe(false); + }); + }); + + describe('fetchTechDocsMetadata', () => { + it('should return tech docs metadata', async () => { + const entityNameMock = { name: 'name', - }, - }; - const entityRootDir = `${entityMock.metadata.namespace}/${entityMock.kind}/${entityMock.metadata.name}`; - mockFs({ - [entityRootDir]: { - 'index.html': 'file-content', - }, + namespace: '/namespace', + kind: 'kind', + }; + const entityRootDir = `${entityNameMock.namespace}/${entityNameMock.kind}/${entityNameMock.name}`; + mockFs({ + [entityRootDir]: { + 'techdocs_metadata.json': 'file-content', + }, + }); + + expect(await publisher.fetchTechDocsMetadata(entityNameMock)).toBe( + 'file-content', + ); + mockFs.restore(); }); - expect(await publisher.hasDocsBeenGenerated(entityMock)).toBe(true); - mockFs.restore(); - }); - - it('should return tech docs metadata', async () => { - const entityNameMock = { - name: 'name', - namespace: '/namespace', - kind: 'kind', - }; - const entityRootDir = `${entityNameMock.namespace}/${entityNameMock.kind}/${entityNameMock.name}`; - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': 'file-content', - }, + it('should return an error if the techdocs_metadata.json file is not present', async () => { + const entityNameMock = { + name: 'name', + namespace: 'namespace', + kind: 'kind', + }; + const entityRootDir = `${entityNameMock.namespace}/${entityNameMock.kind}/${entityNameMock.name}`; + await publisher + .fetchTechDocsMetadata(entityNameMock) + .catch(error => + expect(error).toBe( + `The file ${entityRootDir}/techdocs_metadata.json doest not exist !`, + ), + ); }); - - expect(await publisher.fetchTechDocsMetadata(entityNameMock)).toBe( - 'file-content', - ); - mockFs.restore(); }); }); diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 94b981bbfb..959ffb87b3 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -55,26 +55,26 @@ export class AwsS3Publish implements PublisherBase { // Check if the defined bucket exists. Being able to connect means the configuration is good // and the storage client will work. - storageClient - .headObject({ + storageClient.headBucket( + { Bucket: bucketName, - Key: (credentialsJson as CredentialsOptions).accessKeyId, - }) - .promise() - .then(() => { - logger.info( - `Successfully connected to the AWS S3 bucket ${bucketName}.`, - ); - }) - .catch(reason => { - logger.error( - `Could not retrieve metadata about the AWS S3 bucket ${bucketName}. ` + - 'Make sure the AWS project and the bucket exists and the access key located at the path ' + - "techdocs.publisher.awsS3.credentials defined in app config has the role 'Storage Object Creator'. " + - 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', - ); - throw new Error(`from AWS client library: ${reason.message}`); - }); + }, + err => { + if (err) { + logger.error( + `Could not retrieve metadata about the AWS S3 bucket ${bucketName}. ` + + 'Make sure the AWS project and the bucket exists and the access key located at the path ' + + "techdocs.publisher.awsS3.credentials defined in app config has the role 'Storage Object Creator'. " + + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + ); + throw new Error(`from AWS client library: ${err.message}`); + } else { + logger.info( + `Successfully connected to the AWS S3 bucket ${bucketName}.`, + ); + } + }, + ); return new AwsS3Publish(storageClient, bucketName, logger); } @@ -95,7 +95,7 @@ export class AwsS3Publish implements PublisherBase { */ publish({ entity, directory }: PublishRequest): Promise { return new Promise(async (resolve, reject) => { - // Note: GCS manages creation of parent directories if they do not exist. + // Note: S3 manages creation of parent directories if they do not exist. // So collecting path of only the files is good enough. const allFilesToUpload = await getFileTreeRecursively(directory); @@ -123,7 +123,6 @@ export class AwsS3Publish implements PublisherBase { uploadPromises.push(this.storageClient.upload(params).promise()); }); }); - Promise.all(uploadPromises) .then(() => { this.logger.info( @@ -212,7 +211,7 @@ export class AwsS3Publish implements PublisherBase { return new Promise(resolve => { const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; this.storageClient - .getObject({ + .headObject({ Bucket: this.bucketName, Key: `${entityRootDir}/index.html`, }) diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 49da8695a6..c2ecaab61b 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -150,7 +150,7 @@ export async function createRouter({ await docsBuilder.build(); // With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched // on the user's page. If not, respond with a message asking them to check back later. - // The delay here is to make sure GCS registers newly uploaded files which is usually <1 second + // The delay here is to make sure GCS/AWS/etc. registers newly uploaded files which is usually <1 second let foundDocs = false; for (let attempt = 0; attempt < 5; attempt++) { if (await publisher.hasDocsBeenGenerated(entity)) {