+
+This is an example of how you can manage your policies:
+
+a. Admin user creates a **bucket policy** granting a set of permissions to our
+TechDocs user.
+
+b. Admin user attaches a **user policy** to the TechDocs user granting
+additional permissions.
+
+c. TechDocs User then tries permissions granted via both the **bucket** policy
+and the **user** policy.
+
+**2.1 Creation**
+
+**2.1.1 Create an Admin user** (if you don't have one yet)
+
+Create an **administrator user** account `ADMIN_USER` and grant it administrator
+privileges by attaching a user policy giving the account **full access**.
+Note down the Admin User credentials and IAM User Sign-In URL as you will need
+to use this information in the next step.
+
+**2.1.2 Create an AWS S3 Bucket**
+
+Using the credentials of your Admin User `ADMIN_USER`, and the special IAM user
+sign-in URL, create a dedicated **bucket** for TechDocs sites. techdocs-backend
+will publish documentation to this bucket. TechDocs will fetch files from here
+to serve documentation in Backstage.
+
+Set the name of the bucket to `techdocs.publisher.awsS3.bucketName`.
+
+```yaml
+techdocs:
+ publisher:
+ type: 'awsS3'
+ awsS3:
+ bucketName: 'name-of-techdocs-storage-bucket'
+```
+
+**2.1.3 Create the `TechDocs` user**
+
+This user will be used to interact with your bucket, it will only have
+permissions to **get - put** objects.
+
+In the IAM console, do the following:
+
+- Create a new user, `TechDocs`
+- Note down the TechDocs User credentials
+- Note down the Amazon Resource Name (ARN) for the TechDocs user. In the IAM
+ console, select the TechDocs user, and you can find the user ARN in the
+ Summary tab.
+
+**2.2 Attach policies**
+
+Remember that you can use Bucket policy **or** User policy.
+Just make sure that you grant all the permissions to the TechDocs user:
+`3:PutObject`, `s3:GetObject`, `s3:ListBucket` and `s3:GetBucketLocation`.
+
+**2.2.1 Create the bucket policy**
+
+You now have to attach the following policy to your bucket in the Permission
+section:
+
+```json
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Sid": "statement1",
+ "Effect": "Allow",
+ "Principal": {
+ "AWS": "arn:aws:iam::YOUR_ACCOUNT_ID:user/TechDocs"
+ },
+ "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
+ "Resource": ["arn:aws:s3:::name-of-techdocs-storage-bucket"]
+ },
+ {
+ "Sid": "statement2",
+ "Effect": "Allow",
+ "Principal": {
+ "AWS": "arn:aws:iam::YOUR_ACCOUNT_ID:user/TechDocs"
+ },
+ "Action": ["s3:GetObject"],
+ "Resource": ["arn:aws:s3:::name-of-techdocs-storage-bucket/*"]
+ }
+ ]
+}
+```
+
+- The first statement grants **TechDocs User** the bucket operation permissions
+ `s3:GetBucketLocation` and `s3:ListBucket` which are permissions required by
+ the console.
+- The second statement grants the `s3:GetObject` permission.
+ (**NOTE :** if you do not use the user policy defined below you must also add
+ the `s3:PutObject` permission to allow the TechDocs user to add objects.)
+
+**2.2.2 Create the user policy**
+
+Create an inline policy for the TechDocs user by using the following policy:
+
+```json
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Sid": "PermissionForObjectOperations",
+ "Effect": "Allow",
+ "Action": ["s3:PutObject"],
+ "Resource": ["arn:aws:s3:::name-of-techdocs-storage-bucket/*"]
+ }
+ ]
+}
+```
+
+See more details in the section
+[Working with Inline Policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_manage.html).
+
+Now you need to fill in the environment variables with the `TechDocs` User
+credentials. You can also specify a region if you want to accesses the resources
+in a specific region. Otherwise no region will be selected by default.
+
+```properties
+TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL="TECHDOCS_ACCESS_KEY_ID"
+TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL="TECHDOCS_SECRET_ACCESS_KEY"
+AWSS3_REGION="" // Optional
+```
+
+Make it available in your Backstage server and/or your local development server
+and set it in the app config techdocs.publisher.awsS3.
+
+```yaml
+techdocs:
+ publisher:
+ type: 'awsS3'
+ awsS3:
+ credentials:
+ accessKeyId:
+ $env: TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL
+ secretAccessKey:
+ $env: TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL
+ region:
+ $env: AWSS3_REGION
+```
+
+**3. That's it!**
+
+Your Backstage app is now ready to use AWS S3 for TechDocs, to store the static
+generated documentation files.
diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts
new file mode 100644
index 0000000000..987b3d2800
--- /dev/null
+++ b/packages/techdocs-common/__mocks__/aws-sdk.ts
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type { S3 as S3Types } from 'aws-sdk';
+import { EventEmitter } from 'events';
+import fs from 'fs';
+
+export class S3 {
+ private readonly options;
+
+ constructor(options: S3Types.ClientConfiguration) {
+ this.options = options;
+ }
+
+ headObject({ Key }: { Key: string }) {
+ return {
+ promise: () => this.checkFileExists(Key),
+ };
+ }
+
+ getObject({ Key }: { Key: string }) {
+ return {
+ promise: () => this.checkFileExists(Key),
+ createReadStream: () => {
+ const emitter = new EventEmitter();
+ process.nextTick(() => {
+ if (fs.existsSync(Key)) {
+ emitter.emit('data', Buffer.from(fs.readFileSync(Key)));
+ } else {
+ emitter.emit(
+ 'error',
+ new Error(`The file ${Key} doest not exist !`),
+ );
+ }
+ emitter.emit('end');
+ });
+ return emitter;
+ },
+ };
+ }
+
+ checkFileExists(Key: string) {
+ return new Promise((resolve, reject) => {
+ if (fs.existsSync(Key)) {
+ resolve('');
+ } else {
+ reject({ message: 'The object doest not exist !' });
+ }
+ });
+ }
+
+ headBucket() {
+ return new Promise(resolve => {
+ resolve('');
+ });
+ }
+
+ upload({ Key }: { Key: string }) {
+ return {
+ promise: () =>
+ new Promise((resolve, reject) => {
+ if (!fs.existsSync(Key)) {
+ reject('');
+ } else {
+ resolve('');
+ }
+ }),
+ };
+ }
+}
+
+export default {
+ S3,
+};
diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json
index fd20475352..7ddb0b7dff 100644
--- a/packages/techdocs-common/package.json
+++ b/packages/techdocs-common/package.json
@@ -43,6 +43,7 @@
"@google-cloud/storage": "^5.6.0",
"@types/dockerode": "^3.2.1",
"@types/express": "^4.17.6",
+ "aws-sdk": "^2.817.0",
"cross-fetch": "^3.0.6",
"dockerode": "^3.2.1",
"express": "^4.17.1",
diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts
new file mode 100644
index 0000000000..501c171c2e
--- /dev/null
+++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts
@@ -0,0 +1,197 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import mockFs from 'mock-fs';
+import * as winston from 'winston';
+import { ConfigReader } from '@backstage/config';
+import { AwsS3Publish } from './awsS3';
+import { PublisherBase } from './types';
+
+const createMockEntity = (annotations = {}) => {
+ return {
+ apiVersion: 'version',
+ kind: 'TestKind',
+ metadata: {
+ name: 'test-component-name',
+ namespace: 'test-namespace',
+ annotations: {
+ ...annotations,
+ },
+ },
+ };
+};
+
+const logger = winston.createLogger();
+jest.spyOn(logger, 'info').mockReturnValue(logger);
+jest.spyOn(logger, 'error').mockReturnValue(logger);
+
+let publisher: PublisherBase;
+
+beforeEach(() => {
+ const mockConfig = new ConfigReader({
+ techdocs: {
+ requestUrl: 'http://localhost:7000',
+ publisher: {
+ type: 'awsS3',
+ awsS3: {
+ credentials: {
+ accessKeyId: 'accessKeyId',
+ secretAccessKey: 'secretAccessKey',
+ },
+ bucketName: 'bucketName',
+ },
+ },
+ },
+ });
+
+ publisher = AwsS3Publish.fromConfig(mockConfig, logger);
+});
+
+describe('AwsS3Publish', () => {
+ describe('publish', () => {
+ it('should publish a directory', async () => {
+ const entity = createMockEntity();
+ const {
+ kind,
+ metadata: { namespace, name },
+ } = entity;
+ mockFs({
+ [`${namespace}/${kind}/${name}`]: {
+ 'index.html': '',
+ '404.html': '',
+ assets: {
+ 'main.css': '',
+ },
+ },
+ });
+
+ expect(
+ await publisher.publish({
+ entity,
+ directory: `${namespace}/${kind}/${name}`,
+ }),
+ ).toBeUndefined();
+ mockFs.restore();
+ });
+
+ it('should fail to publish a directory', async () => {
+ const wrongPathToGeneratedDirectory = '/wrong/path/to/generatedDirectory';
+ const entity = createMockEntity();
+
+ const {
+ kind,
+ metadata: { namespace, name },
+ } = entity;
+
+ mockFs({
+ [`${namespace}/${kind}/${name}`]: {
+ 'index.html': '',
+ '404.html': '',
+ assets: {
+ 'main.css': '',
+ },
+ },
+ });
+
+ await publisher
+ .publish({
+ entity,
+ directory: wrongPathToGeneratedDirectory,
+ })
+ .catch(error =>
+ expect(error).toEqual(
+ new Error(
+ `Unable to upload file(s) to AWS S3. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
+ ),
+ ),
+ );
+ mockFs.restore();
+ });
+ });
+
+ describe('hasDocsBeenGenerated', () => {
+ it('should return true if docs has been generated', async () => {
+ const entityMock = {
+ apiVersion: 'apiVersion',
+ kind: 'kind',
+ metadata: {
+ namespace: '/namespace',
+ name: 'name',
+ },
+ };
+ const entityRootDir = `${entityMock.metadata.namespace}/${entityMock.kind}/${entityMock.metadata.name}`;
+ mockFs({
+ [entityRootDir]: {
+ 'index.html': 'file-content',
+ },
+ });
+
+ expect(await publisher.hasDocsBeenGenerated(entityMock)).toBe(true);
+ mockFs.restore();
+ });
+
+ it('should return false if docs has not been generated', async () => {
+ const entityMock = {
+ apiVersion: 'apiVersion',
+ kind: 'kind',
+ metadata: {
+ namespace: 'namespace',
+ name: 'name',
+ },
+ };
+
+ expect(await publisher.hasDocsBeenGenerated(entityMock)).toBe(false);
+ });
+ });
+
+ describe('fetchTechDocsMetadata', () => {
+ it('should return tech docs metadata', async () => {
+ const entityNameMock = {
+ name: 'name',
+ namespace: '/namespace',
+ kind: 'kind',
+ };
+ const entityRootDir = `${entityNameMock.namespace}/${entityNameMock.kind}/${entityNameMock.name}`;
+ mockFs({
+ [entityRootDir]: {
+ 'techdocs_metadata.json': 'file-content',
+ },
+ });
+
+ expect(await publisher.fetchTechDocsMetadata(entityNameMock)).toBe(
+ 'file-content',
+ );
+ mockFs.restore();
+ });
+
+ it('should return an error if the techdocs_metadata.json file is not present', async () => {
+ const entityNameMock = {
+ name: 'name',
+ namespace: 'namespace',
+ kind: 'kind',
+ };
+ const entityRootDir = `${entityNameMock.namespace}/${entityNameMock.kind}/${entityNameMock.name}`;
+ await publisher
+ .fetchTechDocsMetadata(entityNameMock)
+ .catch(error =>
+ expect(error).toEqual(
+ new Error(
+ `TechDocs metadata fetch failed, The file ${entityRootDir}/techdocs_metadata.json doest not exist !`,
+ ),
+ ),
+ );
+ });
+ });
+});
diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts
new file mode 100644
index 0000000000..15ff33408a
--- /dev/null
+++ b/packages/techdocs-common/src/stages/publish/awsS3.ts
@@ -0,0 +1,220 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import path from 'path';
+import express from 'express';
+import aws from 'aws-sdk';
+import { Logger } from 'winston';
+import { Entity, EntityName } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
+import { PublisherBase, PublishRequest } from './types';
+import { ManagedUpload } from 'aws-sdk/clients/s3';
+import fs from 'fs-extra';
+
+export class AwsS3Publish implements PublisherBase {
+ static fromConfig(config: Config, logger: Logger): PublisherBase {
+ let region = null;
+ let accessKeyId = null;
+ let secretAccessKey = null;
+ let bucketName = '';
+ try {
+ accessKeyId = config.getString(
+ 'techdocs.publisher.awsS3.credentials.accessKeyId',
+ );
+ secretAccessKey = config.getString(
+ 'techdocs.publisher.awsS3.credentials.secretAccessKey',
+ );
+ region = config.getOptionalString('techdocs.publisher.awsS3.region');
+ bucketName = config.getString('techdocs.publisher.awsS3.bucketName');
+ } catch (error) {
+ throw new Error(
+ "Since techdocs.publisher.type is set to 'awsS3' in your app config, " +
+ 'credentials and bucketName are required in techdocs.publisher.awsS3 ' +
+ 'required to authenticate with AWS S3.',
+ );
+ }
+
+ const storageClient = new aws.S3({
+ credentials: { accessKeyId, secretAccessKey },
+ ...(region && { region }),
+ });
+
+ // Check if the defined bucket exists. Being able to connect means the configuration is good
+ // and the storage client will work.
+ storageClient.headBucket(
+ {
+ Bucket: bucketName,
+ },
+ err => {
+ if (err) {
+ logger.error(
+ `Could not retrieve metadata about the AWS S3 bucket ${bucketName}. ` +
+ 'Make sure the AWS project and the bucket exists and the access key located at the path ' +
+ "techdocs.publisher.awsS3.credentials defined in app config has the role 'Storage Object Creator'. " +
+ 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
+ );
+ throw new Error(`from AWS client library: ${err.message}`);
+ } else {
+ logger.info(
+ `Successfully connected to the AWS S3 bucket ${bucketName}.`,
+ );
+ }
+ },
+ );
+
+ return new AwsS3Publish(storageClient, bucketName, logger);
+ }
+
+ constructor(
+ private readonly storageClient: aws.S3,
+ private readonly bucketName: string,
+ private readonly logger: Logger,
+ ) {
+ this.storageClient = storageClient;
+ this.bucketName = bucketName;
+ this.logger = logger;
+ }
+
+ /**
+ * Upload all the files from the generated `directory` to the S3 bucket.
+ * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
+ */
+ async publish({ entity, directory }: PublishRequest): Promise