diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index cd4b058644..66e6e15811 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -309,3 +309,94 @@ and read the static generated documentation files. When you start the backend of the app, you should be able to see `techdocs info Successfully connected to the Azure Blob Storage container` in the logs. + +## Configuring OpenStack Swift Container with TechDocs + +Follow the +[official OpenStack Api documentation](https://docs.openstack.org/api-ref/identity/v3/) +for the latest instructions on the following steps involving Azure Blob Storage. + +**1. Set `techdocs.publisher.type` config in your `app-config.yaml`** + +Set `techdocs.publisher.type` to `'openStackSwift'`. + +```yaml +techdocs: + publisher: + type: 'openStackSwift' +``` + +**2. Create an Azure Blob Storage Container** + +Create a dedicated container for TechDocs sites. +[Refer to the official documentation](https://docs.openstack.org/mitaka/user-guide/dashboard_manage_containers.html). + +TechDocs will publish documentation to this container and will fetch files from +here to serve documentation in Backstage. Note that the container names are +globally unique. + +Set the config `techdocs.publisher.openStackSwift.containerName` in your +`app-config.yaml` to the name of the container you just created. + +```yaml +techdocs: + publisher: + type: 'openStackSwift' + openStackSwift: + containerName: 'name-of-techdocs-storage-container' +``` + +**3a. (Recommended) Authentication using environment variable** + +Set the config `techdocs.publisher.openStackSwift.accountName` in +your `app-config.yaml` to the your account name. + +The storage blob client will automatically use the environment variable +`AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET` to authenticate with +Azure Blob Storage. +[Steps to create the service where the variables can be retrieved from](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal). + +https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad for more +details. + +```yaml +techdocs: + publisher: + type: 'azureBlobStorage' + azureBlobStorage: + containerName: 'name-of-techdocs-storage-bucket' + credentials: + accountName: + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME +``` + +**3b. Authentication using app-config.yaml** + +If you do not prefer (3a) and optionally like to use a service account, you can +follow these steps. + +To get credentials, access the Azure Portal and go to "Settings > Access Keys", +and get your Storage account name and Primary Key. +https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key +for more details. + +```yaml +techdocs: + publisher: + type: 'azureBlobStorage' + azureBlobStorage: + containerName: 'name-of-techdocs-storage-bucket' + credentials: + accountName: + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME + accountKey: + $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY +``` + +**4. That's it!** + +Your Backstage app is now ready to use Azure Blob Storage for TechDocs, to store +and read the static generated documentation files. When you start the backend of +the app, you should be able to see +`techdocs info Successfully connected to the Azure Blob Storage container` in +the logs. diff --git a/packages/techdocs-common/__mocks__/pkgcloud.ts b/packages/techdocs-common/__mocks__/pkgcloud.ts index bf729a7bd2..4316630928 100644 --- a/packages/techdocs-common/__mocks__/pkgcloud.ts +++ b/packages/techdocs-common/__mocks__/pkgcloud.ts @@ -16,7 +16,7 @@ import fs from 'fs-extra'; import os from 'os'; import path from 'path'; -import { ObjectWritableMock, BufferReadableMock } from 'stream-mock'; +import { EventEmitter } from 'events'; const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; @@ -61,16 +61,44 @@ class PkgCloudStorageClient { } } - upload() { - return new ObjectWritableMock(); + upload({ remote }: { remote: string }) { + const filePath = path.join(rootDir, remote); + + const emitter = new EventEmitter(); + + process.nextTick(() => { + if (fs.existsSync(filePath)) { + emitter.emit('success'); + (emitter as any).end = () => true; + } else { + emitter.emit( + 'error', + new Error(`The file ${filePath} does not exist !`), + ); + } + }); + + return emitter; } - download() { - const stringify = JSON.stringify({ - "site_description": 'site_content', - "site_name": "backstage" + download({ remote }: { remote: string }) { + const filePath = path.join(rootDir, remote); + + const emitter = new EventEmitter(); + + process.nextTick(() => { + if (fs.existsSync(filePath)) { + emitter.emit('data', Buffer.from(fs.readFileSync(filePath))); + emitter.emit('end'); + } else { + emitter.emit( + 'error', + new Error(`The file ${filePath} does not exist !`), + ); + } }); - return new BufferReadableMock([stringify]); + + return emitter; } } diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 2598de1dce..62d7b73f73 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/techdocs-common", "description": "Common functionalities for TechDocs, to be shared between techdocs-backend plugin and techdocs-cli", - "version": "0.4.2", + "version": "0.5.0", "main": "src/index.ts", "types": "src/index.ts", "private": false, @@ -58,7 +58,6 @@ "p-limit": "^3.1.0", "pkgcloud": "^2.2.0", "recursive-readdir": "^2.2.2", - "stream-mock": "^2.0.5", "winston": "^3.2.1" }, "devDependencies": { diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts index fab0bf3a12..30ec8a59b9 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts @@ -111,14 +111,12 @@ describe('OpenStackSwiftPublish', () => { const entity = createMockEntity(); const entityRootDir = getEntityRootDir(entity); - setTimeout(async () => { - expect( - await publisher.publish({ - entity, - directory: entityRootDir, - }), - ).toBeUndefined() - }, 5000); + expect( + await publisher.publish({ + entity, + directory: entityRootDir, + }), + ).toBeUndefined() }); it('should fail to publish a directory', async () => { diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.ts index 60c8349dee..4c4c15d9c4 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.ts @@ -49,7 +49,7 @@ export class OpenStackSwiftPublish implements PublisherBase { } catch (error) { throw new Error( "Since techdocs.publisher.type is set to 'openStackSwift' in your app config, " + - 'techdocs.publisher.openStackSwift.containerName is required.', + 'techdocs.publisher.openStackSwift.containerName is required.', ); } @@ -59,8 +59,8 @@ export class OpenStackSwiftPublish implements PublisherBase { const storageClient = storage.createClient({ provider: 'openstack', - username: openStackSwiftConfig.getString('username'), - password: openStackSwiftConfig.getString('password'), + username: openStackSwiftConfig.getString('credentials.username'), + password: openStackSwiftConfig.getString('credentials.password'), authUrl: openStackSwiftConfig.getString('authUrl'), keystoneAuthVersion: openStackSwiftConfig.getOptionalString('keystoneAuthVersion') || 'v3', @@ -80,9 +80,9 @@ export class OpenStackSwiftPublish implements PublisherBase { } else { logger.error( `Could not retrieve metadata about the OpenStack Swift container ${containerName}. ` + - 'Make sure the container exists. Also make sure that authentication is setup either by ' + - 'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' + - 'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + 'Make sure the container exists. Also make sure that authentication is setup either by ' + + 'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' + + 'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', ); logger.error(`from OpenStack client library: ${err.message}`); @@ -139,15 +139,15 @@ export class OpenStackSwiftPublish implements PublisherBase { // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => - new Promise((res, rej) => { - const writeStream = this.storageClient.upload(params); + new Promise((res, rej) => { + const writeStream = this.storageClient.upload(params); - writeStream.on('error', rej); + writeStream.on('error', rej); - writeStream.on('success', res); + writeStream.on('success', res); - readStream.pipe(writeStream); - }), + readStream.pipe(writeStream); + }), ); uploadPromises.push(uploadFile); } diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 03bea5233d..0de3393097 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-techdocs-backend", - "version": "0.6.2", + "version": "0.7.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", diff --git a/yarn.lock b/yarn.lock index 5b7893c852..0f2cbbb7d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1816,85 +1816,44 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.2.0": - version "0.2.0" - resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.2.0.tgz#e3fe2a4ddeb6a9b6ec480c80cb2b9c39cb245576" - integrity sha512-Y1ocdRpBlxK/VrJQjHlQd0bgADECd1B2NRjwd8ss46ibT5hwLvMOfD80+Fa7oPLu0ktJrH4lq0pNIIJIml48zA== + version "0.7.2" dependencies: - "@backstage/config" "^0.1.1" + "@backstage/config" "^0.1.3" "@types/json-schema" "^7.0.5" "@types/yup" "^0.29.8" + ajv "^7.0.3" json-schema "^0.2.5" lodash "^4.17.15" uuid "^8.0.0" yup "^0.29.3" "@backstage/catalog-model@^0.3.0": - version "0.3.1" - resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.3.1.tgz#45d08e2f333c9c566b2bf2629fd707fe989bb404" - integrity sha512-9XhV7c4rmVW+Yzj2PiwTQ7DsegWGB3C4ELsDRExuEVZONdqNcC02cyJtrt3fT5F31ZS3tHkB9bMUymFOBLqUSA== + version "0.7.2" dependencies: - "@backstage/config" "^0.1.1" + "@backstage/config" "^0.1.3" "@types/json-schema" "^7.0.5" "@types/yup" "^0.29.8" + ajv "^7.0.3" json-schema "^0.2.5" lodash "^4.17.15" uuid "^8.0.0" yup "^0.29.3" "@backstage/core@^0.3.0": - version "0.3.2" - resolved "https://registry.npmjs.org/@backstage/core/-/core-0.3.2.tgz#a8209126d5076cf4a8b9bd632fe4e5e2edb62916" - integrity sha512-i5d+Wh8js4qEWoAsPY5L7HVSWpumr1OhfF2dUCGYdyW6AMqVJPca6+n6zp1Rg2CO+J9norp44XAVVCbyhtUpig== + version "0.6.3" dependencies: - "@backstage/config" "^0.1.1" - "@backstage/core-api" "^0.2.1" - "@backstage/theme" "^0.2.1" - "@material-ui/core" "^4.11.0" - "@material-ui/icons" "^4.9.1" - "@material-ui/lab" "4.0.0-alpha.45" - "@types/dagre" "^0.7.44" - "@types/react" "^16.9" - "@types/react-sparklines" "^1.7.0" - classnames "^2.2.6" - clsx "^1.1.0" - d3-selection "^2.0.0" - d3-shape "^2.0.0" - d3-zoom "^2.0.0" - dagre "^0.8.5" - immer "^7.0.9" - lodash "^4.17.15" - material-table "^1.69.1" - prop-types "^15.7.2" - qs "^6.9.4" - rc-progress "^3.0.0" - react "^16.12.0" - react-dom "^16.12.0" - react-helmet "6.1.0" - react-hook-form "^6.6.0" - react-markdown "^5.0.2" - react-router "6.0.0-beta.0" - react-router-dom "6.0.0-beta.0" - react-sparklines "^1.7.0" - react-syntax-highlighter "^13.5.1" - react-use "^15.3.3" - remark-gfm "^1.0.0" - zen-observable "^0.8.15" - -"@backstage/core@^0.5.0": - version "0.5.0" - resolved "https://registry.npmjs.org/@backstage/core/-/core-0.5.0.tgz#6ff384adc595c18c7db60b9b2d23ebbb9086ed36" - integrity sha512-lCxgKBavUlLYZjZmRF8A7koP4NUhK/tbdf9SaEod0miZg6JTaDoAm3dmHPyqrMBHgoRRCDTxRIxNhj/8vY87oA== - dependencies: - "@backstage/config" "^0.1.2" - "@backstage/core-api" "^0.2.8" - "@backstage/theme" "^0.2.2" + "@backstage/config" "^0.1.3" + "@backstage/core-api" "^0.2.11" + "@backstage/theme" "^0.2.3" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" + "@testing-library/react-hooks" "^3.4.2" "@types/dagre" "^0.7.44" "@types/prop-types" "^15.7.3" "@types/react" "^16.9" "@types/react-sparklines" "^1.7.0" + "@types/react-text-truncate" "^0.14.0" classnames "^2.2.6" clsx "^1.1.0" d3-selection "^2.0.0" @@ -1916,57 +1875,25 @@ react-router-dom "6.0.0-beta.0" react-sparklines "^1.7.0" react-syntax-highlighter "^13.5.1" + react-text-truncate "^0.16.0" react-use "^15.3.3" remark-gfm "^1.0.0" zen-observable "^0.8.15" -"@backstage/plugin-catalog-react@^0.0.2": - version "0.0.2" - resolved "https://registry.npmjs.org/@backstage/plugin-catalog-react/-/plugin-catalog-react-0.0.2.tgz#e50da2dac9fab3a0d5973f8d1083ee2c368e5e52" - integrity sha512-O6aujFPRaEFTk4XlwOoswbnoHIOqMtj6ycUj6R1mNKOM4plUgGDKKhO3be69FHMJEMbiSvVe6AW+1kXaK+1LqA== - dependencies: - "@backstage/catalog-client" "^0.3.5" - "@backstage/catalog-model" "^0.7.1" - "@backstage/core" "^0.6.0" - "@material-ui/core" "^4.11.0" - "@types/react" "^16.9" - react "^16.13.1" - react-router "6.0.0-beta.0" - react-router-dom "6.0.0-beta.0" - react-use "^15.3.3" - -"@backstage/plugin-catalog-react@^0.0.4": - version "0.0.4" - resolved "https://registry.npmjs.org/@backstage/plugin-catalog-react/-/plugin-catalog-react-0.0.4.tgz#a4c8ba90cf48106ac6af2e03afa6338010a1299b" - integrity sha512-1fAqULJvLyE+3SeZ2yxDJnJ3SbUFv2Im55d3KbMgRaSog1chSJJoO3jbIwIRQIBXjRmCXrZbf56qwwWwxj6OjA== +"@backstage/plugin-catalog@^0.2.1": + version "0.4.0" dependencies: "@backstage/catalog-client" "^0.3.6" - "@backstage/catalog-model" "^0.7.1" - "@backstage/core" "^0.6.2" - "@material-ui/core" "^4.11.0" - "@types/react" "^16.9" - react "^16.13.1" - react-router "6.0.0-beta.0" - react-router-dom "6.0.0-beta.0" - react-use "^15.3.3" - -"@backstage/plugin-catalog@^0.2.1": - version "0.2.14" - resolved "https://registry.npmjs.org/@backstage/plugin-catalog/-/plugin-catalog-0.2.14.tgz#50a4176a55ffa543a426ec78cbc9deaecdbcf2b7" - integrity sha512-lDmNcC+m1zbbzYATUp5yIZ5PUp+YyBc1KKu3CCgqjLWSbJ1aJrU1N4g59euel1l2+qSW+lH76Kkp6ZYpZbSO9A== - dependencies: - "@backstage/catalog-client" "^0.3.5" - "@backstage/catalog-model" "^0.7.0" - "@backstage/core" "^0.5.0" - "@backstage/plugin-scaffolder" "^0.4.1" - "@backstage/theme" "^0.2.2" + "@backstage/catalog-model" "^0.7.2" + "@backstage/core" "^0.6.3" + "@backstage/plugin-catalog-react" "^0.1.0" + "@backstage/theme" "^0.2.3" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" "@types/react" "^16.9" classnames "^2.2.6" git-url-parse "^11.4.4" - moment "^2.26.0" react "^16.13.1" react-dom "^16.13.1" react-helmet "6.1.0" @@ -1976,15 +1903,12 @@ swr "^0.3.0" "@backstage/plugin-catalog@^0.3.1": - version "0.3.2" - resolved "https://registry.npmjs.org/@backstage/plugin-catalog/-/plugin-catalog-0.3.2.tgz#06945f10fd678efdade3f2795590c12433568fa0" - integrity sha512-iHLxPHRN9nYIXwOEAQ06m+PagsFb6Nb/XjJSebCAnSrAxPPITvBCfxc2H1GbyyMdC7KAr1ozORB/FFkNsCaQJg== + version "0.4.0" dependencies: "@backstage/catalog-client" "^0.3.6" - "@backstage/catalog-model" "^0.7.1" - "@backstage/core" "^0.6.2" - "@backstage/plugin-catalog-react" "^0.0.4" - "@backstage/plugin-scaffolder" "^0.5.1" + "@backstage/catalog-model" "^0.7.2" + "@backstage/core" "^0.6.3" + "@backstage/plugin-catalog-react" "^0.1.0" "@backstage/theme" "^0.2.3" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" @@ -2025,56 +1949,6 @@ react-use "^15.3.3" swr "^0.3.0" -"@backstage/plugin-scaffolder@^0.4.1": - version "0.4.2" - resolved "https://registry.npmjs.org/@backstage/plugin-scaffolder/-/plugin-scaffolder-0.4.2.tgz#58159227997f7e248ce52535bc32f19fcd0990dc" - integrity sha512-YuyHM587Rqg6KufxfFqQdI7dsZniBM/11Aj8Q0m5ZszOpCuNmDDkR1VX8MKHTBJ709mnLAqRgArdla7FOrOAXQ== - dependencies: - "@backstage/catalog-model" "^0.7.1" - "@backstage/core" "^0.6.0" - "@backstage/plugin-catalog-react" "^0.0.2" - "@backstage/theme" "^0.2.3" - "@material-ui/core" "^4.11.0" - "@material-ui/icons" "^4.9.1" - "@material-ui/lab" "4.0.0-alpha.45" - "@rjsf/core" "^2.4.0" - "@rjsf/material-ui" "^2.4.0" - classnames "^2.2.6" - git-url-parse "^11.4.4" - moment "^2.26.0" - react "^16.13.1" - react-dom "^16.13.1" - react-lazylog "^4.5.2" - react-router "6.0.0-beta.0" - react-router-dom "6.0.0-beta.0" - react-use "^15.3.3" - swr "^0.3.0" - -"@backstage/plugin-scaffolder@^0.5.1": - version "0.5.1" - resolved "https://registry.npmjs.org/@backstage/plugin-scaffolder/-/plugin-scaffolder-0.5.1.tgz#9d36f6b01991ddd9f9f2996068f3f31c766db8db" - integrity sha512-EG+iUc107bneVBPQpFKGp2jD9Y9+x50g/gY6TBN1je8TkgyluoxMj7wKv9e+d4TeGzXwK/LW/suajG9Zo0TJGQ== - dependencies: - "@backstage/catalog-model" "^0.7.1" - "@backstage/core" "^0.6.2" - "@backstage/plugin-catalog-react" "^0.0.4" - "@backstage/theme" "^0.2.3" - "@material-ui/core" "^4.11.0" - "@material-ui/icons" "^4.9.1" - "@material-ui/lab" "4.0.0-alpha.45" - "@rjsf/core" "^2.4.0" - "@rjsf/material-ui" "^2.4.0" - classnames "^2.2.6" - git-url-parse "^11.4.4" - moment "^2.26.0" - react "^16.13.1" - react-dom "^16.13.1" - react-lazylog "^4.5.2" - react-router "6.0.0-beta.0" - react-router-dom "6.0.0-beta.0" - react-use "^15.3.3" - swr "^0.3.0" - "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -15205,11 +15079,6 @@ immer@1.10.0: resolved "https://registry.npmjs.org/immer/-/immer-1.10.0.tgz#bad67605ba9c810275d91e1c2a47d4582e98286d" integrity sha512-O3sR1/opvCDGLEVcvrGTMtLac8GJ5IwZC4puPrLuRj3l7ICKvkmA0vGuU9OW8mV9WIBRnaxp5GJh9IEAaNOoYg== -immer@^7.0.9: - version "7.0.15" - resolved "https://registry.npmjs.org/immer/-/immer-7.0.15.tgz#dc3bc6db87401659d2e737c67a21b227c484a4ad" - integrity sha512-yM7jo9+hvYgvdCQdqvhCNRRio0SCXc8xDPzA25SvKWa7b1WVPjLwQs1VYU5JPXjcJPTqAa5NP5dqpORGYBQ2AA== - immer@^8.0.1: version "8.0.1" resolved "https://registry.npmjs.org/immer/-/immer-8.0.1.tgz#9c73db683e2b3975c424fb0572af5889877ae656" @@ -18764,7 +18633,7 @@ modify-values@^1.0.0: resolved "https://registry.npmjs.org/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022" integrity sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw== -moment@^2.19.3, moment@^2.25.3, moment@^2.26.0, moment@^2.27.0: +moment@^2.19.3, moment@^2.25.3, moment@^2.27.0: version "2.29.1" resolved "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== @@ -24159,11 +24028,6 @@ stream-http@^2.7.2: to-arraybuffer "^1.0.0" xtend "^4.0.0" -stream-mock@^2.0.5: - version "2.0.5" - resolved "https://registry.npmjs.org/stream-mock/-/stream-mock-2.0.5.tgz#c99d24bd6dbb0eaa57cf6ffefdb064150747826e" - integrity sha512-dx9skT8QYjwLsal+MhGHr4UtgS49brw851C/oTixmhCi4Ip+/qnZmhV1qOcznYYAED6gYKmKea+jjza4/wjpSg== - stream-shift@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d"