diff --git a/docs/assets/techdocs/architecture-basic.drawio.svg b/docs/assets/techdocs/architecture-basic.drawio.svg new file mode 100644 index 0000000000..8e3be76f8b --- /dev/null +++ b/docs/assets/techdocs/architecture-basic.drawio.svg @@ -0,0 +1,462 @@ + + + + + + + + + + + + + + +
+
+
+ Repo 1 +
+ (with docs) +
+
+
+ + + Repo 1... + + + + + + + +
+
+
+ Repo 2 +
+ (with docs) +
+
+
+ + + Repo 2... + + + + + + + +
+
+
+ TechDocs plugin +
+
+
+ + + TechDocs plugin + + + + + + + + +
+
+
+ Request TechDocs site +
+
+
+ + + Request TechDocs site + + + + + + + + + + + + + + + + + + + + + +
+
+
+ Source code hosting +
+
+
+ + + Source code hosting + + + + + + + +
+
+
+ TechDocs Backend Plugin +
+
+
+ + + TechDocs Backend Plugin + + + + + + + +
+
+
+ (Stages) +
+
+
+ + + (Stages) + + + + + + + + + +
+
+
+ Prepare +
+
+
+ + + Prepare + + + + + + + + + +
+
+
+ Generate +
+
+
+ + + Generate + + + + + + + +
+
+
+ Publish +
+
+
+ + + Publish + + + + + + + + +
+
+
+ Local Filesystem +
+
+
+ + + Local File... + + + + + + + +
+
+
+ Fetch markdown files +
+
+
+ + + Fetch mark... + + + + + + + +
+
+
+ Build doc files using mkdocs +
+
+
+ + + Build doc... + + + + + + + +
+
+
+ Store generated static content +
+
+
+ + + Store gene... + + + + + + + +
+
+
+ TechDocs Basic (out-of-the-box) architecture +
+
+
+ + + TechDocs Basic (out-of-the-box) architect... + + + + + + + + + + + +
+
+
+ + Route handler + +
+
+
+ + + Route hand... + + + + + + + +
+
+
+ or +
+
+
+ + + or + + + + + + + +
+
+
+ Trigger new build if docs D.N.E. +
+ or are outdated +
+
+
+ + + Trigger new build... + + + + + + + +
+
+
+ Fetch files to render +
+
+
+ + + Fetch files... + + + + + + + + + + + + + + +
+
+
+ + GCP +
+ Bucket + +
+
+
+ + + GCP... + + + + + + + +
+
+
+ Azure storage +
+
+
+ + + Azure st... + + + + + + + + +
+
+
+ Storage solutions +
+
+
+ + + Storage solutions + + + + + + + + + + + +
+
+
+ + Amazon S3 + +
+
+
+ + + Amazon S3 + + + + + + + + + + Viewer does not support full SVG 1.1 + + + + diff --git a/docs/assets/techdocs/architecture-recommended.drawio.svg b/docs/assets/techdocs/architecture-recommended.drawio.svg new file mode 100644 index 0000000000..e3af4b6b5f --- /dev/null +++ b/docs/assets/techdocs/architecture-recommended.drawio.svg @@ -0,0 +1,391 @@ + + + + + + + + + + + + + + +
+
+
+ Repo 1 +
+ (with docs) +
+
+
+ + + Repo 1... + + + + + + + +
+
+
+ Repo 2 +
+ (with docs) +
+
+
+ + + Repo 2... + + + + + + + + + + + +
+
+
+ CI/CD System +
+
+
+ + + CI/CD System + + + + + + + + + +
+
+
+ techdocs-cli +
+
+
+ + + techdocs-cli + + + + + + + +
+
+
+ Cloud Storage +
+
+
+ + + Cloud Storage + + + + + + + + + +
+
+
+ + Amazon S3 + +
+
+
+ + + Amazon S3 + + + + + + + + + + + + +
+
+
+ + GCP +
+ Bucket + +
+
+
+ + + GCP... + + + + + + + +
+
+
+ Azure storage +
+
+
+ + + Azure st... + + + + + + + +
+
+
+ New commit +
+
+
+ + + New commit + + + + + + + +
+
+
+ New commit +
+
+
+ + + New commit + + + + + + + +
+
+
+ Build docs +
+ & +
+ Store generated static content +
+
+
+ + + Build docs... + + + + + + + + + + +
+
+
+ TechDocs plugin +
+
+
+ + + TechDocs plugin + + + + + + + +
+
+
+ TechDocs Backend plugin +
+
+
+ + + TechDocs Backend plu... + + + + + + + + + +
+
+
+ Request TechDocs site +
+
+
+ + + Request TechDocs site + + + + + + + +
+
+
+ Fetch files to render +
+
+
+ + + Fetch files to render + + + + + + + + + + + + + + + + + + +
+
+
+ Source code hosting +
+
+
+ + + Source code hosting + + + + + + + + +
+
+
+ Caching +
+ (Optional) +
+
+
+ + + Caching... + + + + + + + + + +
+
+
+ Storage solutions +
+
+
+ + + Storage solutions + + + + + + + +
+
+
+ TechDocs Recommended deployment architecture +
+
+
+ + + TechDocs Recommended deployment architecture + + + + + + + + + Viewer does not support full SVG 1.1 + + + + diff --git a/docs/assets/techdocs/techdocs_big_picture.png b/docs/assets/techdocs/techdocs_big_picture.png deleted file mode 100644 index 8b8f7a2338..0000000000 Binary files a/docs/assets/techdocs/techdocs_big_picture.png and /dev/null differ diff --git a/docs/features/techdocs/architecture.md b/docs/features/techdocs/architecture.md index 926ec238e1..01497b31bc 100644 --- a/docs/features/techdocs/architecture.md +++ b/docs/features/techdocs/architecture.md @@ -1,7 +1,153 @@ --- id: architecture -title: Architecture -description: Documentation on Architecture +title: TechDocs Architecture +description: Documentation on TechDocs Architecture --- -![TechDocs Big Picture](../../assets/techdocs/techdocs_big_picture.png) +## Basic (out-of-the-box) + +When you deploy Backstage (with TechDocs enabled by default), you get a basic +out-of-the box experience. + +![TechDocs Architecture diagram](../../assets/techdocs/architecture-basic.drawio.svg) + +> Note: See below for our recommended deployment architecture which takes care +> of stability, scalability and speed. + +When you open a TechDocs site in Backstage, the +[TechDocs Reader](./concepts.md#techdocs-reader) makes a request to +`techdocs-backend` with the entity ID and the path of the current page you are +looking at. In response, it receives the static files (HTML, CSS, JSON, etc.) to +render on the page in TechDocs/Backstage. + +The static files consist of HTML, CSS and Images generated by MkDocs. We remove +all the Javascript before adding them to Backstage for security reasons. And +there are some additional techdocs metadata JSON files that TechDocs needs to +render a site. + +The TechDocs Reader then applies a list of "Transformers" (see +[Concepts](./concepts.md)) which modify the generated static HTML files for a +number of use cases e.g. Remove certain headers, filter out some HTML tags, etc. + +Currently, we use the Backstage server's (or techdocs-backend's) local file +system to store the generated files. Publishing to an external storage system +(AWS S3, GCS, etc.) is also possible, but has not been implemented yet. + +A word about `UrlReader` vs Git preparer - Right now, we have two ways to fetch +files from its source repository for docs site generation. 1. By using Git +and 2. By directly using Source control (GitHub, Azure, etc.) APIs. This work is +heavily in progress. Please reach out to us on Discord in the #docs-like-code +channel to talk about it. + +## Recommended deployment + +This is how we recommend deploying TechDocs in production environment. + +![TechDocs Architecture diagram](../../assets/techdocs/architecture-recommended.drawio.svg) + +The key difference in the recommended deployment approach is where the docs are +built. + +We assume each entity lives in a repository somewhere (GitHub, GitLab, etc.). We +recommend using a CI/CD pipeline with the repository that has a dedicated +step/job to build docs for TechDocs. The generated static files are then stored +in a cloud storage solution of your choice. +[Track progress here](https://github.com/backstage/backstage/issues/3096). + +Similar to how it is done in the Basic setup, the TechDocs Reader requests +`techdocs-backend` plugin for the docs site. `techdocs-backend` then requests +your configured storage solution for the necessary files and returns them to +TechDocs Reader. + +We will provide instructions, scripts and/or templates (e.g. GitHub actions) to +build docs in your CI/CD system. +[Track progress here.](https://github.com/backstage/backstage/issues/3400) You +will be able to use `techdocs-cli` to build docs and publish the generated docs +site files to your cloud storage system. + +Note about caching: We have noticed internally that some storage providers can +be quite slow, which is why we are recommending a cache that sits between the +TechDocs Reader and the Storage. + +_Feel free to suggest better ideas to us in #docs-like-code channel in Discord +or via a GitHub issue._ + +### Security consideration + +Our biggest security concern is managing the access to the docs in the cloud +storage. We also want to have only one security solution for all different types +of storage (GCS, AWS, custom SFTP server, etc.) Restricting access to the +storage and only allowing `techdocs-backend` to fetch files is a good way to +achieve this. + +This would also allow us to use the access control management Backstage when +that is ready. +[Track progress here.](https://github.com/backstage/backstage/issues/3218) + +In theory, you can directly enable TechDocs Reader to read from your storage. +But, you will have to think about how to do it without the docs being public and +how access to user groups is managed. + +For cloud storage access tokens, `techdocs-backend` only needs a token with Read +permissions. But in your CI/CD system, there needs to be a token with Write +permissions to publish the generated docs site files. + +## FAQs + +**Q: Why do you have separate "basic" and "recommended" deployment approaches?** + +A: The basic or out-of-the-box setup is what you get when you create a new app +or do a git clone of the Backstage repository. We want the first experience to +_just work magically_ so that you can have your first experience with TechDocs +which is smooth. However, if you decide to deploy Backstage/TechDocs for +production use, the basic setup would work but there are going to be downsides +as you scale with the number of documentation sites and sizes of them. So you +would want to make sure the deployment is as stable as possible. Hence there is +a recommended approach. There can be even more deployment approaches to TechDocs +and we welcome such "Alternative" ideas from the community. + +**Q: Why don't you recommend techdocs-backend local filesystem to serve static +files?** + +A: It would make scaling a Backstage instance harder. Think about the case where +we have distributed Backstage deployments. Using a separate file storage system +for TechDocs makes it easier to do some operations like delete a docs site and +wipe its contents. + +**Q: Why aren't docs built on the fly i.e. when users visits a page, generate +docs site in real-time?** + +A: Generating the content from Markdown on the fly is not optimal (although that +is how the basic out-of-the-box setup is implemented). Storage solutions act as +a cache for the generated static content. TechDocs is also currently built on +MkDocs which does not allow us to build docs per-page, so we would have to build +all docs for a entity on every request. + +# Future work + +_Ideas here are far fetched and not in the project's milestone for near future +(~6 months)._ + +We currently depend on MkDocs to parse doc sites written in Markdown. And we +store the generated static assets and re-use it later to render in Backstage. A +better (futuristic) approach will be to directly parse whatever type of source +files you have in your docs repository and directly render in Backstage in +real-time. + +# Features status + +Status of all the features mentioned above. + +**In place ✅** + +- Basic setup with techdocs-backend file server as storage. + +**Work in progress 🚧** + +- Basic setup with cloud storage solution. + +**Not implemented yet ❌** + +- `techdocs-cli` is able to generate docs in CI/CD environment. +- `techdocs-cli` is able to publish docs site to any storage. +- `techdocs-backend` integration with Backstage access control management. diff --git a/mkdocs.yml b/mkdocs.yml index 8a4a78ddf1..1086259710 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,9 @@ site_name: 'Backstage' site_description: 'Main documentation for Backstage features and platform APIs' +plugins: + - techdocs-core + nav: - Overview: - What is Backstage?: 'overview/what-is-backstage.md' @@ -111,6 +114,3 @@ nav: - 'support/support.md' - 'support/project-structure.md' - FAQ: FAQ.md - -plugins: - - techdocs-core diff --git a/plugins/techdocs/src/api.ts b/plugins/techdocs/src/api.ts index 31e4a10e4c..194ad14d30 100644 --- a/plugins/techdocs/src/api.ts +++ b/plugins/techdocs/src/api.ts @@ -41,6 +41,11 @@ export interface TechDocs { getMetadata(metadataType: string, entityId: ParsedEntityId): Promise; } +/** + * API to talk to techdocs-backend. + * + * @property {string} apiOrigin Set to techdocs.requestUrl as the URL for techdocs-backend API + */ export class TechDocsApi implements TechDocs { public apiOrigin: string; @@ -60,6 +65,11 @@ export class TechDocsApi implements TechDocs { } } +/** + * API which talks to TechDocs storage to fetch files to render. + * + * @property {string} apiOrigin Set to techdocs.requestUrl as the URL for techdocs-backend API + */ export class TechDocsStorageApi implements TechDocsStorage { public apiOrigin: string; @@ -67,6 +77,14 @@ export class TechDocsStorageApi implements TechDocsStorage { this.apiOrigin = apiOrigin; } + /** + * Fetch HTML content as text for an individual docs page in an entity's docs site. + * + * @param {ParsedEntityId} entityId Object containing entity data like name, namespace, etc. + * @param {string} path The unique path to an individual docs page e.g. overview/what-is-new + * @returns {string} HTML content of the docs page as string + * @throws {Error} Throws error when the page is not found. + */ async getEntityDocs(entityId: ParsedEntityId, path: string) { const { kind, namespace, name } = entityId; diff --git a/plugins/techdocs/src/plugin.ts b/plugins/techdocs/src/plugin.ts index ee2cab60fa..576ba0a292 100644 --- a/plugins/techdocs/src/plugin.ts +++ b/plugins/techdocs/src/plugin.ts @@ -57,6 +57,7 @@ export const rootCatalogDocsRouteRef = createRouteRef({ title: 'Docs', }); +// TODO: Use discovery API for frontend to get URL for techdocs-backend instead of requestUrl export const plugin = createPlugin({ id: 'techdocs', apis: [ diff --git a/plugins/techdocs/src/types.ts b/plugins/techdocs/src/types.ts index ec907cc6fa..341d3f0c92 100644 --- a/plugins/techdocs/src/types.ts +++ b/plugins/techdocs/src/types.ts @@ -14,6 +14,12 @@ * limitations under the License. */ +/** + * To uniquely identify an entity in Backstage. + * @property {string} kind + * @property {string} namespace + * @property {string} name + */ export type ParsedEntityId = { kind: string; namespace?: string;