diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index b9616a029e..242ffd2307 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -45,3 +45,114 @@ createBackendPlugin({ }, }); ``` + +## Providing custom URL readers + +You can also create an internal or bespoke reader and provide it to the backend using a service factory. The following example shows how to create a custom URL reader and provide it to the backend. + +```ts title="packages/backend/src/index.ts" +import { createBackend } from '@backstage/backend-defaults'; +import { + ReaderFactory, + urlReaderFactoriesServiceRef, +} from '@backstage/backend-defaults/urlReader'; +import { + createServiceFactory, + UrlReaderService, +} from '@backstage/backend-plugin-api'; +import { Config } from '@backstage/config'; + +class CustomUrlReader implements UrlReaderService { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { + const reader = new CustomUrlReader(config); + const predicate = (url: URL) => url.host === 'myCustomDomain'; + return [{ reader, predicate }]; + }; + + constructor(private readonly config: Config) {} + // implementations of read, readTree and search methods skipped for this example +} + +const customReader = createServiceFactory({ + service: urlReaderFactoriesServiceRef, + deps: {}, + async factory() { + return CustomUrlReader.factory; + }, +}); + +const backend = createBackend(); +// backend.add() of other plugins and modules excluded +backend.add(customReader); +``` + +## Writing URL Readers + +We want to make sure all URL Readers behave in the same way. Hence if possible, +all the methods of the `UrlReaderService` interface should be implemented. However it +is okay to start by implementing just one of them and creating issues for the +remaining ones. + +You can choose to make new URL Readers open source if the use case is beneficial to other users. Either as its own package or by updating the +[`default` factory](https://github.com/backstage/backstage/blob/ce2ca68f07ad3334401d3277b989bf145b728a64/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts#L82-L102) +method of URL Readers. It's recommended to create an issue in the Backstage repository to discuss the use case and get feedback before starting the implementation of a new core URL Reader. + +Here are some general guidelines for writing URL Readers + +#### `readUrl` + +`readUrl` method expects a user-friendly URL, something which can be copied from +the browser naturally when a person is browsing the provider in their browser. + +- ✅ Valid URL : + `https://github.com/backstage/backstage/blob/master/ADOPTERS.md` +- ❌ Not a valid URL : + `https://raw.githubusercontent.com/backstage/backstage/master/ADOPTERS.md` +- ❌ Not a valid URL : `https://github.com/backstage/backstage/ADOPTERS.md` + +Upon receiving the URL, `readUrl` converts the user-friendly URL into an API URL +which can be used to request the provider's API. + +`readUrl` then makes an authenticated request to the provider API and returns the response containing the file's contents and `ETag` (if the provider supports it). + +#### `readTree` + +`readTree` method also expects user-friendly URLs similar to `read` but the URL +should point to a tree (could be the root of a repository or even a +sub-directory). + +- ✅ Valid URL : `https://github.com/backstage/backstage` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/docs` + +Using the provider's API documentation, find out an API endpoint which can be +used to download either a zip or a tarball. You can download the entire tree +(e.g. a repository) and filter out in case the user is expecting only a +sub-tree. But some APIs are smart enough to accept a path and return only a +sub-tree in the downloaded archive. + +#### `search` + +`search` method expects a glob pattern of a URL and returns a list of files +matching the query. + +- ✅ Valid URL : + `https://github.com/backstage/backstage/blob/master/**/catalog-info.yaml` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/**/*.md` +- ✅ Valid URL : + `https://github.com/backstage/backstage/blob/master/*/package.json` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/README` + +The core logic of `readTree` can be used here to extract all the files inside +the tree and return the files matching the pattern in the `url`. + +### Caching + +All of the methods above support ETag based caching. If the method is called +without an ETag, the response contains the ETag of the resource (should ideally +forward the ETag returned by the provider). If the method is called with an +ETag, it first compares the ETag and returns a `NotModifiedError` in case the +resource has not been modified. This approach is very similar to the actual +[`ETag`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) and +[`If-None-Match`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match) +HTTP headers. diff --git a/docs/plugins/url-reader.md b/docs/plugins/url-reader.md deleted file mode 100644 index 9fab0482bb..0000000000 --- a/docs/plugins/url-reader.md +++ /dev/null @@ -1,265 +0,0 @@ ---- -id: url-reader -title: URL Reader -sidebar_label: URL Reader -# prettier-ignore -description: URL Reader is a backend core API responsible for reading files from external locations. ---- - -## Concept - -Some of the core plugins of Backstage have to read files from an external -location. [Software Catalog](../features/software-catalog/index.md) has to read -the [`catalog-info.yaml`](../features/software-catalog/descriptor-format.md) -entity descriptor files to register and track an entity. -[Software Templates](../features/software-templates/index.md) have to download -the template skeleton files before creating a new component. -[TechDocs](../features/techdocs/README.md) has to download the markdown source -files before generating a documentation site. - -Since, the requirement for reading files is so essential for Backstage plugins, -the -[`coreServices.urlReader`](../reference/backend-plugin-api.coreservices.urlreader.md) -package provides a dedicated API for reading from such URL based remote -locations like GitHub, GitLab, Bitbucket, Google Cloud Storage, etc. This is -commonly referred to as "URL Reader". It takes care of making authenticated -requests to the remote host so that private files can be read securely. If users -have [GitHub App based authentication](../integrations/github/github-apps.md) set up, URL Reader even -refreshes the token, to avoid reaching the GitHub API rate limit. - -As a result, plugin authors do not have to worry about any of these problems -when trying to read files. - -## Interface - -When the Backstage backend starts, a new instance of URL Reader is created. You -can see this in the index file of your Backstage backend -i.e.`packages/backend/src/index.ts`. -[Example](https://github.com/backstage/backstage/blob/ebbe91dbe79038a61d35cf6ed2d96e0e0d5a15f3/packages/backend/src/index.ts#L57) - -```ts -// File: packages/backend/src/index.ts - -import { UrlReaders } from '@backstage/backend-common'; - -function makeCreateEnv(config: Config) { - // .... - const reader = UrlReaders.default({ logger: root, config }); - // -} -``` - -This instance contains all the default URL Reader providers -in the backend-defaults package including GitHub, GitLab, Bitbucket, Azure, Google -GCS. As the need arises, more URL Readers are being written to support different -providers. - -The generic interface of a URL Reader instance looks like this. - -```ts -export type UrlReader = { - /* Used to read a single file and return its content. */ - readUrl(url: string, options?: ReadUrlOptions): Promise; - /* Used to read a file tree and download as a directory. */ - readTree(url: string, options?: ReadTreeOptions): Promise; - /* Used to search a file in a tree. */ - search(url: string, options?: SearchOptions): Promise; -}; -``` - -## Using a URL Reader inside a plugin - -The `reader` instance is available in the backend plugin environment and passed -on to all the backend plugins. You can see an -[example](https://github.com/backstage/backstage/blob/b0be185369ebaad22255b7cdf18535d1d4ffd0e7/packages/backend/src/plugins/techdocs.ts#L31). -When any of the methods on this instance is called with a URL, URL Reader -extracts the host for that URL (e.g. `github.com`, `ghe.mycompany.com`, etc.). -Using the -[`@backstage/integration`](https://github.com/backstage/backstage/tree/master/packages/integration) -package, it looks inside the -[`integrations:`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/app-config.yaml#L134-L158) -config of the `app-config.yaml` to find out how to work with the host based on -the configs provided like authentication token, API base URL, etc. - -Make sure your plugin-specific backend file at -`packages/backend/src/plugins/.ts` is forwarding the `reader` instance -passed on as the `PluginEnvironment` to the actual plugin's `createRouter` -function. See how this is done in -[Catalog](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend/src/plugins/catalog.ts#L25-L27) -and -[TechDocs](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend/src/plugins/techdocs.ts#L31-L36) -backend plugins. - -Once the reader instance is available inside the plugin, one of its methods can -directly be used with a URL. Some example usages - - -- [`readUrl`](https://github.com/backstage/backstage/blob/a7607b5/plugins/catalog-backend/src/modules/codeowners/lib/read.ts#L24-L33) - - Catalog using the `readUrl` method to read the CODEOWNERS file in a repository. -- [`readTree`](https://github.com/backstage/backstage/blob/84a8788/plugins/techdocs-node/src/helpers.ts#L146-L167) - - TechDocs using the `readTree` method to download markdown files in order to - generate the documentation site. -- [`readTree`](https://github.com/backstage/backstage/blob/cb4f0e4/plugins/techdocs-node/src/stages/prepare/url.ts#L51-L73) - - TechDocs using `NotModifiedError` to maintain cache and speed up and limit the - number of requests. -- [`search`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts#L88-L108) - - Catalog using the `search` method to find files for a location URL containing - a glob pattern. - -Note that URL Readers which target git-based version control systems may, under -the hood, leverage the ability to create tar archives based on a specific git -commit-ish. A consequence of this is that files and directories configured with -[the `export-ignore` attribute](https://git-scm.com/docs/gitattributes#_creating_an_archive) -via `.gitattributes` will not be visible to the URL reader when using the -`readTree` or `search` methods. - -Be aware of this limitation and ensure that end-users of your plugin are also -aware via, for example, documentation. - -## Writing a new URL Reader - -If the available URL Readers are not sufficient for your use case and you want -to add a new URL Reader for any other provider, you are most welcome to -contribute one! - -Feel free to use the -[GitHub URL Reader](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/GithubUrlReader.ts) -as a source of inspiration. - -### 1. Add an integration - -The provider for your new URL Reader can also be called an "integration" in -Backstage. The `integrations:` section of your Backstage `app-config.yaml` -config file is supposed to be the place where a Backstage integrator defines the -host URL for the integration, authentication details and other integration -related configurations. - -The `@backstage/integration` package is where most of the integration specific -code lives, so that it is shareable across Backstage. Functions like "read the -integrations config and process it", "construct headers for authenticated -requests to the host" or "convert a plain file URL into its API URL for -downloading the file" would live in this package. - -### 2. Create the URL Reader - -Create a new class which implements the -[`UrlReader` type](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/types.ts#L21-L28) -inside `@backstage/backend-common`. Create and export a static `factory` method -which reads the integration config and returns a map of host URLs the new reader -should be used for. See the -[GitHub URL Reader](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/GithubUrlReader.ts#L50-L63) -for example. - -### 3. Implement the methods - -We want to make sure all URL Readers behave in the same way. Hence if possible, -all the methods of the `UrlReader` interface should be implemented. However it -is okay to start by implementing just one of them and create issues for the -remaining. - -#### `readUrl` - -`readUrl` method expects a user-friendly URL, something which can be copied from -the browser naturally when a person is browsing the provider in their browser. - -- ✅ Valid URL : - `https://github.com/backstage/backstage/blob/master/ADOPTERS.md` -- ❌ Not a valid URL : - `https://raw.githubusercontent.com/backstage/backstage/master/ADOPTERS.md` -- ❌ Not a valid URL : `https://github.com/backstage/backstage/ADOPTERS.md` - -Upon receiving the URL, `readUrl` converts the user-friendly URL into an API URL -which can be used to request the provider's API. - -`readUrl` then makes an authenticated request to the provider API and returns the response containing the file's contents and ETag(if the provider supports it). - -#### `readTree` - -`readTree` method also expects user-friendly URLs similar to `read` but the URL -should point to a tree (could be the root of a repository or even a -sub-directory). - -- ✅ Valid URL : `https://github.com/backstage/backstage` -- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master` -- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/docs` - -Using the provider's API documentation, find out an API endpoint which can be -used to download either a zip or a tarball. You can download the entire tree -(e.g. a repository) and filter out in case the user is expecting only a -sub-tree. But some APIs are smart enough to accept a path and return only a -sub-tree in the downloaded archive. - -#### search - -`search` method expects a glob pattern of a URL and returns a list of files -matching the query. - -- ✅ Valid URL : - `https://github.com/backstage/backstage/blob/master/**/catalog-info.yaml` -- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/**/*.md` -- ✅ Valid URL : - `https://github.com/backstage/backstage/blob/master/*/package.json` -- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/READM` - -The core logic of `readTree` can be used here to extract all the files inside -the tree and return the files matching the pattern in the `url`. - -### 4. Add to available URL Readers - -There are two ways to make your new URL Reader available for use. - -You can choose to make it open source, by updating the -[`default` factory](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/UrlReaders.ts#L62-L81) -method of URL Readers. - -But for something internal which you don't want to make open source, you can -update your `packages/backend/src/index.ts` file and update how the `reader` -instance is created. - -```ts -// File: packages/backend/src/index.ts -const reader = UrlReaders.default({ - logger: root, - config, - // This is where your internal URL Readers would go. - factories: [myCustomReader.factory], -}); -``` - -### 5. Caching - -All of the methods above support an ETag based caching. If the method is called -without an `etag`, the response contains an ETag of the resource (should ideally -forward the ETag returned by the provider). If the method is called with an -`etag`, it first compares the ETag and returns a `NotModifiedError` in case the -resource has not been modified. This approach is very similar to the actual -[`ETag`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) and -[`If-None-Match`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match) -HTTP headers. - -### 6. Debugging - -When debugging one of the URL Readers, you can straightforward use the -[`reader` instance created](https://github.com/backstage/backstage/blob/ebbe91dbe79038a61d35cf6ed2d96e0e0d5a15f3/packages/backend/src/index.ts#L57) -when the backend starts and call one of the methods with your debugging URL. - -```ts -// File: packages/backend/src/index.ts - -async function main() { - // ... - const createEnv = makeCreateEnv(config); - - const testReader = createEnv('test-url-reader').reader; - const response = await testReader.readUrl( - 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', - ); - console.log((await response.buffer()).toString()); - // ... -} -``` - -This will be run every time you restart the backend. Note that after any change -in the URL Reader code, you need to stop the backend and restart, since the -`reader` instance is memoized and does not update on hot module reloading. Also, -there are a lot of unit tests written for the URL Readers, which you can make -use of. diff --git a/microsite/docusaurus.config.ts b/microsite/docusaurus.config.ts index 67018c6e33..a6909f139f 100644 --- a/microsite/docusaurus.config.ts +++ b/microsite/docusaurus.config.ts @@ -195,6 +195,10 @@ const config: Config = { from: '/docs/local-dev/debugging/', to: '/docs/tooling/local-dev/debugging', }, + { + from: '/docs/plugins/url-reader/', + to: '/docs/backend-system/core-services/url-reader', + }, ], }, ], diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 79a43e165b..baca4f4db6 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -274,8 +274,7 @@ "items": [ "plugins/proxying", "plugins/backend-plugin", - "plugins/call-existing-api", - "plugins/url-reader" + "plugins/call-existing-api" ] }, { diff --git a/mkdocs.yml b/mkdocs.yml index 33c9af3286..7bc56695b8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -138,7 +138,6 @@ nav: - Proxying: 'plugins/proxying.md' - Backend plugin: 'plugins/backend-plugin.md' - Call existing API: 'plugins/call-existing-api.md' - - URL Reader: 'plugins/url-reader.md' - Testing: - Testing with Jest: 'plugins/testing.md' - Publishing: