From 6853aac3e0fc3b374a1a2028ac02b7870e6d54ed Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 27 Aug 2024 12:04:02 +0200 Subject: [PATCH 01/15] Update url reader documentation Signed-off-by: Johan Haals --- docs/plugins/url-reader.md | 181 +++++++++++++++++++++++-------------- 1 file changed, 111 insertions(+), 70 deletions(-) diff --git a/docs/plugins/url-reader.md b/docs/plugins/url-reader.md index 9fab0482bb..3141fdb6ec 100644 --- a/docs/plugins/url-reader.md +++ b/docs/plugins/url-reader.md @@ -3,7 +3,7 @@ id: url-reader title: URL Reader sidebar_label: URL Reader # prettier-ignore -description: URL Reader is a backend core API responsible for reading files from external locations. +description: URL Reader is a backend core service responsible for reading files from external locations. --- ## Concept @@ -32,24 +32,7 @@ when trying to read files. ## Interface -When the Backstage backend starts, a new instance of URL Reader is created. You -can see this in the index file of your Backstage backend -i.e.`packages/backend/src/index.ts`. -[Example](https://github.com/backstage/backstage/blob/ebbe91dbe79038a61d35cf6ed2d96e0e0d5a15f3/packages/backend/src/index.ts#L57) - -```ts -// File: packages/backend/src/index.ts - -import { UrlReaders } from '@backstage/backend-common'; - -function makeCreateEnv(config: Config) { - // .... - const reader = UrlReaders.default({ logger: root, config }); - // -} -``` - -This instance contains all the default URL Reader providers +This service instance contains all the default URL Reader providers in the backend-defaults package including GitHub, GitLab, Bitbucket, Azure, Google GCS. As the need arises, more URL Readers are being written to support different providers. @@ -57,21 +40,68 @@ providers. The generic interface of a URL Reader instance looks like this. ```ts -export type UrlReader = { - /* Used to read a single file and return its content. */ - readUrl(url: string, options?: ReadUrlOptions): Promise; - /* Used to read a file tree and download as a directory. */ - readTree(url: string, options?: ReadTreeOptions): Promise; - /* Used to search a file in a tree. */ - search(url: string, options?: SearchOptions): Promise; -}; +export interface UrlReaderService { + /** + * Reads a single file and return its content. + */ + readUrl( + url: string, + options?: UrlReaderServiceReadUrlOptions, + ): Promise; + + /** + * Reads a full or partial file tree. + */ + readTree( + url: string, + options?: UrlReaderServiceReadTreeOptions, + ): Promise; + + /** + * Searches for a file in a tree using a glob pattern. + */ + search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise; +} ``` -## Using a URL Reader inside a plugin +## Using the URL Reader service inside a plugin + +The following example shows how to get the URL Reader service in your `example` backend plugin to read a file and a directory from a GitHub repository. + +```ts +import { + coreServices, + createBackendPlugin, +} from '@backstage/backend-plugin-api'; +import os from 'os'; + +createBackendPlugin({ + pluginId: 'example', + register(env) { + env.registerInit({ + deps: { + urlReader: coreServices.urlReader, + }, + async init({ urlReader }) { + const buffer = await urlReader + .read('https://github.com/backstage/backstage/blob/master/README.md') + .then(r => r.buffer()); + + const tmpDir = os.tmpdir(); + const directory = await urlReader + .readTree( + 'https://github.com/backstage/backstage/tree/master/packages/backend', + ) + .then(tree => tree.dir({ targetDir: tmpDir })); + }, + }); + }, +}); +``` -The `reader` instance is available in the backend plugin environment and passed -on to all the backend plugins. You can see an -[example](https://github.com/backstage/backstage/blob/b0be185369ebaad22255b7cdf18535d1d4ffd0e7/packages/backend/src/plugins/techdocs.ts#L31). When any of the methods on this instance is called with a URL, URL Reader extracts the host for that URL (e.g. `github.com`, `ghe.mycompany.com`, etc.). Using the @@ -81,27 +111,18 @@ package, it looks inside the config of the `app-config.yaml` to find out how to work with the host based on the configs provided like authentication token, API base URL, etc. -Make sure your plugin-specific backend file at -`packages/backend/src/plugins/.ts` is forwarding the `reader` instance -passed on as the `PluginEnvironment` to the actual plugin's `createRouter` -function. See how this is done in -[Catalog](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend/src/plugins/catalog.ts#L25-L27) -and -[TechDocs](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend/src/plugins/techdocs.ts#L31-L36) -backend plugins. - Once the reader instance is available inside the plugin, one of its methods can directly be used with a URL. Some example usages - -- [`readUrl`](https://github.com/backstage/backstage/blob/a7607b5/plugins/catalog-backend/src/modules/codeowners/lib/read.ts#L24-L33) - +- [`readUrl`](https://github.com/backstage/backstage/blob/38f3827/plugins/catalog-backend/src/modules/codeowners/lib/read.ts#L24-L34) - Catalog using the `readUrl` method to read the CODEOWNERS file in a repository. -- [`readTree`](https://github.com/backstage/backstage/blob/84a8788/plugins/techdocs-node/src/helpers.ts#L146-L167) - +- [`readTree`](https://github.com/backstage/backstage/blob/33ebb28/plugins/techdocs-node/src/helpers.ts#L147-L165) - TechDocs using the `readTree` method to download markdown files in order to generate the documentation site. -- [`readTree`](https://github.com/backstage/backstage/blob/cb4f0e4/plugins/techdocs-node/src/stages/prepare/url.ts#L51-L73) - +- [`readTree`](https://github.com/backstage/backstage/blob/33ebb28/plugins/techdocs-node/src/stages/prepare/url.ts#L55-L78) - TechDocs using `NotModifiedError` to maintain cache and speed up and limit the number of requests. -- [`search`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts#L88-L108) - +- [`search`](https://github.com/backstage/backstage/blob/38f3827/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts#L120-L144) - Catalog using the `search` method to find files for a location URL containing a glob pattern. @@ -122,7 +143,7 @@ to add a new URL Reader for any other provider, you are most welcome to contribute one! Feel free to use the -[GitHub URL Reader](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/GithubUrlReader.ts) +[GitHub URL Reader service](https://github.com/backstage/backstage/blob/ce2ca68f07ad3334401d3277b989bf145b728a64/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts#L60) as a source of inspiration. ### 1. Add an integration @@ -133,26 +154,26 @@ config file is supposed to be the place where a Backstage integrator defines the host URL for the integration, authentication details and other integration related configurations. -The `@backstage/integration` package is where most of the integration specific -code lives, so that it is shareable across Backstage. Functions like "read the -integrations config and process it", "construct headers for authenticated -requests to the host" or "convert a plain file URL into its API URL for -downloading the file" would live in this package. +The `@backstage/backend-defaults` package is where the URL Reader specific +code lives. Functions like "read the integrations config and process it", +"construct headers for authenticated requests to the host" or +"convert a plain file URL into its API URL for downloading the file" +would live in `@backstage/integrations` so that it is sharable across Backstage. ### 2. Create the URL Reader Create a new class which implements the -[`UrlReader` type](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/types.ts#L21-L28) -inside `@backstage/backend-common`. Create and export a static `factory` method +[`UrlReaderService` type](https://github.com/backstage/backstage/blob/c4b8169/packages/backend-plugin-api/src/services/definitions/UrlReaderService.ts#L26) +inside `@backstage/backend-defaults`. Create and export a static `factory` method which reads the integration config and returns a map of host URLs the new reader should be used for. See the -[GitHub URL Reader](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/GithubUrlReader.ts#L50-L63) +[GitHub URL Reader](https://github.com/backstage/backstage/blob/ce2ca68f07ad3334401d3277b989bf145b728a64/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts#L61-L73) for example. ### 3. Implement the methods We want to make sure all URL Readers behave in the same way. Hence if possible, -all the methods of the `UrlReader` interface should be implemented. However it +all the methods of the `UrlReaderService` interface should be implemented. However it is okay to start by implementing just one of them and create issues for the remaining. @@ -208,7 +229,7 @@ the tree and return the files matching the pattern in the `url`. There are two ways to make your new URL Reader available for use. You can choose to make it open source, by updating the -[`default` factory](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/UrlReaders.ts#L62-L81) +[`default` factory](https://github.com/backstage/backstage/blob/ce2ca68f07ad3334401d3277b989bf145b728a64/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts#L82-L102) method of URL Readers. But for something internal which you don't want to make open source, you can @@ -217,12 +238,23 @@ instance is created. ```ts // File: packages/backend/src/index.ts -const reader = UrlReaders.default({ - logger: root, - config, - // This is where your internal URL Readers would go. - factories: [myCustomReader.factory], +import { urlReaderFactoriesServiceRef } from '@backstage/backend-defaults/urlReader'; +import { + createBackendFeatureLoader, + createServiceFactory, +} from '@backstage/backend-plugin-api'; + +const customReader = createServiceFactory({ + service: urlReaderFactoriesServiceRef, + deps: {}, + async factory() { + return CustomUrlReader.factory; + }, }); + +const backend = createBackend(); +// backend.add() of other plugins and modules excluded +backend.add(customReader); ``` ### 5. Caching @@ -244,18 +276,27 @@ when the backend starts and call one of the methods with your debugging URL. ```ts // File: packages/backend/src/index.ts +import { + coreServices, + createBackendPlugin, +} from '@backstage/backend-plugin-api'; -async function main() { - // ... - const createEnv = makeCreateEnv(config); +const demoPlugin = createBackendPlugin({ + pluginId: 'demo', + register(env) { + env.registerInit({ + deps: { + urlReader: coreServices.urlReader, + }, + async init({ urlReader }) { + const resp = urlReader.read('http://my-url'); + console.log('RESPONSE', resp); + }); + }, +}); - const testReader = createEnv('test-url-reader').reader; - const response = await testReader.readUrl( - 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', - ); - console.log((await response.buffer()).toString()); - // ... -} +const backend = createBackend(); +backend.add(demoPlugin); ``` This will be run every time you restart the backend. Note that after any change From 1d06e061ab509eddce22b4bf4d6795b378bedd62 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 27 Aug 2024 14:30:01 +0200 Subject: [PATCH 02/15] Consolidate url reader docs Signed-off-by: Johan Haals --- .../core-services/url-reader.md | 96 ++++++ docs/plugins/url-reader.md | 306 ------------------ 2 files changed, 96 insertions(+), 306 deletions(-) delete mode 100644 docs/plugins/url-reader.md diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index b9616a029e..ae6c9770de 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -45,3 +45,99 @@ createBackendPlugin({ }, }); ``` + +## Providing custom URL readers + +Custom URL readers that are not intended for open sourcing via a custom service factory for that specific reader. The following example shows how to create a custom URL reader and provide it to the backend. + +```ts +// File: packages/backend/src/index.ts +import { urlReaderFactoriesServiceRef } from '@backstage/backend-defaults/urlReader'; +import { + createBackendFeatureLoader, + createServiceFactory, +} from '@backstage/backend-plugin-api'; + +const customReader = createServiceFactory({ + service: urlReaderFactoriesServiceRef, + deps: {}, + async factory() { + return CustomUrlReader.factory; + }, +}); + +const backend = createBackend(); +// backend.add() of other plugins and modules excluded +backend.add(customReader); +``` + +## Writing URL Readers + +We want to make sure all URL Readers behave in the same way. Hence if possible, +all the methods of the `UrlReaderService` interface should be implemented. However it +is okay to start by implementing just one of them and create issues for the +remaining. + +You can choose to make new URL Readers open source if the use case is beneficial to other users. Either as its own package or by updating the +[`default` factory](https://github.com/backstage/backstage/blob/ce2ca68f07ad3334401d3277b989bf145b728a64/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts#L82-L102) +method of URL Readers. It's recommended to create an issue in the Backstage repository to discuss the use case and get feedback before starting the implementation of a new core URL Reader. + +Here are some general guidelines for writing URL Readers + +#### `readUrl` + +`readUrl` method expects a user-friendly URL, something which can be copied from +the browser naturally when a person is browsing the provider in their browser. + +- ✅ Valid URL : + `https://github.com/backstage/backstage/blob/master/ADOPTERS.md` +- ❌ Not a valid URL : + `https://raw.githubusercontent.com/backstage/backstage/master/ADOPTERS.md` +- ❌ Not a valid URL : `https://github.com/backstage/backstage/ADOPTERS.md` + +Upon receiving the URL, `readUrl` converts the user-friendly URL into an API URL +which can be used to request the provider's API. + +`readUrl` then makes an authenticated request to the provider API and returns the response containing the file's contents and ETag(if the provider supports it). + +#### `readTree` + +`readTree` method also expects user-friendly URLs similar to `read` but the URL +should point to a tree (could be the root of a repository or even a +sub-directory). + +- ✅ Valid URL : `https://github.com/backstage/backstage` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/docs` + +Using the provider's API documentation, find out an API endpoint which can be +used to download either a zip or a tarball. You can download the entire tree +(e.g. a repository) and filter out in case the user is expecting only a +sub-tree. But some APIs are smart enough to accept a path and return only a +sub-tree in the downloaded archive. + +#### search + +`search` method expects a glob pattern of a URL and returns a list of files +matching the query. + +- ✅ Valid URL : + `https://github.com/backstage/backstage/blob/master/**/catalog-info.yaml` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/**/*.md` +- ✅ Valid URL : + `https://github.com/backstage/backstage/blob/master/*/package.json` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/READM` + +The core logic of `readTree` can be used here to extract all the files inside +the tree and return the files matching the pattern in the `url`. + +### Caching + +All of the methods above support an ETag based caching. If the method is called +without an `etag`, the response contains an ETag of the resource (should ideally +forward the ETag returned by the provider). If the method is called with an +`etag`, it first compares the ETag and returns a `NotModifiedError` in case the +resource has not been modified. This approach is very similar to the actual +[`ETag`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) and +[`If-None-Match`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match) +HTTP headers. diff --git a/docs/plugins/url-reader.md b/docs/plugins/url-reader.md deleted file mode 100644 index 3141fdb6ec..0000000000 --- a/docs/plugins/url-reader.md +++ /dev/null @@ -1,306 +0,0 @@ ---- -id: url-reader -title: URL Reader -sidebar_label: URL Reader -# prettier-ignore -description: URL Reader is a backend core service responsible for reading files from external locations. ---- - -## Concept - -Some of the core plugins of Backstage have to read files from an external -location. [Software Catalog](../features/software-catalog/index.md) has to read -the [`catalog-info.yaml`](../features/software-catalog/descriptor-format.md) -entity descriptor files to register and track an entity. -[Software Templates](../features/software-templates/index.md) have to download -the template skeleton files before creating a new component. -[TechDocs](../features/techdocs/README.md) has to download the markdown source -files before generating a documentation site. - -Since, the requirement for reading files is so essential for Backstage plugins, -the -[`coreServices.urlReader`](../reference/backend-plugin-api.coreservices.urlreader.md) -package provides a dedicated API for reading from such URL based remote -locations like GitHub, GitLab, Bitbucket, Google Cloud Storage, etc. This is -commonly referred to as "URL Reader". It takes care of making authenticated -requests to the remote host so that private files can be read securely. If users -have [GitHub App based authentication](../integrations/github/github-apps.md) set up, URL Reader even -refreshes the token, to avoid reaching the GitHub API rate limit. - -As a result, plugin authors do not have to worry about any of these problems -when trying to read files. - -## Interface - -This service instance contains all the default URL Reader providers -in the backend-defaults package including GitHub, GitLab, Bitbucket, Azure, Google -GCS. As the need arises, more URL Readers are being written to support different -providers. - -The generic interface of a URL Reader instance looks like this. - -```ts -export interface UrlReaderService { - /** - * Reads a single file and return its content. - */ - readUrl( - url: string, - options?: UrlReaderServiceReadUrlOptions, - ): Promise; - - /** - * Reads a full or partial file tree. - */ - readTree( - url: string, - options?: UrlReaderServiceReadTreeOptions, - ): Promise; - - /** - * Searches for a file in a tree using a glob pattern. - */ - search( - url: string, - options?: UrlReaderServiceSearchOptions, - ): Promise; -} -``` - -## Using the URL Reader service inside a plugin - -The following example shows how to get the URL Reader service in your `example` backend plugin to read a file and a directory from a GitHub repository. - -```ts -import { - coreServices, - createBackendPlugin, -} from '@backstage/backend-plugin-api'; -import os from 'os'; - -createBackendPlugin({ - pluginId: 'example', - register(env) { - env.registerInit({ - deps: { - urlReader: coreServices.urlReader, - }, - async init({ urlReader }) { - const buffer = await urlReader - .read('https://github.com/backstage/backstage/blob/master/README.md') - .then(r => r.buffer()); - - const tmpDir = os.tmpdir(); - const directory = await urlReader - .readTree( - 'https://github.com/backstage/backstage/tree/master/packages/backend', - ) - .then(tree => tree.dir({ targetDir: tmpDir })); - }, - }); - }, -}); -``` - -When any of the methods on this instance is called with a URL, URL Reader -extracts the host for that URL (e.g. `github.com`, `ghe.mycompany.com`, etc.). -Using the -[`@backstage/integration`](https://github.com/backstage/backstage/tree/master/packages/integration) -package, it looks inside the -[`integrations:`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/app-config.yaml#L134-L158) -config of the `app-config.yaml` to find out how to work with the host based on -the configs provided like authentication token, API base URL, etc. - -Once the reader instance is available inside the plugin, one of its methods can -directly be used with a URL. Some example usages - - -- [`readUrl`](https://github.com/backstage/backstage/blob/38f3827/plugins/catalog-backend/src/modules/codeowners/lib/read.ts#L24-L34) - - Catalog using the `readUrl` method to read the CODEOWNERS file in a repository. -- [`readTree`](https://github.com/backstage/backstage/blob/33ebb28/plugins/techdocs-node/src/helpers.ts#L147-L165) - - TechDocs using the `readTree` method to download markdown files in order to - generate the documentation site. -- [`readTree`](https://github.com/backstage/backstage/blob/33ebb28/plugins/techdocs-node/src/stages/prepare/url.ts#L55-L78) - - TechDocs using `NotModifiedError` to maintain cache and speed up and limit the - number of requests. -- [`search`](https://github.com/backstage/backstage/blob/38f3827/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts#L120-L144) - - Catalog using the `search` method to find files for a location URL containing - a glob pattern. - -Note that URL Readers which target git-based version control systems may, under -the hood, leverage the ability to create tar archives based on a specific git -commit-ish. A consequence of this is that files and directories configured with -[the `export-ignore` attribute](https://git-scm.com/docs/gitattributes#_creating_an_archive) -via `.gitattributes` will not be visible to the URL reader when using the -`readTree` or `search` methods. - -Be aware of this limitation and ensure that end-users of your plugin are also -aware via, for example, documentation. - -## Writing a new URL Reader - -If the available URL Readers are not sufficient for your use case and you want -to add a new URL Reader for any other provider, you are most welcome to -contribute one! - -Feel free to use the -[GitHub URL Reader service](https://github.com/backstage/backstage/blob/ce2ca68f07ad3334401d3277b989bf145b728a64/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts#L60) -as a source of inspiration. - -### 1. Add an integration - -The provider for your new URL Reader can also be called an "integration" in -Backstage. The `integrations:` section of your Backstage `app-config.yaml` -config file is supposed to be the place where a Backstage integrator defines the -host URL for the integration, authentication details and other integration -related configurations. - -The `@backstage/backend-defaults` package is where the URL Reader specific -code lives. Functions like "read the integrations config and process it", -"construct headers for authenticated requests to the host" or -"convert a plain file URL into its API URL for downloading the file" -would live in `@backstage/integrations` so that it is sharable across Backstage. - -### 2. Create the URL Reader - -Create a new class which implements the -[`UrlReaderService` type](https://github.com/backstage/backstage/blob/c4b8169/packages/backend-plugin-api/src/services/definitions/UrlReaderService.ts#L26) -inside `@backstage/backend-defaults`. Create and export a static `factory` method -which reads the integration config and returns a map of host URLs the new reader -should be used for. See the -[GitHub URL Reader](https://github.com/backstage/backstage/blob/ce2ca68f07ad3334401d3277b989bf145b728a64/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts#L61-L73) -for example. - -### 3. Implement the methods - -We want to make sure all URL Readers behave in the same way. Hence if possible, -all the methods of the `UrlReaderService` interface should be implemented. However it -is okay to start by implementing just one of them and create issues for the -remaining. - -#### `readUrl` - -`readUrl` method expects a user-friendly URL, something which can be copied from -the browser naturally when a person is browsing the provider in their browser. - -- ✅ Valid URL : - `https://github.com/backstage/backstage/blob/master/ADOPTERS.md` -- ❌ Not a valid URL : - `https://raw.githubusercontent.com/backstage/backstage/master/ADOPTERS.md` -- ❌ Not a valid URL : `https://github.com/backstage/backstage/ADOPTERS.md` - -Upon receiving the URL, `readUrl` converts the user-friendly URL into an API URL -which can be used to request the provider's API. - -`readUrl` then makes an authenticated request to the provider API and returns the response containing the file's contents and ETag(if the provider supports it). - -#### `readTree` - -`readTree` method also expects user-friendly URLs similar to `read` but the URL -should point to a tree (could be the root of a repository or even a -sub-directory). - -- ✅ Valid URL : `https://github.com/backstage/backstage` -- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master` -- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/docs` - -Using the provider's API documentation, find out an API endpoint which can be -used to download either a zip or a tarball. You can download the entire tree -(e.g. a repository) and filter out in case the user is expecting only a -sub-tree. But some APIs are smart enough to accept a path and return only a -sub-tree in the downloaded archive. - -#### search - -`search` method expects a glob pattern of a URL and returns a list of files -matching the query. - -- ✅ Valid URL : - `https://github.com/backstage/backstage/blob/master/**/catalog-info.yaml` -- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/**/*.md` -- ✅ Valid URL : - `https://github.com/backstage/backstage/blob/master/*/package.json` -- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/READM` - -The core logic of `readTree` can be used here to extract all the files inside -the tree and return the files matching the pattern in the `url`. - -### 4. Add to available URL Readers - -There are two ways to make your new URL Reader available for use. - -You can choose to make it open source, by updating the -[`default` factory](https://github.com/backstage/backstage/blob/ce2ca68f07ad3334401d3277b989bf145b728a64/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts#L82-L102) -method of URL Readers. - -But for something internal which you don't want to make open source, you can -update your `packages/backend/src/index.ts` file and update how the `reader` -instance is created. - -```ts -// File: packages/backend/src/index.ts -import { urlReaderFactoriesServiceRef } from '@backstage/backend-defaults/urlReader'; -import { - createBackendFeatureLoader, - createServiceFactory, -} from '@backstage/backend-plugin-api'; - -const customReader = createServiceFactory({ - service: urlReaderFactoriesServiceRef, - deps: {}, - async factory() { - return CustomUrlReader.factory; - }, -}); - -const backend = createBackend(); -// backend.add() of other plugins and modules excluded -backend.add(customReader); -``` - -### 5. Caching - -All of the methods above support an ETag based caching. If the method is called -without an `etag`, the response contains an ETag of the resource (should ideally -forward the ETag returned by the provider). If the method is called with an -`etag`, it first compares the ETag and returns a `NotModifiedError` in case the -resource has not been modified. This approach is very similar to the actual -[`ETag`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) and -[`If-None-Match`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match) -HTTP headers. - -### 6. Debugging - -When debugging one of the URL Readers, you can straightforward use the -[`reader` instance created](https://github.com/backstage/backstage/blob/ebbe91dbe79038a61d35cf6ed2d96e0e0d5a15f3/packages/backend/src/index.ts#L57) -when the backend starts and call one of the methods with your debugging URL. - -```ts -// File: packages/backend/src/index.ts -import { - coreServices, - createBackendPlugin, -} from '@backstage/backend-plugin-api'; - -const demoPlugin = createBackendPlugin({ - pluginId: 'demo', - register(env) { - env.registerInit({ - deps: { - urlReader: coreServices.urlReader, - }, - async init({ urlReader }) { - const resp = urlReader.read('http://my-url'); - console.log('RESPONSE', resp); - }); - }, -}); - -const backend = createBackend(); -backend.add(demoPlugin); -``` - -This will be run every time you restart the backend. Note that after any change -in the URL Reader code, you need to stop the backend and restart, since the -`reader` instance is memoized and does not update on hot module reloading. Also, -there are a lot of unit tests written for the URL Readers, which you can make -use of. From abb3fe2b44029da89ac6a6e9adc67f68185dbf2a Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 09:22:08 +0200 Subject: [PATCH 03/15] update sidebar, add redirect Signed-off-by: Johan Haals --- microsite/docusaurus.config.ts | 4 ++++ microsite/sidebars.json | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/microsite/docusaurus.config.ts b/microsite/docusaurus.config.ts index 67018c6e33..a6909f139f 100644 --- a/microsite/docusaurus.config.ts +++ b/microsite/docusaurus.config.ts @@ -195,6 +195,10 @@ const config: Config = { from: '/docs/local-dev/debugging/', to: '/docs/tooling/local-dev/debugging', }, + { + from: '/docs/plugins/url-reader/', + to: '/docs/backend-system/core-services/url-reader', + }, ], }, ], diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 79a43e165b..baca4f4db6 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -274,8 +274,7 @@ "items": [ "plugins/proxying", "plugins/backend-plugin", - "plugins/call-existing-api", - "plugins/url-reader" + "plugins/call-existing-api" ] }, { From 079abacea34f1e0dcce567cd620d908abe307196 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 09:37:15 +0200 Subject: [PATCH 04/15] remove urlReader from mkdocs Signed-off-by: Johan Haals --- mkdocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 33c9af3286..7bc56695b8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -138,7 +138,6 @@ nav: - Proxying: 'plugins/proxying.md' - Backend plugin: 'plugins/backend-plugin.md' - Call existing API: 'plugins/call-existing-api.md' - - URL Reader: 'plugins/url-reader.md' - Testing: - Testing with Jest: 'plugins/testing.md' - Publishing: From 1342aaf9bd7015736581403ca14ab00cf7cc914e Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 11:24:27 +0200 Subject: [PATCH 05/15] Update docs/backend-system/core-services/url-reader.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- docs/backend-system/core-services/url-reader.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index ae6c9770de..eb6564d744 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -50,8 +50,7 @@ createBackendPlugin({ Custom URL readers that are not intended for open sourcing via a custom service factory for that specific reader. The following example shows how to create a custom URL reader and provide it to the backend. -```ts -// File: packages/backend/src/index.ts +```ts title="packages/backend/src/index.ts" import { urlReaderFactoriesServiceRef } from '@backstage/backend-defaults/urlReader'; import { createBackendFeatureLoader, From 2e4d019f4c306d9b9ce4da3bcf3314ddbf8e8b77 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 11:24:38 +0200 Subject: [PATCH 06/15] Update docs/backend-system/core-services/url-reader.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- docs/backend-system/core-services/url-reader.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index eb6564d744..eaa47f8c6c 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -74,8 +74,8 @@ backend.add(customReader); We want to make sure all URL Readers behave in the same way. Hence if possible, all the methods of the `UrlReaderService` interface should be implemented. However it -is okay to start by implementing just one of them and create issues for the -remaining. +is okay to start by implementing just one of them and creating issues for the +remaining ones. You can choose to make new URL Readers open source if the use case is beneficial to other users. Either as its own package or by updating the [`default` factory](https://github.com/backstage/backstage/blob/ce2ca68f07ad3334401d3277b989bf145b728a64/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts#L82-L102) From c26c2cec2504c5154ccc84d36cceddc605184ca8 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 11:24:53 +0200 Subject: [PATCH 07/15] Update docs/backend-system/core-services/url-reader.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- docs/backend-system/core-services/url-reader.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index eaa47f8c6c..62d452fc98 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -125,7 +125,7 @@ matching the query. - ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/**/*.md` - ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/*/package.json` -- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/READM` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/README` The core logic of `readTree` can be used here to extract all the files inside the tree and return the files matching the pattern in the `url`. From 30e12c8388524f6ac2384ed774352b6e19131ca3 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 11:25:02 +0200 Subject: [PATCH 08/15] Update docs/backend-system/core-services/url-reader.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- docs/backend-system/core-services/url-reader.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index 62d452fc98..ca25dcda85 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -133,7 +133,7 @@ the tree and return the files matching the pattern in the `url`. ### Caching All of the methods above support an ETag based caching. If the method is called -without an `etag`, the response contains an ETag of the resource (should ideally +without an ETag, the response contains the ETag of the resource (should ideally forward the ETag returned by the provider). If the method is called with an `etag`, it first compares the ETag and returns a `NotModifiedError` in case the resource has not been modified. This approach is very similar to the actual From db0bcbadbb1ded6044d4dc9642eb28c92186c49c Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 11:25:14 +0200 Subject: [PATCH 09/15] Update docs/backend-system/core-services/url-reader.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- docs/backend-system/core-services/url-reader.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index ca25dcda85..84834b295c 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -135,7 +135,7 @@ the tree and return the files matching the pattern in the `url`. All of the methods above support an ETag based caching. If the method is called without an ETag, the response contains the ETag of the resource (should ideally forward the ETag returned by the provider). If the method is called with an -`etag`, it first compares the ETag and returns a `NotModifiedError` in case the +ETag, it first compares the ETag and returns a `NotModifiedError` in case the resource has not been modified. This approach is very similar to the actual [`ETag`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) and [`If-None-Match`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match) From 54c414609ae2caaaee32486fff151f7e277081ff Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 11:25:28 +0200 Subject: [PATCH 10/15] Update docs/backend-system/core-services/url-reader.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- docs/backend-system/core-services/url-reader.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index 84834b295c..cba5473e42 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -115,7 +115,7 @@ used to download either a zip or a tarball. You can download the entire tree sub-tree. But some APIs are smart enough to accept a path and return only a sub-tree in the downloaded archive. -#### search +#### `search` `search` method expects a glob pattern of a URL and returns a list of files matching the query. From 89866aef1ab2d7f37869a6f132810bdfe71cbee5 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 11:25:38 +0200 Subject: [PATCH 11/15] Update docs/backend-system/core-services/url-reader.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- docs/backend-system/core-services/url-reader.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index cba5473e42..11c6dbfa8e 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -132,7 +132,7 @@ the tree and return the files matching the pattern in the `url`. ### Caching -All of the methods above support an ETag based caching. If the method is called +All of the methods above support ETag based caching. If the method is called without an ETag, the response contains the ETag of the resource (should ideally forward the ETag returned by the provider). If the method is called with an ETag, it first compares the ETag and returns a `NotModifiedError` in case the From da24e673bfd52be51fee0cbc0ea2a542f855d003 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 11:25:50 +0200 Subject: [PATCH 12/15] Update docs/backend-system/core-services/url-reader.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Haals Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- docs/backend-system/core-services/url-reader.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index 11c6dbfa8e..9d7d7e7b1a 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -97,7 +97,7 @@ the browser naturally when a person is browsing the provider in their browser. Upon receiving the URL, `readUrl` converts the user-friendly URL into an API URL which can be used to request the provider's API. -`readUrl` then makes an authenticated request to the provider API and returns the response containing the file's contents and ETag(if the provider supports it). +`readUrl` then makes an authenticated request to the provider API and returns the response containing the file's contents and `ETag` (if the provider supports it). #### `readTree` From 76d1c4a1d3d67b864dad6ac4e85cfee9863a0356 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 13:26:51 +0200 Subject: [PATCH 13/15] extended docs on custom readers Signed-off-by: Johan Haals --- .../core-services/url-reader.md | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index 9d7d7e7b1a..6c2b8451ff 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -48,7 +48,7 @@ createBackendPlugin({ ## Providing custom URL readers -Custom URL readers that are not intended for open sourcing via a custom service factory for that specific reader. The following example shows how to create a custom URL reader and provide it to the backend. +You can also create an internal or bespoke reader and provide it to the backend using a service factory. The following example shows how to create a custom URL reader and provide it to the backend. ```ts title="packages/backend/src/index.ts" import { urlReaderFactoriesServiceRef } from '@backstage/backend-defaults/urlReader'; @@ -57,6 +57,24 @@ import { createServiceFactory, } from '@backstage/backend-plugin-api'; + +class CustomUrlReader implements UrlReaderService { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { + const integrations = ScmIntegrations.fromConfig(config); + const reader = new GithubUrlReader(integration, { treeResponseFactory }); + const predicate = (url: URL) => url.host === integration.config.host; + return { reader, predicate }; + }); + + constructor( + private readonly integration: BitbucketServerIntegration, + private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, + ) {} + + // implementations of read, readTree and search methods skipped for this example + }; + + const customReader = createServiceFactory({ service: urlReaderFactoriesServiceRef, deps: {}, From 26eeefed3ab70dbf2b7f8d2ce97edb791d9c230f Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 15:16:45 +0200 Subject: [PATCH 14/15] update custom url reader example Signed-off-by: Johan Haals --- .../core-services/url-reader.md | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index 6c2b8451ff..27b9658eda 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -51,29 +51,33 @@ createBackendPlugin({ You can also create an internal or bespoke reader and provide it to the backend using a service factory. The following example shows how to create a custom URL reader and provide it to the backend. ```ts title="packages/backend/src/index.ts" -import { urlReaderFactoriesServiceRef } from '@backstage/backend-defaults/urlReader'; +import { createBackend } from '@backstage/backend-defaults'; +import { + ReaderFactory, + urlReaderFactoriesServiceRef, +} from '@backstage/backend-defaults/urlReader'; import { - createBackendFeatureLoader, createServiceFactory, + UrlReaderService, } from '@backstage/backend-plugin-api'; +import { ScmIntegrations } from '@backstage/integration'; +type CustomIntegrationConfig = { + // custom config fields +}; class CustomUrlReader implements UrlReaderService { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); - const reader = new GithubUrlReader(integration, { treeResponseFactory }); - const predicate = (url: URL) => url.host === integration.config.host; - return { reader, predicate }; - }); - - constructor( - private readonly integration: BitbucketServerIntegration, - private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, - ) {} - - // implementations of read, readTree and search methods skipped for this example + const integrationsConfig = integrations.byHost('myCustomDomain') ?? {}; + const reader = new CustomUrlReader(integrationsConfig); + const predicate = (url: URL) => url.host === 'myCustomDomain'; + return [{ reader, predicate }]; }; + constructor(private readonly integrationConfig: CustomIntegrationConfig) {} + // implementations of read, readTree and search methods skipped for this example +} const customReader = createServiceFactory({ service: urlReaderFactoriesServiceRef, From d181a6fc50a4ce99022078e2024f93a2edca8494 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Aug 2024 15:23:22 +0200 Subject: [PATCH 15/15] simplify example Signed-off-by: Johan Haals --- docs/backend-system/core-services/url-reader.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/docs/backend-system/core-services/url-reader.md b/docs/backend-system/core-services/url-reader.md index 27b9658eda..242ffd2307 100644 --- a/docs/backend-system/core-services/url-reader.md +++ b/docs/backend-system/core-services/url-reader.md @@ -60,22 +60,16 @@ import { createServiceFactory, UrlReaderService, } from '@backstage/backend-plugin-api'; -import { ScmIntegrations } from '@backstage/integration'; - -type CustomIntegrationConfig = { - // custom config fields -}; +import { Config } from '@backstage/config'; class CustomUrlReader implements UrlReaderService { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { - const integrations = ScmIntegrations.fromConfig(config); - const integrationsConfig = integrations.byHost('myCustomDomain') ?? {}; - const reader = new CustomUrlReader(integrationsConfig); + const reader = new CustomUrlReader(config); const predicate = (url: URL) => url.host === 'myCustomDomain'; return [{ reader, predicate }]; }; - constructor(private readonly integrationConfig: CustomIntegrationConfig) {} + constructor(private readonly config: Config) {} // implementations of read, readTree and search methods skipped for this example }