diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 784f3295c6..abb45b0fe7 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -33,9 +33,19 @@ import { MergeResult } from 'isomorphic-git'; import { PushResult } from 'isomorphic-git'; import { Readable } from 'stream'; import { ReadCommitResult } from 'isomorphic-git'; +import { ReadTreeOptions } from '@backstage/backend-plugin-api'; +import { ReadTreeResponse } from '@backstage/backend-plugin-api'; +import { ReadTreeResponseDirOptions } from '@backstage/backend-plugin-api'; +import { ReadTreeResponseFile } from '@backstage/backend-plugin-api'; +import { ReadUrlOptions } from '@backstage/backend-plugin-api'; +import { ReadUrlResponse } from '@backstage/backend-plugin-api'; import { RequestHandler } from 'express'; import { Router } from 'express'; +import { SearchOptions } from '@backstage/backend-plugin-api'; +import { SearchResponse } from '@backstage/backend-plugin-api'; +import { SearchResponseFile } from '@backstage/backend-plugin-api'; import { Server } from 'http'; +import { UrlReaderService as UrlReader } from '@backstage/backend-plugin-api'; import { V1PodTemplateSpec } from '@kubernetes/client-node'; import * as winston from 'winston'; import { Writable } from 'stream'; @@ -533,30 +543,11 @@ export type ReaderFactory = (options: { treeResponseFactory: ReadTreeResponseFactory; }) => UrlReaderPredicateTuple[]; -// @public -export type ReadTreeOptions = { - filter?( - path: string, - info?: { - size: number; - }, - ): boolean; - etag?: string; - signal?: AbortSignal; -}; +export { ReadTreeOptions }; -// @public -export type ReadTreeResponse = { - files(): Promise; - archive(): Promise; - dir(options?: ReadTreeResponseDirOptions): Promise; - etag: string; -}; +export { ReadTreeResponse }; -// @public -export type ReadTreeResponseDirOptions = { - targetDir?: string; -}; +export { ReadTreeResponseDirOptions }; // @public export interface ReadTreeResponseFactory { @@ -587,24 +578,11 @@ export type ReadTreeResponseFactoryOptions = { ) => boolean; }; -// @public -export type ReadTreeResponseFile = { - path: string; - content(): Promise; -}; +export { ReadTreeResponseFile }; -// @public -export type ReadUrlOptions = { - etag?: string; - signal?: AbortSignal; -}; +export { ReadUrlOptions }; -// @public -export type ReadUrlResponse = { - buffer(): Promise; - stream?(): Readable; - etag?: string; -}; +export { ReadUrlResponse }; // @public export class ReadUrlResponseFactory { @@ -652,23 +630,11 @@ export type RunContainerOptions = { pullImage?: boolean; }; -// @public -export type SearchOptions = { - etag?: string; - signal?: AbortSignal; -}; +export { SearchOptions }; -// @public -export type SearchResponse = { - files: SearchResponseFile[]; - etag: string; -}; +export { SearchResponse }; -// @public -export type SearchResponseFile = { - url: string; - content(): Promise; -}; +export { SearchResponseFile }; // @public export class ServerTokenManager implements TokenManager { @@ -755,13 +721,7 @@ export interface TokenManager { }>; } -// @public -export type UrlReader = { - read(url: string): Promise; - readUrl(url: string, options?: ReadUrlOptions): Promise; - readTree(url: string, options?: ReadTreeOptions): Promise; - search(url: string, options?: SearchOptions): Promise; -}; +export { UrlReader }; // @public export type UrlReaderPredicateTuple = { diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 0af96cc41e..ca9b652fd0 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -34,6 +34,7 @@ "test:kubernetes": "backstage-cli package test -t KubernetesContainerRunner --no-watch" }, "dependencies": { + "@backstage/backend-plugin-api": "workspace:^", "@backstage/cli-common": "workspace:^", "@backstage/config": "workspace:^", "@backstage/config-loader": "workspace:^", diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index b9fc58ef5c..dd7a3da3d4 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -117,7 +117,7 @@ export function parseUrl( } /** - * Implements a {@link UrlReader} for AWS S3 buckets. + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for AWS S3 buckets. * * @public */ diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index 140496c181..08e3917b38 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -40,7 +40,7 @@ import { import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; /** - * Implements a {@link UrlReader} for Azure repos. + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for Azure repos. * * @public */ diff --git a/packages/backend-common/src/reading/BitbucketCloudUrlReader.ts b/packages/backend-common/src/reading/BitbucketCloudUrlReader.ts index 7bc4cd9c11..b83e012423 100644 --- a/packages/backend-common/src/reading/BitbucketCloudUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketCloudUrlReader.ts @@ -42,7 +42,7 @@ import { import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; /** - * Implements a {@link UrlReader} for files from Bitbucket Cloud. + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Cloud. * * @public */ diff --git a/packages/backend-common/src/reading/BitbucketServerUrlReader.ts b/packages/backend-common/src/reading/BitbucketServerUrlReader.ts index bcab226b0f..a97c3b39bb 100644 --- a/packages/backend-common/src/reading/BitbucketServerUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketServerUrlReader.ts @@ -41,7 +41,7 @@ import { import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; /** - * Implements a {@link UrlReader} for files from Bitbucket Server APIs. + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Server APIs. * * @public */ diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 805697bf27..2b0325d644 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -43,7 +43,7 @@ import { import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; /** - * Implements a {@link UrlReader} for files from Bitbucket v1 and v2 APIs, such + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket v1 and v2 APIs, such * as the one exposed by Bitbucket Cloud itself. * * @public diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts index d38cc510b1..20b78d63c3 100644 --- a/packages/backend-common/src/reading/FetchUrlReader.ts +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -63,7 +63,7 @@ const parsePortPredicate = (port: string | undefined) => { }; /** - * A {@link UrlReader} that does a plain fetch of the URL. + * A {@link @backstage/backend-plugin-api#UrlReaderService} that does a plain fetch of the URL. * * @public */ diff --git a/packages/backend-common/src/reading/GerritUrlReader.ts b/packages/backend-common/src/reading/GerritUrlReader.ts index 59b33bb2cb..5d86e5d9ac 100644 --- a/packages/backend-common/src/reading/GerritUrlReader.ts +++ b/packages/backend-common/src/reading/GerritUrlReader.ts @@ -52,7 +52,7 @@ const createTemporaryDirectory = async (workDir: string): Promise => await fs.mkdtemp(joinPath(workDir, '/gerrit-clone-')); /** - * Implements a {@link UrlReader} for files in Gerrit. + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files in Gerrit. * * @remarks * To be able to link to Git contents for Gerrit providers in a user friendly diff --git a/packages/backend-common/src/reading/GiteaUrlReader.ts b/packages/backend-common/src/reading/GiteaUrlReader.ts index da6b791949..1471bab608 100644 --- a/packages/backend-common/src/reading/GiteaUrlReader.ts +++ b/packages/backend-common/src/reading/GiteaUrlReader.ts @@ -36,7 +36,7 @@ import { import { Readable } from 'stream'; /** - * Implements a {@link UrlReader} for the Gitea v1 api. + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for the Gitea v1 api. * * @public */ diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 7abf1b2518..d91c35621d 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -51,7 +51,7 @@ export type GhBlobResponse = RestEndpointMethodTypes['git']['getBlob']['response']['data']; /** - * Implements a {@link UrlReader} for files through the GitHub v3 APIs, such as + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files through the GitHub v3 APIs, such as * the one exposed by GitHub itself. * * @public diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index c80f1b566c..bebc7453d0 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -42,7 +42,7 @@ import { trimEnd, trimStart } from 'lodash'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; /** - * Implements a {@link UrlReader} for files on GitLab. + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files on GitLab. * * @public */ diff --git a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts index 56b15a60e7..c9f5a58895 100644 --- a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts +++ b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts @@ -51,7 +51,7 @@ const parseURL = ( }; /** - * Implements a {@link UrlReader} for files on Google GCS. + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files on Google GCS. * * @public */ diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.test.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.test.ts index 38473557e0..89dac93fff 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.test.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.test.ts @@ -70,7 +70,6 @@ describe('UrlReaderPredicateMux', () => { mux.register({ predicate: url => url.hostname === 'foo', reader: { - read: jest.fn(), readUrl: jest.fn(), readTree: jest.fn(), search: jest.fn(), diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index d5d30605b2..a7c2e5f983 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -45,18 +45,6 @@ export class UrlReaderPredicateMux implements UrlReader { this.readers.push(tuple); } - async read(url: string): Promise { - const parsed = new URL(url); - - for (const { predicate, reader } of this.readers) { - if (predicate(parsed)) { - return reader.read(url); - } - } - - throw new NotAllowedError(notAllowedMessage(url)); - } - async readUrl( url: string, options?: ReadUrlOptions, diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 9fb403e7e9..a567eef412 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -32,7 +32,7 @@ import { AwsS3UrlReader } from './AwsS3UrlReader'; import { GiteaUrlReader } from './GiteaUrlReader'; /** - * Creation options for {@link UrlReaders}. + * Creation options for {@link @backstage/backend-plugin-api#UrlReaderService}. * * @public */ @@ -46,13 +46,13 @@ export type UrlReadersOptions = { }; /** - * Helps construct {@link UrlReader}s. + * Helps construct {@link @backstage/backend-plugin-api#UrlReaderService}s. * * @public */ export class UrlReaders { /** - * Creates a custom {@link UrlReader} wrapper for your own set of factories. + * Creates a custom {@link @backstage/backend-plugin-api#UrlReaderService} wrapper for your own set of factories. */ static create(options: UrlReadersOptions): UrlReader { const { logger, config, factories } = options; @@ -73,7 +73,7 @@ export class UrlReaders { } /** - * Creates a {@link UrlReader} wrapper that includes all the default factories + * Creates a {@link @backstage/backend-plugin-api#UrlReaderService} wrapper that includes all the default factories * from this package. * * Any additional factories passed will be loaded before the default ones. diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index bce737f3c7..640b549532 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -17,56 +17,38 @@ import { Readable } from 'stream'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; +import { + UrlReaderService, + ReadTreeResponse, +} from '@backstage/backend-plugin-api'; + +export type { + UrlReaderService as UrlReader, + ReadTreeOptions, + ReadTreeResponse, + ReadTreeResponseDirOptions, + ReadTreeResponseFile, + ReadUrlResponse, + ReadUrlOptions, + SearchOptions, + SearchResponse, + SearchResponseFile, +} from '@backstage/backend-plugin-api'; /** - * A generic interface for fetching plain data from URLs. - * - * @public - */ -export type UrlReader = { - /** - * Reads a single file and return its content. - * @deprecated use readUrl instead. - */ - read(url: string): Promise; - - /** - * Reads a single file and return its content. - * - * @remarks - * - * This is a replacement for the read method that supports options and - * complex responses. - * - * Use this as the read method will be removed in a future release. - */ - readUrl(url: string, options?: ReadUrlOptions): Promise; - - /** - * Reads a full or partial file tree. - */ - readTree(url: string, options?: ReadTreeOptions): Promise; - - /** - * Searches for a file in a tree using a glob pattern. - */ - search(url: string, options?: SearchOptions): Promise; -}; - -/** - * A predicate that decides whether a specific {@link UrlReader} can handle a + * A predicate that decides whether a specific {@link @backstage/backend-plugin-api#UrlReaderService} can handle a * given URL. * * @public */ export type UrlReaderPredicateTuple = { predicate: (url: URL) => boolean; - reader: UrlReader; + reader: UrlReaderService; }; /** * A factory function that can read config to construct zero or more - * {@link UrlReader}s along with a predicate for when it should be used. + * {@link @backstage/backend-plugin-api#UrlReaderService}s along with a predicate for when it should be used. * * @public */ @@ -76,72 +58,6 @@ export type ReaderFactory = (options: { treeResponseFactory: ReadTreeResponseFactory; }) => UrlReaderPredicateTuple[]; -/** - * An options object for readUrl operations. - * - * @public - */ -export type ReadUrlOptions = { - /** - * An ETag which can be provided to check whether a - * {@link UrlReader.readUrl} response has changed from a previous execution. - * - * @remarks - * - * In the {@link UrlReader.readUrl} response, an ETag is returned along with - * the data. The ETag is a unique identifier of the data, usually the commit - * SHA or ETag from the target. - * - * When an ETag is given in ReadUrlOptions, {@link UrlReader.readUrl} will - * first compare the ETag against the ETag of the target. If they match, - * {@link UrlReader.readUrl} will throw a - * {@link @backstage/errors#NotModifiedError} indicating that the response - * will not differ from the previous response which included this particular - * ETag. If they do not match, {@link UrlReader.readUrl} will return the rest - * of the response along with a new ETag. - */ - etag?: string; - - /** - * An abort signal to pass down to the underlying request. - * - * @remarks - * - * Not all reader implementations may take this field into account. - */ - signal?: AbortSignal; -}; - -/** - * A response object for {@link UrlReader.readUrl} operations. - * - * @public - */ -export type ReadUrlResponse = { - /** - * Returns the data that was read from the remote URL. - */ - buffer(): Promise; - - /** - * Returns the data that was read from the remote URL as a Readable stream. - * - * @remarks - * - * This method will be required in a future release. - */ - stream?(): Readable; - - /** - * Etag returned by content provider. - * - * @remarks - * - * Can be used to compare and cache responses when doing subsequent calls. - */ - etag?: string; -}; - /** * An options object for {@link ReadUrlResponseFactory} factory methods. * @@ -151,119 +67,6 @@ export type ReadUrlResponseFactoryFromStreamOptions = { etag?: string; }; -/** - * An options object for {@link UrlReader.readTree} operations. - * - * @public - */ -export type ReadTreeOptions = { - /** - * A filter that can be used to select which files should be included. - * - * @remarks - * - * The path passed to the filter function is the relative path from the URL - * that the file tree is fetched from, without any leading '/'. - * - * For example, given the URL https://github.com/my/repo/tree/master/my-dir, a file - * at https://github.com/my/repo/blob/master/my-dir/my-subdir/my-file.txt will - * be represented as my-subdir/my-file.txt - * - * If no filter is provided, all files are extracted. - */ - filter?(path: string, info?: { size: number }): boolean; - - /** - * An ETag which can be provided to check whether a - * {@link UrlReader.readTree} response has changed from a previous execution. - * - * @remarks - * - * In the {@link UrlReader.readTree} response, an ETag is returned along with - * the tree blob. The ETag is a unique identifier of the tree blob, usually - * the commit SHA or ETag from the target. - * - * When an ETag is given as a request option, {@link UrlReader.readTree} will - * first compare the ETag against the ETag on the target branch. If they - * match, {@link UrlReader.readTree} will throw a - * {@link @backstage/errors#NotModifiedError} indicating that the response - * will not differ from the previous response which included this particular - * ETag. If they do not match, {@link UrlReader.readTree} will return the - * rest of the response along with a new ETag. - */ - etag?: string; - - /** - * An abort signal to pass down to the underlying request. - * - * @remarks - * - * Not all reader implementations may take this field into account. - */ - signal?: AbortSignal; -}; - -/** - * Options that control {@link ReadTreeResponse.dir} execution. - * - * @public - */ -export type ReadTreeResponseDirOptions = { - /** - * The directory to write files to. - * - * @remarks - * - * Defaults to the OS tmpdir, or `backend.workingDirectory` if set in config. - */ - targetDir?: string; -}; - -/** - * A response object for {@link UrlReader.readTree} operations. - * - * @public - */ -export type ReadTreeResponse = { - /** - * Returns an array of all the files inside the tree, and corresponding - * functions to read their content. - */ - files(): Promise; - - /** - * Returns the tree contents as a binary archive, using a stream. - */ - archive(): Promise; - - /** - * Extracts the tree response into a directory and returns the path of the - * directory. - * - * **NOTE**: It is the responsibility of the caller to remove the directory after use. - */ - dir(options?: ReadTreeResponseDirOptions): Promise; - - /** - * Etag returned by content provider. - * - * @remarks - * - * Can be used to compare and cache responses when doing subsequent calls. - */ - etag: string; -}; - -/** - * Represents a single file in a {@link UrlReader.readTree} response. - * - * @public - */ -export type ReadTreeResponseFile = { - path: string; - content(): Promise; -}; - /** * Options that control execution of {@link ReadTreeResponseFactory} methods. * @@ -315,66 +118,3 @@ export interface ReadTreeResponseFactory { options: FromReadableArrayOptions, ): Promise; } - -/** - * An options object for search operations. - * - * @public - */ -export type SearchOptions = { - /** - * An etag can be provided to check whether the search response has changed from a previous execution. - * - * In the search() response, an etag is returned along with the files. The etag is a unique identifier - * of the current tree, usually the commit SHA or etag from the target. - * - * When an etag is given in SearchOptions, search will first compare the etag against the etag - * on the target branch. If they match, search will throw a NotModifiedError indicating that the search - * response will not differ from the previous response which included this particular etag. If they mismatch, - * search will return the rest of SearchResponse along with a new etag. - */ - etag?: string; - - /** - * An abort signal to pass down to the underlying request. - * - * @remarks - * - * Not all reader implementations may take this field into account. - */ - signal?: AbortSignal; -}; - -/** - * The output of a search operation. - * - * @public - */ -export type SearchResponse = { - /** - * The files that matched the search query. - */ - files: SearchResponseFile[]; - - /** - * A unique identifier of the current remote tree, usually the commit SHA or etag from the target. - */ - etag: string; -}; - -/** - * Represents a single file in a search response. - * - * @public - */ -export type SearchResponseFile = { - /** - * The full URL to the file. - */ - url: string; - - /** - * The binary contents of the file. - */ - content(): Promise; -}; diff --git a/packages/backend-plugin-api/api-report.md b/packages/backend-plugin-api/api-report.md index 545728b386..58f1ee2fa6 100644 --- a/packages/backend-plugin-api/api-report.md +++ b/packages/backend-plugin-api/api-report.md @@ -3,6 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { Config } from '@backstage/config'; import { Handler } from 'express'; import { Logger } from 'winston'; @@ -12,9 +14,9 @@ import { PluginCacheManager } from '@backstage/backend-common'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; +import { Readable } from 'stream'; import { TokenManager } from '@backstage/backend-common'; import { TransportStreamOptions } from 'winston-transport'; -import { UrlReader } from '@backstage/backend-common'; // @public (undocumented) export interface BackendFeature { @@ -245,6 +247,50 @@ export interface PluginMetadataService { // @public (undocumented) const pluginMetadataServiceRef: ServiceRef; +// @public +export type ReadTreeOptions = { + filter?( + path: string, + info?: { + size: number; + }, + ): boolean; + etag?: string; + signal?: AbortSignal; +}; + +// @public +export type ReadTreeResponse = { + files(): Promise; + archive(): Promise; + dir(options?: ReadTreeResponseDirOptions): Promise; + etag: string; +}; + +// @public +export type ReadTreeResponseDirOptions = { + targetDir?: string; +}; + +// @public +export type ReadTreeResponseFile = { + path: string; + content(): Promise; +}; + +// @public +export type ReadUrlOptions = { + etag?: string; + signal?: AbortSignal; +}; + +// @public +export type ReadUrlResponse = { + buffer(): Promise; + stream?(): Readable; + etag?: string; +}; + // @public (undocumented) export type RootLoggerService = LoggerService; @@ -257,6 +303,24 @@ export type SchedulerService = PluginTaskScheduler; // @public (undocumented) const schedulerServiceRef: ServiceRef; +// @public +export type SearchOptions = { + etag?: string; + signal?: AbortSignal; +}; + +// @public +export type SearchResponse = { + files: SearchResponseFile[]; + etag: string; +}; + +// @public +export type SearchResponseFile = { + url: string; + content(): Promise; +}; + // @public (undocumented) export type ServiceFactory = | { @@ -307,9 +371,13 @@ export type TypesToServiceRef = { [key in keyof T]: ServiceRef; }; -// @public (undocumented) -export type UrlReaderService = UrlReader; +// @public +export type UrlReaderService = { + readUrl(url: string, options?: ReadUrlOptions): Promise; + readTree(url: string, options?: ReadTreeOptions): Promise; + search(url: string, options?: SearchOptions): Promise; +}; // @public (undocumented) -const urlReaderServiceRef: ServiceRef; +const urlReaderServiceRef: ServiceRef; ``` diff --git a/packages/backend-plugin-api/src/services/definitions/index.ts b/packages/backend-plugin-api/src/services/definitions/index.ts index 3cf2ab4c77..31d0c8542f 100644 --- a/packages/backend-plugin-api/src/services/definitions/index.ts +++ b/packages/backend-plugin-api/src/services/definitions/index.ts @@ -32,4 +32,15 @@ export type { PluginMetadataService } from './pluginMetadataServiceRef'; export type { RootLoggerService } from './rootLoggerServiceRef'; export type { SchedulerService } from './schedulerServiceRef'; export type { TokenManagerService } from './tokenManagerServiceRef'; -export type { UrlReaderService } from './urlReaderServiceRef'; +export type { + ReadTreeOptions, + ReadTreeResponse, + ReadTreeResponseDirOptions, + ReadTreeResponseFile, + ReadUrlResponse, + ReadUrlOptions, + SearchOptions, + SearchResponse, + SearchResponseFile, + UrlReaderService, +} from './urlReaderServiceRef'; diff --git a/packages/backend-plugin-api/src/services/definitions/urlReaderServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/urlReaderServiceRef.ts index 4fc3680763..ada0660885 100644 --- a/packages/backend-plugin-api/src/services/definitions/urlReaderServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/urlReaderServiceRef.ts @@ -15,10 +15,278 @@ */ import { createServiceRef } from '../system/types'; -import { UrlReader } from '@backstage/backend-common'; +import { Readable } from 'stream'; -/** @public */ -export type UrlReaderService = UrlReader; +/** + * A generic interface for fetching plain data from URLs. + * + * @public + */ +export type UrlReaderService = { + /** + * Reads a single file and return its content. + * + * @remarks + * + * This is a replacement for the read method that supports options and + * complex responses. + * + * Use this as the read method will be removed in a future release. + */ + readUrl(url: string, options?: ReadUrlOptions): Promise; + + /** + * Reads a full or partial file tree. + */ + readTree(url: string, options?: ReadTreeOptions): Promise; + + /** + * Searches for a file in a tree using a glob pattern. + */ + search(url: string, options?: SearchOptions): Promise; +}; + +/** + * An options object for readUrl operations. + * + * @public + */ +export type ReadUrlOptions = { + /** + * An ETag which can be provided to check whether a + * {@link UrlReaderService.readUrl} response has changed from a previous execution. + * + * @remarks + * + * In the {@link UrlReaderService.readUrl} response, an ETag is returned along with + * the data. The ETag is a unique identifier of the data, usually the commit + * SHA or ETag from the target. + * + * When an ETag is given in ReadUrlOptions, {@link UrlReaderService.readUrl} will + * first compare the ETag against the ETag of the target. If they match, + * {@link UrlReaderService.readUrl} will throw a + * {@link @backstage/errors#NotModifiedError} indicating that the response + * will not differ from the previous response which included this particular + * ETag. If they do not match, {@link UrlReaderService.readUrl} will return the rest + * of the response along with a new ETag. + */ + etag?: string; + + /** + * An abort signal to pass down to the underlying request. + * + * @remarks + * + * Not all reader implementations may take this field into account. + */ + signal?: AbortSignal; +}; + +/** + * A response object for {@link UrlReaderService.readUrl} operations. + * + * @public + */ +export type ReadUrlResponse = { + /** + * Returns the data that was read from the remote URL. + */ + buffer(): Promise; + + /** + * Returns the data that was read from the remote URL as a Readable stream. + * + * @remarks + * + * This method will be required in a future release. + */ + stream?(): Readable; + + /** + * Etag returned by content provider. + * + * @remarks + * + * Can be used to compare and cache responses when doing subsequent calls. + */ + etag?: string; +}; + +/** + * An options object for {@link UrlReaderService.readTree} operations. + * + * @public + */ +export type ReadTreeOptions = { + /** + * A filter that can be used to select which files should be included. + * + * @remarks + * + * The path passed to the filter function is the relative path from the URL + * that the file tree is fetched from, without any leading '/'. + * + * For example, given the URL https://github.com/my/repo/tree/master/my-dir, a file + * at https://github.com/my/repo/blob/master/my-dir/my-subdir/my-file.txt will + * be represented as my-subdir/my-file.txt + * + * If no filter is provided, all files are extracted. + */ + filter?(path: string, info?: { size: number }): boolean; + + /** + * An ETag which can be provided to check whether a + * {@link UrlReaderService.readTree} response has changed from a previous execution. + * + * @remarks + * + * In the {@link UrlReaderService.readTree} response, an ETag is returned along with + * the tree blob. The ETag is a unique identifier of the tree blob, usually + * the commit SHA or ETag from the target. + * + * When an ETag is given as a request option, {@link UrlReaderService.readTree} will + * first compare the ETag against the ETag on the target branch. If they + * match, {@link UrlReaderService.readTree} will throw a + * {@link @backstage/errors#NotModifiedError} indicating that the response + * will not differ from the previous response which included this particular + * ETag. If they do not match, {@link UrlReaderService.readTree} will return the + * rest of the response along with a new ETag. + */ + etag?: string; + + /** + * An abort signal to pass down to the underlying request. + * + * @remarks + * + * Not all reader implementations may take this field into account. + */ + signal?: AbortSignal; +}; + +/** + * Options that control {@link ReadTreeResponse.dir} execution. + * + * @public + */ +export type ReadTreeResponseDirOptions = { + /** + * The directory to write files to. + * + * @remarks + * + * Defaults to the OS tmpdir, or `backend.workingDirectory` if set in config. + */ + targetDir?: string; +}; + +/** + * A response object for {@link UrlReaderService.readTree} operations. + * + * @public + */ +export type ReadTreeResponse = { + /** + * Returns an array of all the files inside the tree, and corresponding + * functions to read their content. + */ + files(): Promise; + + /** + * Returns the tree contents as a binary archive, using a stream. + */ + archive(): Promise; + + /** + * Extracts the tree response into a directory and returns the path of the + * directory. + * + * **NOTE**: It is the responsibility of the caller to remove the directory after use. + */ + dir(options?: ReadTreeResponseDirOptions): Promise; + + /** + * Etag returned by content provider. + * + * @remarks + * + * Can be used to compare and cache responses when doing subsequent calls. + */ + etag: string; +}; + +/** + * Represents a single file in a {@link UrlReaderService.readTree} response. + * + * @public + */ +export type ReadTreeResponseFile = { + path: string; + content(): Promise; +}; + +/** + * An options object for search operations. + * + * @public + */ +export type SearchOptions = { + /** + * An etag can be provided to check whether the search response has changed from a previous execution. + * + * In the search() response, an etag is returned along with the files. The etag is a unique identifier + * of the current tree, usually the commit SHA or etag from the target. + * + * When an etag is given in SearchOptions, search will first compare the etag against the etag + * on the target branch. If they match, search will throw a NotModifiedError indicating that the search + * response will not differ from the previous response which included this particular etag. If they mismatch, + * search will return the rest of SearchResponse along with a new etag. + */ + etag?: string; + + /** + * An abort signal to pass down to the underlying request. + * + * @remarks + * + * Not all reader implementations may take this field into account. + */ + signal?: AbortSignal; +}; + +/** + * The output of a search operation. + * + * @public + */ +export type SearchResponse = { + /** + * The files that matched the search query. + */ + files: SearchResponseFile[]; + + /** + * A unique identifier of the current remote tree, usually the commit SHA or etag from the target. + */ + etag: string; +}; + +/** + * Represents a single file in a search response. + * + * @public + */ +export type SearchResponseFile = { + /** + * The full URL to the file. + */ + url: string; + + /** + * The binary contents of the file. + */ + content(): Promise; +}; /** * @public diff --git a/plugins/azure-devops-backend/src/utils/azure-devops-utils.test.ts b/plugins/azure-devops-backend/src/utils/azure-devops-utils.test.ts index 74932c9653..4b912a8340 100644 --- a/plugins/azure-devops-backend/src/utils/azure-devops-utils.test.ts +++ b/plugins/azure-devops-backend/src/utils/azure-devops-utils.test.ts @@ -255,7 +255,6 @@ describe('replaceReadme', () => { }), readTree: jest.fn(), search: jest.fn(), - read: jest.fn(), }; const result = await replaceReadme( diff --git a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts index 6298aea77d..9dcadad61d 100644 --- a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts +++ b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts @@ -84,10 +84,14 @@ export class OpenApiRefProcessor implements CatalogProcessor { this.logger.debug(`Bundling OpenAPI specification from ${location.target}`); try { + const read = async (url: string) => { + const { buffer } = await this.reader.readUrl(url); + return await buffer(); + }; const bundledSpec = await bundleFileWithRefs( definition.toString(), location.target, - this.reader.read, + read, resolveUrl, ); diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index 903ac1ac6a..352ebf5df9 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -32,7 +32,6 @@ const integrations = ScmIntegrations.fromConfig(new ConfigReader({})); describe('PlaceholderProcessor', () => { const reader: jest.Mocked = { - read: jest.fn(), readTree: jest.fn(), search: jest.fn(), readUrl: jest.fn(), diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index 14c5710afc..c9575ce5cd 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -194,7 +194,6 @@ describe('UrlReaderProcessor', () => { const logger = getVoidLogger(); const reader: jest.Mocked = { - read: jest.fn(), readUrl: jest.fn(), readTree: jest.fn(), search: jest.fn().mockImplementation(async () => []), diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.test.ts index bfa4aa0507..15e5d25183 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.test.ts +++ b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.test.ts @@ -75,7 +75,6 @@ describe('fetch:cookiecutter', () => { const mockReader: UrlReader = { readUrl: jest.fn(), - read: jest.fn(), readTree: jest.fn(), search: jest.fn(), }; diff --git a/plugins/scaffolder-backend-module-rails/src/actions/fetch/rails/index.test.ts b/plugins/scaffolder-backend-module-rails/src/actions/fetch/rails/index.test.ts index 98ac3a403f..56f85d002e 100644 --- a/plugins/scaffolder-backend-module-rails/src/actions/fetch/rails/index.test.ts +++ b/plugins/scaffolder-backend-module-rails/src/actions/fetch/rails/index.test.ts @@ -74,7 +74,6 @@ describe('fetch:rails', () => { }; const mockReader: UrlReader = { - read: jest.fn(), readUrl: jest.fn(), readTree: jest.fn(), search: jest.fn(), diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts index 50ec94e099..86d5f17803 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts @@ -39,7 +39,6 @@ describe('fetchContent helper', () => { const readTree = jest.fn(); const reader: UrlReader = { - read: jest.fn(), readUrl: jest.fn(), readTree, search: jest.fn(), diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts index 1595578666..0469ffec06 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts @@ -34,7 +34,6 @@ describe('fetch:plain', () => { ); const reader: UrlReader = { readUrl: jest.fn(), - read: jest.fn(), readTree: jest.fn(), search: jest.fn(), }; diff --git a/plugins/search-backend-node/src/collators/NewlineDelimitedJsonCollatorFactory.test.ts b/plugins/search-backend-node/src/collators/NewlineDelimitedJsonCollatorFactory.test.ts index c08db8f28e..addcb87497 100644 --- a/plugins/search-backend-node/src/collators/NewlineDelimitedJsonCollatorFactory.test.ts +++ b/plugins/search-backend-node/src/collators/NewlineDelimitedJsonCollatorFactory.test.ts @@ -53,7 +53,6 @@ describe('DefaultCatalogCollatorFactory', () => { readable._read = () => {}; reader = { search: jest.fn(), - read: jest.fn(), readTree: jest.fn(), readUrl: jest.fn(), }; diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index 5751a460bf..7e7e8b24f8 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -53,7 +53,6 @@ export async function startStandaloneServer( }); const discovery = SingleHostDiscovery.fromConfig(config); const mockUrlReader: jest.Mocked = { - read: jest.fn(), readUrl: jest.fn(), readTree: jest.fn(), search: jest.fn(), diff --git a/plugins/techdocs-node/src/stages/prepare/dir.test.ts b/plugins/techdocs-node/src/stages/prepare/dir.test.ts index ddefb40f03..164cfde687 100644 --- a/plugins/techdocs-node/src/stages/prepare/dir.test.ts +++ b/plugins/techdocs-node/src/stages/prepare/dir.test.ts @@ -45,7 +45,6 @@ const createMockEntity = (annotations: {}) => { const mockConfig = new ConfigReader({}); const mockUrlReader: jest.Mocked = { - read: jest.fn(), readUrl: jest.fn(), readTree: jest.fn(), search: jest.fn(), diff --git a/yarn.lock b/yarn.lock index 6f4f0b06b4..f32ce5d727 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3392,6 +3392,7 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/backend-common@workspace:packages/backend-common" dependencies: + "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/cli-common": "workspace:^"