From f41ea01bbe913a7cdf74874e89e0e7bd81325a42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sun, 1 Dec 2024 16:14:35 +0100 Subject: [PATCH] some more MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/sixty-shirts-rescue.md | 3 ++ .../scaffolder/template-testing-dry-run.md | 1 - .../backend-system/core-services/discovery.md | 1 - .../backend-system/core-services/scheduler.md | 1 - .../integrating-search-into-plugins.md | 3 -- .../backend-defaults/report-urlReader.api.md | 37 ++++++++++++----- .../urlReader/lib/AzureUrlReader.ts | 6 +-- .../urlReader/lib/BitbucketCloudUrlReader.ts | 12 +----- .../urlReader/lib/BitbucketServerUrlReader.ts | 12 +----- .../urlReader/lib/BitbucketUrlReader.ts | 12 +----- .../urlReader/lib/FetchUrlReader.ts | 9 +--- .../urlReader/lib/GiteaUrlReader.ts | 3 +- .../urlReader/lib/GitlabUrlReader.ts | 12 +----- .../urlReader/lib/ReadUrlResponseFactory.ts | 22 +++++++++- .../lib/tree/ReadTreeResponseFactory.ts | 30 ++++++++++++-- .../src/entrypoints/urlReader/lib/types.ts | 33 ++++++++++----- .../src/entrypoints/urlReader/lib/util.ts | 41 ++++++++++++++++++- .../userInfo/DefaultUserInfoService.ts | 1 - .../package.json | 3 +- .../analyzers/GithubLocationAnalyzer.test.ts | 16 ++++---- .../src/lib/github.test.ts | 27 ++++++------ .../package.json | 1 - .../src/actions/confluence/helpers.ts | 6 ++- ...bProjectAccessTokenCreate.examples.test.ts | 4 +- .../gitlabProjectAccessTokenCreate.test.ts | 4 +- yarn.lock | 4 +- 26 files changed, 177 insertions(+), 127 deletions(-) diff --git a/.changeset/sixty-shirts-rescue.md b/.changeset/sixty-shirts-rescue.md index bad0402939..b6c1443eea 100644 --- a/.changeset/sixty-shirts-rescue.md +++ b/.changeset/sixty-shirts-rescue.md @@ -33,6 +33,9 @@ '@backstage/plugin-app-backend': patch '@backstage/plugin-auth-node': patch '@backstage/cli': patch +'@backstage/backend-defaults': patch +'@backstage/plugin-catalog-backend-module-github': patch +'@backstage/plugin-scaffolder-backend-module-confluence-to-markdown': patch --- Use native fetch instead of node-fetch diff --git a/contrib/scaffolder/template-testing-dry-run.md b/contrib/scaffolder/template-testing-dry-run.md index 70e0c572ab..12c4210ed6 100644 --- a/contrib/scaffolder/template-testing-dry-run.md +++ b/contrib/scaffolder/template-testing-dry-run.md @@ -26,7 +26,6 @@ import { readFile, writeFile } from 'node:fs/promises'; import { gzipSync } from 'node:zlib'; import { ensureDir } from 'fs-extra'; import { program } from 'commander'; -import fetch from 'node-fetch'; import ora from 'ora'; import readdir from 'recursive-readdir'; import { parse } from 'yaml'; diff --git a/docs/backend-system/core-services/discovery.md b/docs/backend-system/core-services/discovery.md index 455e398d8b..4324edbbf9 100644 --- a/docs/backend-system/core-services/discovery.md +++ b/docs/backend-system/core-services/discovery.md @@ -16,7 +16,6 @@ import { coreServices, createBackendPlugin, } from '@backstage/backend-plugin-api'; -import { fetch } from 'node-fetch'; createBackendPlugin({ pluginId: 'example', diff --git a/docs/backend-system/core-services/scheduler.md b/docs/backend-system/core-services/scheduler.md index e4c67a1199..154878ddfc 100644 --- a/docs/backend-system/core-services/scheduler.md +++ b/docs/backend-system/core-services/scheduler.md @@ -16,7 +16,6 @@ import { coreServices, createBackendPlugin, } from '@backstage/backend-plugin-api'; -import { fetch } from 'node-fetch'; createBackendPlugin({ pluginId: 'example', diff --git a/docs/plugins/integrating-search-into-plugins.md b/docs/plugins/integrating-search-into-plugins.md index 05d07feb08..e51b4cfefa 100644 --- a/docs/plugins/integrating-search-into-plugins.md +++ b/docs/plugins/integrating-search-into-plugins.md @@ -106,15 +106,12 @@ Imagine your FAQs can be retrieved at the URL `https://backstage.example.biz/faq Below we provide an example implementation of how the FAQ collator factory could look like using our new document type, placed in the `plugins/search-backend-module-faq-snippets-collator/src/factory.ts` file: ```ts -import fetch from 'node-fetch'; import { Readable } from 'stream'; - import { LoggerService, RootConfigService, } from '@backstage/backend-plugin-api'; import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; - import { FaqSnippetDocument } from './types'; const DEFAULT_BASE_URL = 'https://backstage.example.biz/faq-snippets'; diff --git a/packages/backend-defaults/report-urlReader.api.md b/packages/backend-defaults/report-urlReader.api.md index 4813643d68..efe4a9ae47 100644 --- a/packages/backend-defaults/report-urlReader.api.md +++ b/packages/backend-defaults/report-urlReader.api.md @@ -415,17 +415,29 @@ export interface ReadTreeResponseFactory { } // @public -export type ReadTreeResponseFactoryOptions = { - stream: Readable; - subpath?: string; - etag: string; - filter?: ( - path: string, - info?: { - size: number; - }, - ) => boolean; -}; +export type ReadTreeResponseFactoryOptions = + | { + stream: Readable; + subpath?: string; + etag: string; + filter?: ( + path: string, + info?: { + size: number; + }, + ) => boolean; + } + | { + response: Response; + subpath?: string; + etag?: string; + filter?: ( + path: string, + info?: { + size: number; + }, + ) => boolean; + }; // @public export class ReadUrlResponseFactory { @@ -437,6 +449,9 @@ export class ReadUrlResponseFactory { stream: Readable, options?: ReadUrlResponseFactoryFromStreamOptions, ): Promise; + static fromResponse( + response: Response, + ): Promise; } // @public diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts index 50513fcdcc..b46e7c20cd 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts @@ -32,9 +32,7 @@ import { ScmIntegrations, AzureIntegration, } from '@backstage/integration'; -import fetch, { Response } from 'node-fetch'; import { Minimatch } from 'minimatch'; -import { Readable } from 'stream'; import { NotFoundError, NotModifiedError } from '@backstage/errors'; import { ReadTreeResponseFactory, ReaderFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; @@ -101,7 +99,7 @@ export class AzureUrlReader implements UrlReaderService { // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html if (response.ok && response.status !== 203) { - return ReadUrlResponseFactory.fromNodeJSReadable(response.body); + return ReadUrlResponseFactory.fromResponse(response); } const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; @@ -172,7 +170,7 @@ export class AzureUrlReader implements UrlReaderService { } return await this.deps.treeResponseFactory.fromZipArchive({ - stream: Readable.from(archiveAzureResponse.body), + response: archiveAzureResponse, etag: commitSha, filter, subpath, diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts index b7b733867e..f6742c7423 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts @@ -32,14 +32,11 @@ import { getBitbucketCloudRequestOptions, ScmIntegrations, } from '@backstage/integration'; -import fetch, { Response } from 'node-fetch'; import parseGitUrl from 'git-url-parse'; import { trimEnd } from 'lodash'; import { Minimatch } from 'minimatch'; -import { Readable } from 'stream'; import { ReaderFactory, ReadTreeResponseFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; -import { parseLastModified } from './util'; /** * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Cloud. @@ -116,12 +113,7 @@ export class BitbucketCloudUrlReader implements UrlReaderService { } if (response.ok) { - return ReadUrlResponseFactory.fromNodeJSReadable(response.body, { - etag: response.headers.get('ETag') ?? undefined, - lastModifiedAt: parseLastModified( - response.headers.get('Last-Modified'), - ), - }); + return ReadUrlResponseFactory.fromResponse(response); } const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`; @@ -159,7 +151,7 @@ export class BitbucketCloudUrlReader implements UrlReaderService { } return await this.deps.treeResponseFactory.fromTarArchive({ - stream: Readable.from(archiveResponse.body), + response: archiveResponse, subpath: filepath, etag: lastCommitShortHash, filter: options?.filter, diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts index 332e300cb8..c37cb59308 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts @@ -31,14 +31,11 @@ import { getBitbucketServerRequestOptions, ScmIntegrations, } from '@backstage/integration'; -import fetch, { Response } from 'node-fetch'; import parseGitUrl from 'git-url-parse'; import { trimEnd } from 'lodash'; import { Minimatch } from 'minimatch'; -import { Readable } from 'stream'; import { ReaderFactory, ReadTreeResponseFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; -import { parseLastModified } from './util'; /** * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Server APIs. @@ -107,12 +104,7 @@ export class BitbucketServerUrlReader implements UrlReaderService { } if (response.ok) { - return ReadUrlResponseFactory.fromNodeJSReadable(response.body, { - etag: response.headers.get('ETag') ?? undefined, - lastModifiedAt: parseLastModified( - response.headers.get('Last-Modified'), - ), - }); + return ReadUrlResponseFactory.fromResponse(response); } const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`; @@ -150,7 +142,7 @@ export class BitbucketServerUrlReader implements UrlReaderService { } return await this.deps.treeResponseFactory.fromTarArchive({ - stream: Readable.from(archiveResponse.body), + response: archiveResponse, subpath: filepath, etag: lastCommitShortHash, filter: options?.filter, diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts index 55d676ed91..c12998509e 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts @@ -32,15 +32,12 @@ import { getBitbucketRequestOptions, ScmIntegrations, } from '@backstage/integration'; -import fetch, { Response } from 'node-fetch'; import parseGitUrl from 'git-url-parse'; import { trimEnd } from 'lodash'; import { Minimatch } from 'minimatch'; -import { Readable } from 'stream'; import { LoggerService } from '@backstage/backend-plugin-api'; import { ReaderFactory, ReadTreeResponseFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; -import { parseLastModified } from './util'; /** * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket v1 and v2 APIs, such @@ -127,12 +124,7 @@ export class BitbucketUrlReader implements UrlReaderService { } if (response.ok) { - return ReadUrlResponseFactory.fromNodeJSReadable(response.body, { - etag: response.headers.get('ETag') ?? undefined, - lastModifiedAt: parseLastModified( - response.headers.get('Last-Modified'), - ), - }); + return ReadUrlResponseFactory.fromResponse(response); } const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`; @@ -170,7 +162,7 @@ export class BitbucketUrlReader implements UrlReaderService { } return await this.deps.treeResponseFactory.fromTarArchive({ - stream: Readable.from(archiveBitbucketResponse.body), + response: archiveBitbucketResponse, subpath: filepath, etag: lastCommitShortHash, filter: options?.filter, diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts index 55041cf036..0e5a279c4d 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts @@ -22,11 +22,9 @@ import { UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; import { NotFoundError, NotModifiedError } from '@backstage/errors'; -import fetch, { Response } from 'node-fetch'; import { ReaderFactory } from './types'; import path from 'path'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; -import { parseLastModified } from './util'; const isInRange = (num: number, [start, end]: [number, number]) => { return num >= start && num <= end; @@ -150,12 +148,7 @@ export class FetchUrlReader implements UrlReaderService { } if (response.ok) { - return ReadUrlResponseFactory.fromNodeJSReadable(response.body, { - etag: response.headers.get('ETag') ?? undefined, - lastModifiedAt: parseLastModified( - response.headers.get('Last-Modified'), - ), - }); + return ReadUrlResponseFactory.fromResponse(response); } const message = `could not read ${url}, ${response.status} ${response.statusText}`; diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts index d8471c43f3..502c2d2ff9 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts @@ -32,7 +32,6 @@ import { ScmIntegrations, } from '@backstage/integration'; import { ReaderFactory, ReadTreeResponseFactory } from './types'; -import fetch, { Response } from 'node-fetch'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import { AuthenticationError, @@ -149,7 +148,7 @@ export class GiteaUrlReader implements UrlReaderService { const parsedUri = parseGiteaUrl(this.integration.config, url); return this.deps.treeResponseFactory.fromTarArchive({ - stream: Readable.from(response.body), + response: response, subpath: parsedUri.path, etag: lastCommitHash, filter: options?.filter, diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts index f5050cca5e..c1d8c02a53 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts @@ -34,11 +34,8 @@ import { import parseGitUrl from 'git-url-parse'; import { trimEnd, trimStart } from 'lodash'; import { Minimatch } from 'minimatch'; -import fetch, { Response } from 'node-fetch'; -import { Readable } from 'stream'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import { ReadTreeResponseFactory, ReaderFactory } from './types'; -import { parseLastModified } from './util'; /** * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files on GitLab. @@ -101,12 +98,7 @@ export class GitlabUrlReader implements UrlReaderService { } if (response.ok) { - return ReadUrlResponseFactory.fromNodeJSReadable(response.body, { - etag: response.headers.get('ETag') ?? undefined, - lastModifiedAt: parseLastModified( - response.headers.get('Last-Modified'), - ), - }); + return ReadUrlResponseFactory.fromResponse(response); } const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; @@ -228,7 +220,7 @@ export class GitlabUrlReader implements UrlReaderService { } return await this.deps.treeResponseFactory.fromTarArchive({ - stream: Readable.from(archiveGitLabResponse.body), + response: archiveGitLabResponse, subpath: filepath, etag: commitSha, filter: options?.filter, diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/ReadUrlResponseFactory.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/ReadUrlResponseFactory.ts index ecc1bd6101..1f4f297c5a 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/ReadUrlResponseFactory.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/ReadUrlResponseFactory.ts @@ -19,6 +19,7 @@ import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; import getRawBody from 'raw-body'; import { Readable } from 'stream'; import { ReadUrlResponseFactoryFromStreamOptions } from './types'; +import { parseLastModified, responseToReadable } from './util'; /** * Utility class for UrlReader implementations to create valid ReadUrlResponse @@ -28,7 +29,7 @@ import { ReadUrlResponseFactoryFromStreamOptions } from './types'; */ export class ReadUrlResponseFactory { /** - * Resolves a ReadUrlResponse from a Readable stream. + * Resolves a UrlReaderServiceReadUrlResponse from a Readable stream. */ static async fromReadable( stream: Readable, @@ -64,7 +65,7 @@ export class ReadUrlResponseFactory { } /** - * Resolves a ReadUrlResponse from an old-style NodeJS.ReadableStream. + * Resolves a UrlReaderServiceReadUrlResponse from an old-style NodeJS.ReadableStream. */ static async fromNodeJSReadable( oldStyleStream: NodeJS.ReadableStream, @@ -73,4 +74,21 @@ export class ReadUrlResponseFactory { const readable = Readable.from(oldStyleStream); return ReadUrlResponseFactory.fromReadable(readable, options); } + + /** + * Resolves a UrlReaderServiceReadUrlResponse from a native fetch response body. + */ + static async fromResponse( + response: Response, + ): Promise { + const etag = response.headers.get('etag') || undefined; + const lastModifiedAt = parseLastModified( + response.headers.get('last-modified'), + ); + + return ReadUrlResponseFactory.fromReadable(responseToReadable(response), { + etag, + lastModifiedAt, + }); + } } diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/tree/ReadTreeResponseFactory.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/tree/ReadTreeResponseFactory.ts index 1973b2158f..a8f187d7b8 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/tree/ReadTreeResponseFactory.ts @@ -15,6 +15,7 @@ */ import os from 'os'; +import { Readable } from 'stream'; import { Config } from '@backstage/config'; import { ReadTreeResponseFactoryOptions, @@ -25,6 +26,7 @@ import { TarArchiveResponse } from './TarArchiveResponse'; import { ZipArchiveResponse } from './ZipArchiveResponse'; import { ReadableArrayResponse } from './ReadableArrayResponse'; import { UrlReaderServiceReadTreeResponse } from '@backstage/backend-plugin-api'; +import { responseToReadable } from '../util'; export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { static create(options: { config: Config }): DefaultReadTreeResponseFactory { @@ -41,11 +43,21 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { stripFirstDirectory?: boolean; }, ): Promise { + let stream: Readable; + let etag: string; + if ('stream' in options) { + stream = options.stream; + etag = options.etag; + } else { + stream = responseToReadable(options.response); + etag = (options.etag ?? options.response.headers.get('etag')) || ''; + } + return new TarArchiveResponse( - options.stream, + stream, options.subpath ?? '', this.workDir, - options.etag, + etag, options.filter, options.stripFirstDirectory ?? true, ); @@ -54,11 +66,21 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { async fromZipArchive( options: ReadTreeResponseFactoryOptions, ): Promise { + let stream: Readable; + let etag: string; + if ('stream' in options) { + stream = options.stream; + etag = options.etag; + } else { + stream = responseToReadable(options.response); + etag = (options.etag ?? options.response.headers.get('etag')) || ''; + } + return new ZipArchiveResponse( - options.stream, + stream, options.subpath ?? '', this.workDir, - options.etag, + etag, options.filter, ); } diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/types.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/types.ts index 5afbed178f..f8f03a6f3b 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/types.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/types.ts @@ -60,17 +60,28 @@ export type ReadUrlResponseFactoryFromStreamOptions = { * * @public */ -export type ReadTreeResponseFactoryOptions = { - // A binary stream of a tar archive. - stream: Readable; - // If unset, the files at the root of the tree will be read. - // subpath must not contain the name of the top level directory. - subpath?: string; - // etag of the blob - etag: string; - // Filter passed on from the ReadTreeOptions - filter?: (path: string, info?: { size: number }) => boolean; -}; +export type ReadTreeResponseFactoryOptions = + | { + // A binary stream of a tar archive. + stream: Readable; + // If unset, the files at the root of the tree will be read. + // subpath must not contain the name of the top level directory. + subpath?: string; + // etag of the blob + etag: string; + // Filter passed on from the ReadTreeOptions + filter?: (path: string, info?: { size: number }) => boolean; + } + | { + /** A response from a fetch request */ + response: Response; + /** If unset, the files at the root of the tree will be read. Subpath must not contain the name of the top level directory. */ + subpath?: string; + /** etag of the blob, to optionally override what the response header may or may not say */ + etag?: string; + /** Filter passed on from the ReadTreeOptions */ + filter?: (path: string, info?: { size: number }) => boolean; + }; /** * Options that control {@link ReadTreeResponseFactory.fromReadableArray} diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/util.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/util.ts index 3d65c2c527..bd19feef2d 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/util.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/util.ts @@ -14,10 +14,47 @@ * limitations under the License. */ -export function parseLastModified(value: string | null | undefined) { +import { PassThrough, Readable } from 'stream'; +import { ReadableStream as WebReadableStream } from 'stream/web'; + +export function parseLastModified( + value: string | null | undefined, +): Date | undefined { if (!value) { return undefined; } - return new Date(value); + try { + const result = new Date(value); + result.toISOString(); // triggers exception if input was invalid + return result; + } catch { + return undefined; + } +} + +export function responseToReadable(response: Response): Readable { + return response.body + ? Readable.from(toWeb(response.body)) + : new PassThrough().end(); +} + +// The NodeJS ReadableStream is that fetch returns is super basic and not even +// iterable. This function converts it to the smarter, iterable stream/web +// variant instead. +export function toWeb( + responseBody: ReadableStream, +): WebReadableStream { + const reader = responseBody.getReader(); + return new WebReadableStream({ + async pull(controller) { + const { value, done } = await reader.read(); + if (value) { + controller.enqueue(value); + } + if (done) { + controller.close(); + } + }, + }); } diff --git a/packages/backend-defaults/src/entrypoints/userInfo/DefaultUserInfoService.ts b/packages/backend-defaults/src/entrypoints/userInfo/DefaultUserInfoService.ts index 83bbba89b7..21f8de5631 100644 --- a/packages/backend-defaults/src/entrypoints/userInfo/DefaultUserInfoService.ts +++ b/packages/backend-defaults/src/entrypoints/userInfo/DefaultUserInfoService.ts @@ -22,7 +22,6 @@ import { } from '@backstage/backend-plugin-api'; import { ResponseError } from '@backstage/errors'; import { decodeJwt } from 'jose'; -import fetch from 'node-fetch'; import { toInternalBackstageCredentials } from '../auth/helpers'; export type Options = { diff --git a/plugins/catalog-backend-module-github/package.json b/plugins/catalog-backend-module-github/package.json index 0407bd1f9c..ef8c40c75d 100644 --- a/plugins/catalog-backend-module-github/package.json +++ b/plugins/catalog-backend-module-github/package.json @@ -66,7 +66,6 @@ "git-url-parse": "^15.0.0", "lodash": "^4.17.21", "minimatch": "^9.0.0", - "node-fetch": "^2.7.0", "uuid": "^11.0.0" }, "devDependencies": { @@ -74,7 +73,7 @@ "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", "luxon": "^3.0.0", - "msw": "^1.0.0" + "msw": "^2.0.0" }, "configSchema": "config.d.ts" } diff --git a/plugins/catalog-backend-module-github/src/analyzers/GithubLocationAnalyzer.test.ts b/plugins/catalog-backend-module-github/src/analyzers/GithubLocationAnalyzer.test.ts index 29c8cbc66d..d3be18f489 100644 --- a/plugins/catalog-backend-module-github/src/analyzers/GithubLocationAnalyzer.test.ts +++ b/plugins/catalog-backend-module-github/src/analyzers/GithubLocationAnalyzer.test.ts @@ -39,7 +39,7 @@ import { mockServices, } from '@backstage/backend-test-utils'; import { setupServer } from 'msw/node'; -import { rest } from 'msw'; +import { http, HttpResponse } from 'msw'; import { ConfigReader } from '@backstage/config'; const server = setupServer(); @@ -67,10 +67,9 @@ describe('GithubLocationAnalyzer', () => { beforeEach(() => { server.use( - rest.post('http://localhost:7007/locations', async (_, res, ctx) => { - return res( - ctx.status(201), - ctx.json({ + http.post('http://localhost:7007/locations', () => + HttpResponse.json( + { location: 'test', exists: false, entities: [ @@ -100,9 +99,10 @@ describe('GithubLocationAnalyzer', () => { }, }, ], - }), - ); - }), + }, + { status: 201 }, + ), + ), ); octokit.repos.get.mockResolvedValue({ diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index 9eae9e1161..cbe8d8c525 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -17,7 +17,7 @@ import { registerMswTestHooks } from '@backstage/backend-test-utils'; import { GroupEntity, UserEntity } from '@backstage/catalog-model'; import { graphql as graphqlOctokit } from '@octokit/graphql'; -import { graphql as graphqlMsw } from 'msw'; +import { graphql as graphqlMsw, HttpResponse } from 'msw'; import { setupServer } from 'msw/node'; import { TeamTransformer, UserTransformer } from './defaultTransformers'; import { @@ -33,14 +33,13 @@ import { createRemoveEntitiesOperation, createReplaceEntitiesOperation, } from './github'; -import fetch from 'node-fetch'; describe('github', () => { - const graphql = graphqlOctokit.defaults({ request: { fetch } }); - const server = setupServer(); registerMswTestHooks(server); + const graphql = graphqlOctokit.defaults({}); + describe('getOrganizationUsers using defaultUserMapper', () => { it('reads members', async () => { const input: QueryResponse = { @@ -73,7 +72,7 @@ describe('github', () => { }; server.use( - graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))), + graphqlMsw.query('users', () => HttpResponse.json({ data: input })), ); await expect( @@ -136,7 +135,7 @@ describe('github', () => { }; server.use( - graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))), + graphqlMsw.query('users', () => HttpResponse.json({ data: input })), ); await expect( @@ -180,7 +179,7 @@ describe('github', () => { }; server.use( - graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))), + graphqlMsw.query('users', () => HttpResponse.json({ data: input })), ); const users = await getOrganizationUsers( @@ -255,7 +254,7 @@ describe('github', () => { }; server.use( - graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))), + graphqlMsw.query('teams', () => HttpResponse.json({ data: input })), ); await expect(getOrganizationTeams(graphql, 'a')).resolves.toEqual(output); @@ -355,7 +354,7 @@ describe('github', () => { }; server.use( - graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))), + graphqlMsw.query('teams', () => HttpResponse.json({ data: input })), ); await expect( @@ -435,7 +434,7 @@ describe('github', () => { }; server.use( - graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))), + graphqlMsw.query('teams', () => HttpResponse.json({ data: input })), ); const teams = await getOrganizationTeams( @@ -471,7 +470,7 @@ describe('github', () => { }; server.use( - graphqlMsw.query('orgs', (_req, res, ctx) => res(ctx.data(input))), + graphqlMsw.query('orgs', () => HttpResponse.json({ data: input })), ); await expect(getOrganizationsFromUser(graphql, 'foo')).resolves.toEqual({ @@ -500,7 +499,7 @@ describe('github', () => { }; server.use( - graphqlMsw.query('members', (_req, res, ctx) => res(ctx.data(input))), + graphqlMsw.query('members', () => HttpResponse.json({ data: input })), ); await expect(getTeamMembers(graphql, 'a', 'b')).resolves.toEqual(output); @@ -587,8 +586,8 @@ describe('github', () => { }; server.use( - graphqlMsw.query('repositories', (_req, res, ctx) => - res(ctx.data(input)), + graphqlMsw.query('repositories', () => + HttpResponse.json({ data: input }), ), ); diff --git a/plugins/scaffolder-backend-module-confluence-to-markdown/package.json b/plugins/scaffolder-backend-module-confluence-to-markdown/package.json index 8efa18b1b2..e3d9d61457 100644 --- a/plugins/scaffolder-backend-module-confluence-to-markdown/package.json +++ b/plugins/scaffolder-backend-module-confluence-to-markdown/package.json @@ -53,7 +53,6 @@ "@backstage/plugin-scaffolder-node": "workspace:^", "fs-extra": "^11.2.0", "git-url-parse": "^15.0.0", - "node-fetch": "^2.7.0", "node-html-markdown": "^1.3.0", "yaml": "^2.0.0" }, diff --git a/plugins/scaffolder-backend-module-confluence-to-markdown/src/actions/confluence/helpers.ts b/plugins/scaffolder-backend-module-confluence-to-markdown/src/actions/confluence/helpers.ts index 65992b10f3..bbc722b1d5 100644 --- a/plugins/scaffolder-backend-module-confluence-to-markdown/src/actions/confluence/helpers.ts +++ b/plugins/scaffolder-backend-module-confluence-to-markdown/src/actions/confluence/helpers.ts @@ -17,7 +17,7 @@ import { Config } from '@backstage/config'; import { ResponseError, ConflictError, InputError } from '@backstage/errors'; import fs from 'fs-extra'; -import fetch, { Response } from 'node-fetch'; +import { Readable } from 'stream'; interface Links { webui: string; @@ -164,7 +164,9 @@ export const getAndWriteAttachments = async ( const writeStream = fs.createWriteStream( `${workspace}/${mkdocsDir}docs/img/${downloadTitle}`, ); - res.body.pipe(writeStream); + // TODO(freben): This cast is sketchy, but for some reason the node types don't quite line up here + // https://stackoverflow.com/questions/44672942/stream-response-to-file-using-fetch-api-and-fs-createwritestream/73879265#73879265 + Readable.fromWeb(res.body as any).pipe(writeStream); await new Promise((resolve, reject) => { writeStream.on('finish', () => { resolve(`${workspace}/${mkdocsDir}docs/img/${downloadTitle}`); diff --git a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabProjectAccessTokenCreate.examples.test.ts b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabProjectAccessTokenCreate.examples.test.ts index f69ee2beb7..3ad0237d76 100644 --- a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabProjectAccessTokenCreate.examples.test.ts +++ b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabProjectAccessTokenCreate.examples.test.ts @@ -13,17 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { ConfigReader } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; import { createMockActionContext } from '@backstage/plugin-scaffolder-node-test-utils'; import yaml from 'yaml'; import { createGitlabProjectAccessTokenAction } from './gitlabProjectAccessTokenCreate'; // Adjust the import based on your project structure import { examples } from './gitlabProjectAccessTokenCreate.examples'; - import { DateTime } from 'luxon'; -jest.mock('node-fetch'); - const mockGitlabClient = { ProjectAccessTokens: { create: jest.fn(), diff --git a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabProjectAccessTokenCreate.test.ts b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabProjectAccessTokenCreate.test.ts index f5f53c30ec..0cbc0fc122 100644 --- a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabProjectAccessTokenCreate.test.ts +++ b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabProjectAccessTokenCreate.test.ts @@ -13,15 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { ConfigReader } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; import { createMockActionContext } from '@backstage/plugin-scaffolder-node-test-utils'; import { createGitlabProjectAccessTokenAction } from './gitlabProjectAccessTokenCreate'; // Adjust the import based on your project structure - import { DateTime } from 'luxon'; -jest.mock('node-fetch'); - const mockGitlabClient = { ProjectAccessTokens: { create: jest.fn(), diff --git a/yarn.lock b/yarn.lock index 6c1ef15420..5de73334cb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5921,8 +5921,7 @@ __metadata: lodash: ^4.17.21 luxon: ^3.0.0 minimatch: ^9.0.0 - msw: ^1.0.0 - node-fetch: ^2.7.0 + msw: ^2.0.0 uuid: ^11.0.0 languageName: unknown linkType: soft @@ -7483,7 +7482,6 @@ __metadata: fs-extra: ^11.2.0 git-url-parse: ^15.0.0 msw: ^1.0.0 - node-fetch: ^2.7.0 node-html-markdown: ^1.3.0 yaml: ^2.0.0 languageName: unknown