From e65ad0989461cfb2476ba316d83a27decb689b3f Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 22 Jun 2023 12:39:46 +0200 Subject: [PATCH 1/4] chore: body is not a buffer, let's make it a Readable first Signed-off-by: blam --- packages/backend-common/src/reading/AzureUrlReader.ts | 2 +- .../backend-common/src/reading/BitbucketCloudUrlReader.ts | 2 +- .../backend-common/src/reading/BitbucketServerUrlReader.ts | 2 +- packages/backend-common/src/reading/BitbucketUrlReader.ts | 2 +- packages/backend-common/src/reading/GerritUrlReader.ts | 2 +- packages/backend-common/src/reading/GithubUrlReader.ts | 3 +-- packages/backend-common/src/reading/GitlabUrlReader.ts | 2 +- .../backend-common/src/reading/tree/TarArchiveResponse.ts | 4 +++- 8 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index 26fe3f4a12..94abf3a178 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -158,7 +158,7 @@ export class AzureUrlReader implements UrlReader { } return await this.deps.treeResponseFactory.fromZipArchive({ - stream: archiveAzureResponse.body as unknown as Readable, + stream: Readable.from(archiveAzureResponse.body), etag: commitSha, filter, subpath, diff --git a/packages/backend-common/src/reading/BitbucketCloudUrlReader.ts b/packages/backend-common/src/reading/BitbucketCloudUrlReader.ts index 1522b7678d..93587cd6ae 100644 --- a/packages/backend-common/src/reading/BitbucketCloudUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketCloudUrlReader.ts @@ -160,7 +160,7 @@ export class BitbucketCloudUrlReader implements UrlReader { } return await this.deps.treeResponseFactory.fromTarArchive({ - stream: archiveResponse.body as unknown as Readable, + stream: Readable.from(archiveResponse.body), subpath: filepath, etag: lastCommitShortHash, filter: options?.filter, diff --git a/packages/backend-common/src/reading/BitbucketServerUrlReader.ts b/packages/backend-common/src/reading/BitbucketServerUrlReader.ts index 2b6b5a1f8f..8e2f1708c4 100644 --- a/packages/backend-common/src/reading/BitbucketServerUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketServerUrlReader.ts @@ -151,7 +151,7 @@ export class BitbucketServerUrlReader implements UrlReader { } return await this.deps.treeResponseFactory.fromTarArchive({ - stream: archiveResponse.body as unknown as Readable, + stream: Readable.from(archiveResponse.body), subpath: filepath, etag: lastCommitShortHash, filter: options?.filter, diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 3a34c725c6..090fe9098d 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -171,7 +171,7 @@ export class BitbucketUrlReader implements UrlReader { } return await this.deps.treeResponseFactory.fromTarArchive({ - stream: archiveBitbucketResponse.body as unknown as Readable, + stream: Readable.from(archiveBitbucketResponse.body), subpath: filepath, etag: lastCommitShortHash, filter: options?.filter, diff --git a/packages/backend-common/src/reading/GerritUrlReader.ts b/packages/backend-common/src/reading/GerritUrlReader.ts index 5d86e5d9ac..417eca2e57 100644 --- a/packages/backend-common/src/reading/GerritUrlReader.ts +++ b/packages/backend-common/src/reading/GerritUrlReader.ts @@ -204,7 +204,7 @@ export class GerritUrlReader implements UrlReader { }); const tarArchive = Readable.from(data); return await this.deps.treeResponseFactory.fromTarArchive({ - stream: tarArchive as unknown as Readable, + stream: Readable.from(tarArchive), subpath: filePath === '/' ? undefined : filePath, etag: branchInfo.revision, filter: options?.filter, diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 8300127c5b..764431386d 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -238,7 +238,7 @@ export class GithubUrlReader implements UrlReader { return await this.deps.treeResponseFactory.fromTarArchive({ // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want // to stick to using that in exclusively backend code. - stream: archive.body as unknown as Readable, + stream: Readable.from(archive.body), subpath, etag: sha, filter: options?.filter, @@ -347,7 +347,6 @@ export class GithubUrlReader implements UrlReader { init: RequestInit, ): Promise { const urlAsString = url.toString(); - const response = await fetch(urlAsString, init); if (!response.ok) { diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index ab59c19b16..178e65b40a 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -229,7 +229,7 @@ export class GitlabUrlReader implements UrlReader { } return await this.deps.treeResponseFactory.fromTarArchive({ - stream: archiveGitLabResponse.body as unknown as Readable, + stream: Readable.from(archiveGitLabResponse.body), subpath: filepath, etag: commitSha, filter: options?.filter, diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts index 493e283c5f..eeda6e25f3 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -151,7 +151,9 @@ export class TarArchiveResponse implements ReadTreeResponse { const strip = this.subPath ? this.subPath.split('/').length : 1; let filterError: Error | undefined = undefined; - + this.stream.on('error', console.error); + this.stream.on('close', console.error); + console.log('we are here!'); await pipeline( this.stream, tar.extract({ From 2e0637835bb16ed6f3fb0004cb9993ce94b6af58 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 22 Jun 2023 12:44:43 +0200 Subject: [PATCH 2/4] chore: cleanup Signed-off-by: blam --- packages/backend-common/src/reading/GerritUrlReader.ts | 2 +- packages/backend-common/src/reading/tree/TarArchiveResponse.ts | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/backend-common/src/reading/GerritUrlReader.ts b/packages/backend-common/src/reading/GerritUrlReader.ts index 417eca2e57..8ac5452a4c 100644 --- a/packages/backend-common/src/reading/GerritUrlReader.ts +++ b/packages/backend-common/src/reading/GerritUrlReader.ts @@ -204,7 +204,7 @@ export class GerritUrlReader implements UrlReader { }); const tarArchive = Readable.from(data); return await this.deps.treeResponseFactory.fromTarArchive({ - stream: Readable.from(tarArchive), + stream: tarArchive, subpath: filePath === '/' ? undefined : filePath, etag: branchInfo.revision, filter: options?.filter, diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts index eeda6e25f3..e1529c196a 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -151,9 +151,6 @@ export class TarArchiveResponse implements ReadTreeResponse { const strip = this.subPath ? this.subPath.split('/').length : 1; let filterError: Error | undefined = undefined; - this.stream.on('error', console.error); - this.stream.on('close', console.error); - console.log('we are here!'); await pipeline( this.stream, tar.extract({ From 787ddcc5ef99077d7e9b704220047f35d26ec3f6 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 22 Jun 2023 12:46:28 +0200 Subject: [PATCH 3/4] chore: added changeset Signed-off-by: blam --- .changeset/many-ants-poke.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/many-ants-poke.md diff --git a/.changeset/many-ants-poke.md b/.changeset/many-ants-poke.md new file mode 100644 index 0000000000..57856e0ef2 --- /dev/null +++ b/.changeset/many-ants-poke.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +use `Readable.from` to convert the buffer to explicitly convert the buffer to a `Readable` stream From 39bbbc52612da56f0862e8b5082bdfa5caf9286b Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 22 Jun 2023 12:56:18 +0200 Subject: [PATCH 4/4] chore: too many words in the changeset and not enough sense Signed-off-by: blam --- .changeset/many-ants-poke.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/many-ants-poke.md b/.changeset/many-ants-poke.md index 57856e0ef2..e6d94fdee5 100644 --- a/.changeset/many-ants-poke.md +++ b/.changeset/many-ants-poke.md @@ -2,4 +2,4 @@ '@backstage/backend-common': patch --- -use `Readable.from` to convert the buffer to explicitly convert the buffer to a `Readable` stream +use `Readable.from` to explicitly convert the `buffer` from `node-fetch` to a `Readable` stream