paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 39 Packages Projects Releases Wiki Activity

Merge pull request #8083 from backstage/freben/abort-urlreader

Browse Source 
Add `AbortSignal` support to `UrlReader`
This commit is contained in:
Fredrik Adelöw
2021-11-17 13:16:04 +01:00
committed by GitHub
parent d0a8bdb179 01f74aa878
commit d701630972
10 changed files with 121 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/proud-bottles-cheat.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/backend-common': patch
---
Add `AbortSignal` support to `UrlReader`
packages/backend-common/api-report.md
+7 -3
View File
@@ -6,6 +6,7 @@
/// <reference types="node" />
/// <reference types="webpack-env" />
import { AbortSignal as AbortSignal_2 } from 'node-abort-controller';
import { AwsS3Integration } from '@backstage/integration';
import { AzureIntegration } from '@backstage/integration';
import { BitbucketIntegration } from '@backstage/integration';
@@ -48,7 +49,7 @@ export class AwsS3UrlReader implements UrlReader {
// (undocumented)
read(url: string): Promise<Buffer>;
// (undocumented)
readTree(url: string): Promise<ReadTreeResponse>;
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
// (undocumented)
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
// (undocumented)
@@ -72,7 +73,7 @@ export class AzureUrlReader implements UrlReader {
// (undocumented)
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
// (undocumented)
readUrl(url: string, _options?: ReadUrlOptions): Promise<ReadUrlResponse>;
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
// (undocumented)
search(url: string, options?: SearchOptions): Promise<SearchResponse>;
// (undocumented)
@@ -94,7 +95,7 @@ export class BitbucketUrlReader implements UrlReader {
// (undocumented)
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
// (undocumented)
readUrl(url: string, _options?: ReadUrlOptions): Promise<ReadUrlResponse>;
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
// (undocumented)
search(url: string, options?: SearchOptions): Promise<SearchResponse>;
// (undocumented)
@@ -418,6 +419,7 @@ export type ReadTreeOptions = {
},
): boolean;
etag?: string;
signal?: AbortSignal_2;
};
// @public
@@ -471,6 +473,7 @@ export type ReadTreeResponseFile = {
// @public
export type ReadUrlOptions = {
etag?: string;
signal?: AbortSignal_2;
};
// @public
@@ -508,6 +511,7 @@ export type RunContainerOptions = {
// @public
export type SearchOptions = {
etag?: string;
signal?: AbortSignal_2;
};
// @public
packages/backend-common/package.json
+1
View File
@@ -62,6 +62,7 @@
"minimatch": "^3.0.4",
"minimist": "^1.2.5",
"morgan": "^1.10.0",
"node-abort-controller": "^3.0.1",
"raw-body": "^2.4.1",
"selfsigned": "^1.10.7",
"stoppable": "^1.1.0",
packages/backend-common/src/reading/AwsS3UrlReader.ts
+17 -12
View File
@@ -18,6 +18,7 @@ import aws, { Credentials, S3 } from 'aws-sdk';
import { CredentialsOptions } from 'aws-sdk/lib/credentials';
import {
ReaderFactory,
ReadTreeOptions,
ReadTreeResponse,
ReadTreeResponseFactory,
ReadUrlOptions,
@@ -92,7 +93,7 @@ export class AwsS3UrlReader implements UrlReader {
) {}
/**
* If accesKeyId and secretAccessKey are missing, the standard credentials provider chain will be used:
* If accessKeyId and secretAccessKey are missing, the standard credentials provider chain will be used:
* https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
*/
private static buildCredentials(
@@ -154,9 +155,10 @@ export class AwsS3UrlReader implements UrlReader {
};
}
const response = this.deps.s3.getObject(params);
const buffer = await getRawBody(response.createReadStream());
const etag = (await response.promise()).ETag;
const request = this.deps.s3.getObject(params);
options?.signal?.addEventListener('abort', () => request.abort());
const buffer = await getRawBody(request.createReadStream());
const etag = (await request.promise()).ETag;
return {
buffer: async () => buffer,
@@ -171,7 +173,10 @@ export class AwsS3UrlReader implements UrlReader {
}
}
async readTree(url: string): Promise<ReadTreeResponse> {
async readTree(
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
try {
const { path, bucket, region } = parseURL(url);
const allObjects: ObjectList = [];
@@ -180,13 +185,13 @@ export class AwsS3UrlReader implements UrlReader {
let output: ListObjectsV2Output;
do {
aws.config.update({ region: region });
output = await this.deps.s3
.listObjectsV2({
Bucket: bucket,
ContinuationToken: continuationToken,
Prefix: path,
})
.promise();
const request = this.deps.s3.listObjectsV2({
Bucket: bucket,
ContinuationToken: continuationToken,
Prefix: path,
});
options?.signal?.addEventListener('abort', () => request.abort());
output = await request.promise();
if (output.Contents) {
output.Contents.forEach(contents => {
allObjects.push(contents);
packages/backend-common/src/reading/AzureUrlReader.ts
+27 -20
View File
@@ -55,21 +55,34 @@ export class AzureUrlReader implements UrlReader {
) {}
async read(url: string): Promise<Buffer> {
const response = await this.readUrl(url);
return response.buffer();
}
async readUrl(
url: string,
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
// TODO: etag is not implemented yet.
const { signal } = options ?? {};
const builtUrl = getAzureFileFetchUrl(url);
let response: Response;
try {
response = await fetch(
builtUrl,
getAzureRequestOptions(this.integration.config),
);
response = await fetch(builtUrl, {
...getAzureRequestOptions(this.integration.config),
...(signal && { signal }),
});
} catch (e) {
throw new Error(`Unable to read ${url}, ${e}`);
}
// for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html
if (response.ok && response.status !== 203) {
return Buffer.from(await response.arrayBuffer());
return {
buffer: async () => Buffer.from(await response.arrayBuffer()),
};
}
const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`;
@@ -79,19 +92,12 @@ export class AzureUrlReader implements UrlReader {
throw new Error(message);
}
async readUrl(
url: string,
_options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
// TODO etag is not implemented yet.
const buffer = await this.read(url);
return { buffer: async () => buffer };
}
async readTree(
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const { etag, filter, signal } = options ?? {};
// TODO: Support filepath based reading tree feature like other providers
// Get latest commit SHA
@@ -109,16 +115,16 @@ export class AzureUrlReader implements UrlReader {
}
const commitSha = (await commitsAzureResponse.json()).value[0].commitId;
if (options?.etag && options.etag === commitSha) {
if (etag && etag === commitSha) {
throw new NotModifiedError();
}
const archiveAzureResponse = await fetch(
getAzureDownloadUrl(url),
getAzureRequestOptions(this.integration.config, {
const archiveAzureResponse = await fetch(getAzureDownloadUrl(url), {
...getAzureRequestOptions(this.integration.config, {
Accept: 'application/zip',
}),
);
...(signal && { signal }),
});
if (!archiveAzureResponse.ok) {
const message = `Failed to read tree from ${url}, ${archiveAzureResponse.status} ${archiveAzureResponse.statusText}`;
if (archiveAzureResponse.status === 404) {
@@ -139,7 +145,7 @@ export class AzureUrlReader implements UrlReader {
return await this.deps.treeResponseFactory.fromZipArchive({
stream: archiveAzureResponse.body as unknown as Readable,
etag: commitSha,
filter: options?.filter,
filter,
subpath,
});
}
@@ -158,6 +164,7 @@ export class AzureUrlReader implements UrlReader {
const tree = await this.readTree(treeUrl.toString(), {
etag: options?.etag,
signal: options?.signal,
filter: p => (matcher ? matcher.match(p) : true),
});
const files = await tree.files();
packages/backend-common/src/reading/BitbucketUrlReader.ts
+18 -12
View File
@@ -77,18 +77,33 @@ export class BitbucketUrlReader implements UrlReader {
}
async read(url: string): Promise<Buffer> {
const response = await this.readUrl(url);
return response.buffer();
}
async readUrl(
url: string,
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
// TODO: etag is not supported yet
const { signal } = options ?? {};
const bitbucketUrl = getBitbucketFileFetchUrl(url, this.integration.config);
const options = getBitbucketRequestOptions(this.integration.config);
const requestOptions = getBitbucketRequestOptions(this.integration.config);
let response: Response;
try {
response = await fetch(bitbucketUrl.toString(), options);
response = await fetch(bitbucketUrl.toString(), {
...requestOptions,
...(signal && { signal }),
});
} catch (e) {
throw new Error(`Unable to read ${url}, ${e}`);
}
if (response.ok) {
return Buffer.from(await response.arrayBuffer());
return {
buffer: async () => Buffer.from(await response.arrayBuffer()),
};
}
const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`;
@@ -98,15 +113,6 @@ export class BitbucketUrlReader implements UrlReader {
throw new Error(message);
}
async readUrl(
url: string,
_options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
// TODO etag is not implemented yet.
const buffer = await this.read(url);
return { buffer: async () => buffer };
}
async readTree(
url: string,
options?: ReadTreeOptions,
packages/backend-common/src/reading/FetchUrlReader.ts
+1
View File
@@ -86,6 +86,7 @@ export class FetchUrlReader implements UrlReader {
headers: {
...(options?.etag && { 'If-None-Match': options.etag }),
},
signal: options?.signal,
});
} catch (e) {
throw new Error(`Unable to read ${url}, ${e}`);
packages/backend-common/src/reading/GithubUrlReader.ts
+3 -2
View File
@@ -110,6 +110,7 @@ export class GithubUrlReader implements UrlReader {
...(options?.etag && { 'If-None-Match': options.etag }),
Accept: 'application/vnd.github.v3.raw',
},
signal: options?.signal,
});
} catch (e) {
throw new Error(`Unable to read ${url}, ${e}`);
@@ -164,7 +165,7 @@ export class GithubUrlReader implements UrlReader {
repoDetails.repo.archive_url,
commitSha,
filepath,
{ headers },
{ headers, signal: options?.signal },
options,
);
}
@@ -188,7 +189,7 @@ export class GithubUrlReader implements UrlReader {
repoDetails.repo.archive_url,
commitSha,
filepath,
{ headers },
{ headers, signal: options?.signal },
);
return { files, etag: commitSha };
packages/backend-common/src/reading/GitlabUrlReader.ts
+14 -4
View File
@@ -66,6 +66,7 @@ export class GitlabUrlReader implements UrlReader {
url: string,
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
const { etag, signal } = options ?? {};
const builtUrl = await getGitLabFileFetchUrl(url, this.integration.config);
let response: Response;
@@ -73,8 +74,9 @@ export class GitlabUrlReader implements UrlReader {
response = await fetch(builtUrl, {
headers: {
...getGitLabRequestOptions(this.integration.config).headers,
...(options?.etag && { 'If-None-Match': options.etag }),
...(etag && { 'If-None-Match': etag }),
},
...(signal && { signal }),
});
} catch (e) {
throw new Error(`Unable to read ${url}, ${e}`);
@@ -102,6 +104,7 @@ export class GitlabUrlReader implements UrlReader {
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const { etag, signal } = options ?? {};
const { ref, full_name, filepath } = parseGitUrl(url);
// Use GitLab API to get the default branch
@@ -140,7 +143,10 @@ export class GitlabUrlReader implements UrlReader {
full_name,
)}/repository/commits?${commitsReqParams.toString()}`,
).toString(),
getGitLabRequestOptions(this.integration.config),
{
...getGitLabRequestOptions(this.integration.config),
...(signal && { signal }),
},
);
if (!commitsGitlabResponse.ok) {
const message = `Failed to read tree (branch) from ${url}, ${commitsGitlabResponse.status} ${commitsGitlabResponse.statusText}`;
@@ -152,7 +158,7 @@ export class GitlabUrlReader implements UrlReader {
const commitSha = (await commitsGitlabResponse.json())[0].id;
if (options?.etag && options.etag === commitSha) {
if (etag && etag === commitSha) {
throw new NotModifiedError();
}
@@ -161,7 +167,10 @@ export class GitlabUrlReader implements UrlReader {
`${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent(
full_name,
)}/repository/archive?sha=${branch}`,
getGitLabRequestOptions(this.integration.config),
{
...getGitLabRequestOptions(this.integration.config),
...(signal && { signal }),
},
);
if (!archiveGitLabResponse.ok) {
const message = `Failed to read tree (archive) from ${url}, ${archiveGitLabResponse.status} ${archiveGitLabResponse.statusText}`;
@@ -191,6 +200,7 @@ export class GitlabUrlReader implements UrlReader {
const tree = await this.readTree(treeUrl, {
etag: options?.etag,
signal: options?.signal,
filter: path => matcher.match(stripFirstDirectoryFromPath(path)),
});
const files = await tree.files();
packages/backend-common/src/reading/types.ts
+28
View File
@@ -17,6 +17,7 @@
import { Readable } from 'stream';
import { Logger } from 'winston';
import { Config } from '@backstage/config';
import { AbortSignal } from 'node-abort-controller';
/**
* A generic interface for fetching plain data from URLs.
@@ -101,6 +102,15 @@ export type ReadUrlOptions = {
* of the response along with a new ETag.
*/
etag?: string;
/**
* An abort signal to pass down to the underlying request.
*
* @remarks
*
* Not all reader implementations may take this field into account.
*/
signal?: AbortSignal;
};
/**
@@ -165,6 +175,15 @@ export type ReadTreeOptions = {
* rest of the response along with a new ETag.
*/
etag?: string;
/**
* An abort signal to pass down to the underlying request.
*
* @remarks
*
* Not all reader implementations may take this field into account.
*/
signal?: AbortSignal;
};
/**
@@ -291,6 +310,15 @@ export type SearchOptions = {
* search will return the rest of SearchResponse along with a new etag.
*/
etag?: string;
/**
* An abort signal to pass down to the underlying request.
*
* @remarks
*
* Not all reader implementations may take this field into account.
*/
signal?: AbortSignal;
};
/**