moved over the urlreader to backend-defaults
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
@@ -49,26 +49,32 @@ import { PluginMetadataService } from '@backstage/backend-plugin-api';
|
||||
import { PushResult } from 'isomorphic-git';
|
||||
import { Readable } from 'stream';
|
||||
import { ReadCommitResult } from 'isomorphic-git';
|
||||
import { ReadTreeOptions } from '@backstage/backend-plugin-api';
|
||||
import { ReadTreeResponse } from '@backstage/backend-plugin-api';
|
||||
import { ReadTreeResponseDirOptions } from '@backstage/backend-plugin-api';
|
||||
import { ReadTreeResponseFile } from '@backstage/backend-plugin-api';
|
||||
import { ReadUrlOptions } from '@backstage/backend-plugin-api';
|
||||
import { ReadUrlResponse } from '@backstage/backend-plugin-api';
|
||||
import type { ReadTreeOptions as ReadTreeOptions_2 } from '@backstage/backend-plugin-api';
|
||||
import type { ReadTreeResponse as ReadTreeResponse_2 } from '@backstage/backend-plugin-api';
|
||||
import type { ReadTreeResponseDirOptions as ReadTreeResponseDirOptions_2 } from '@backstage/backend-plugin-api';
|
||||
import type { ReadTreeResponseFile as ReadTreeResponseFile_2 } from '@backstage/backend-plugin-api';
|
||||
import type { ReadUrlOptions as ReadUrlOptions_2 } from '@backstage/backend-plugin-api';
|
||||
import type { ReadUrlResponse as ReadUrlResponse_2 } from '@backstage/backend-plugin-api';
|
||||
import { RequestHandler } from 'express';
|
||||
import { resolvePackagePath as resolvePackagePath_2 } from '@backstage/backend-plugin-api';
|
||||
import { resolveSafeChildPath as resolveSafeChildPath_2 } from '@backstage/backend-plugin-api';
|
||||
import { RootConfigService } from '@backstage/backend-plugin-api';
|
||||
import { Router } from 'express';
|
||||
import { SchedulerService } from '@backstage/backend-plugin-api';
|
||||
import { SearchOptions } from '@backstage/backend-plugin-api';
|
||||
import { SearchResponse } from '@backstage/backend-plugin-api';
|
||||
import { SearchResponseFile } from '@backstage/backend-plugin-api';
|
||||
import type { SearchOptions as SearchOptions_2 } from '@backstage/backend-plugin-api';
|
||||
import type { SearchResponse as SearchResponse_2 } from '@backstage/backend-plugin-api';
|
||||
import type { SearchResponseFile as SearchResponseFile_2 } from '@backstage/backend-plugin-api';
|
||||
import { Server } from 'http';
|
||||
import { ServiceRef } from '@backstage/backend-plugin-api';
|
||||
import { TokenManagerService as TokenManager } from '@backstage/backend-plugin-api';
|
||||
import { TransportStreamOptions } from 'winston-transport';
|
||||
import { UrlReaderService as UrlReader } from '@backstage/backend-plugin-api';
|
||||
import { UrlReaderReadTreeOptions } from '@backstage/backend-plugin-api';
|
||||
import { UrlReaderReadTreeResponse } from '@backstage/backend-plugin-api';
|
||||
import { UrlReaderReadUrlOptions } from '@backstage/backend-plugin-api';
|
||||
import { UrlReaderReadUrlResponse } from '@backstage/backend-plugin-api';
|
||||
import { UrlReaderSearchOptions } from '@backstage/backend-plugin-api';
|
||||
import { UrlReaderSearchResponse } from '@backstage/backend-plugin-api';
|
||||
import { UrlReaderService } from '@backstage/backend-plugin-api';
|
||||
import { UserInfoService } from '@backstage/backend-plugin-api';
|
||||
import { V1PodTemplateSpec } from '@kubernetes/client-node';
|
||||
import * as winston from 'winston';
|
||||
@@ -80,118 +86,30 @@ export type AuthCallbackOptions = {
|
||||
logger?: LoggerService;
|
||||
};
|
||||
|
||||
// @public
|
||||
export class AwsS3UrlReader implements UrlReader {
|
||||
constructor(
|
||||
credsManager: AwsCredentialsManager,
|
||||
integration: AwsS3Integration,
|
||||
deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
);
|
||||
// (undocumented)
|
||||
static factory: ReaderFactory;
|
||||
// (undocumented)
|
||||
read(url: string): Promise<Buffer>;
|
||||
// (undocumented)
|
||||
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
|
||||
// (undocumented)
|
||||
search(): Promise<SearchResponse>;
|
||||
// (undocumented)
|
||||
toString(): string;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "AwsS3UrlReader_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const AwsS3UrlReader: typeof AwsS3UrlReader_2;
|
||||
|
||||
// @public
|
||||
export class AzureUrlReader implements UrlReader {
|
||||
constructor(
|
||||
integration: AzureIntegration,
|
||||
deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
credentialsProvider: AzureDevOpsCredentialsProvider;
|
||||
},
|
||||
);
|
||||
// (undocumented)
|
||||
static factory: ReaderFactory;
|
||||
// (undocumented)
|
||||
read(url: string): Promise<Buffer>;
|
||||
// (undocumented)
|
||||
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
|
||||
// (undocumented)
|
||||
search(url: string, options?: SearchOptions): Promise<SearchResponse>;
|
||||
// (undocumented)
|
||||
toString(): string;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "AzureUrlReader_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const AzureUrlReader: typeof AzureUrlReader_2;
|
||||
|
||||
// @public
|
||||
export class BitbucketCloudUrlReader implements UrlReader {
|
||||
constructor(
|
||||
integration: BitbucketCloudIntegration,
|
||||
deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
);
|
||||
// (undocumented)
|
||||
static factory: ReaderFactory;
|
||||
// (undocumented)
|
||||
read(url: string): Promise<Buffer>;
|
||||
// (undocumented)
|
||||
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
|
||||
// (undocumented)
|
||||
search(url: string, options?: SearchOptions): Promise<SearchResponse>;
|
||||
// (undocumented)
|
||||
toString(): string;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "BitbucketCloudUrlReader_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const BitbucketCloudUrlReader: typeof BitbucketCloudUrlReader_2;
|
||||
|
||||
// @public
|
||||
export class BitbucketServerUrlReader implements UrlReader {
|
||||
constructor(
|
||||
integration: BitbucketServerIntegration,
|
||||
deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
);
|
||||
// (undocumented)
|
||||
static factory: ReaderFactory;
|
||||
// (undocumented)
|
||||
read(url: string): Promise<Buffer>;
|
||||
// (undocumented)
|
||||
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
|
||||
// (undocumented)
|
||||
search(url: string, options?: SearchOptions): Promise<SearchResponse>;
|
||||
// (undocumented)
|
||||
toString(): string;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "BitbucketServerUrlReader_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const BitbucketServerUrlReader: typeof BitbucketServerUrlReader_2;
|
||||
|
||||
// @public @deprecated
|
||||
export class BitbucketUrlReader implements UrlReader {
|
||||
constructor(
|
||||
integration: BitbucketIntegration,
|
||||
logger: LoggerService,
|
||||
deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
);
|
||||
// (undocumented)
|
||||
static factory: ReaderFactory;
|
||||
// (undocumented)
|
||||
read(url: string): Promise<Buffer>;
|
||||
// (undocumented)
|
||||
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
|
||||
// (undocumented)
|
||||
search(url: string, options?: SearchOptions): Promise<SearchResponse>;
|
||||
// (undocumented)
|
||||
toString(): string;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "BitbucketUrlReader_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const BitbucketUrlReader: typeof BitbucketUrlReader_2;
|
||||
|
||||
// @public @deprecated (undocumented)
|
||||
export type CacheClient = CacheService;
|
||||
@@ -330,50 +248,20 @@ export type ErrorHandlerOptions = {
|
||||
logClientErrors?: boolean;
|
||||
};
|
||||
|
||||
// @public
|
||||
export class FetchUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory;
|
||||
// (undocumented)
|
||||
read(url: string): Promise<Buffer>;
|
||||
// (undocumented)
|
||||
readTree(): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
|
||||
// (undocumented)
|
||||
search(): Promise<SearchResponse>;
|
||||
// (undocumented)
|
||||
toString(): string;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "FetchUrlReader_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const FetchUrlReader: typeof FetchUrlReader_2;
|
||||
|
||||
// @public
|
||||
export type FromReadableArrayOptions = Array<{
|
||||
data: Readable;
|
||||
path: string;
|
||||
lastModifiedAt?: Date;
|
||||
}>;
|
||||
// Warning: (ae-forgotten-export) The symbol "FromReadableArrayOptions_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export type FromReadableArrayOptions = FromReadableArrayOptions_2;
|
||||
|
||||
// @public
|
||||
export class GerritUrlReader implements UrlReader {
|
||||
constructor(
|
||||
integration: GerritIntegration,
|
||||
deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
workDir: string,
|
||||
);
|
||||
// (undocumented)
|
||||
static factory: ReaderFactory;
|
||||
// (undocumented)
|
||||
read(url: string): Promise<Buffer>;
|
||||
// (undocumented)
|
||||
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
|
||||
// (undocumented)
|
||||
search(): Promise<SearchResponse>;
|
||||
// (undocumented)
|
||||
toString(): string;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "GerritUrlReader_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const GerritUrlReader: typeof GerritUrlReader_2;
|
||||
|
||||
// @public @deprecated
|
||||
export function getRootLogger(): winston.Logger;
|
||||
@@ -457,94 +345,25 @@ export class Git {
|
||||
resolveRef(options: { dir: string; ref: string }): Promise<string>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class GiteaUrlReader implements UrlReader {
|
||||
constructor(
|
||||
integration: GiteaIntegration,
|
||||
deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
);
|
||||
// (undocumented)
|
||||
static factory: ReaderFactory;
|
||||
// (undocumented)
|
||||
read(url: string): Promise<Buffer>;
|
||||
// (undocumented)
|
||||
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
|
||||
// (undocumented)
|
||||
search(): Promise<SearchResponse>;
|
||||
// (undocumented)
|
||||
toString(): string;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "GiteaUrlReader_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const GiteaUrlReader: typeof GiteaUrlReader_2;
|
||||
|
||||
// @public
|
||||
export class GithubUrlReader implements UrlReader {
|
||||
constructor(
|
||||
integration: GithubIntegration,
|
||||
deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
credentialsProvider: GithubCredentialsProvider;
|
||||
},
|
||||
);
|
||||
// (undocumented)
|
||||
static factory: ReaderFactory;
|
||||
// (undocumented)
|
||||
read(url: string): Promise<Buffer>;
|
||||
// (undocumented)
|
||||
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
|
||||
// (undocumented)
|
||||
search(url: string, options?: SearchOptions): Promise<SearchResponse>;
|
||||
// (undocumented)
|
||||
toString(): string;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "GithubUrlReader_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const GithubUrlReader: typeof GithubUrlReader_2;
|
||||
|
||||
// @public
|
||||
export class GitlabUrlReader implements UrlReader {
|
||||
constructor(
|
||||
integration: GitLabIntegration,
|
||||
deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
);
|
||||
// (undocumented)
|
||||
static factory: ReaderFactory;
|
||||
// (undocumented)
|
||||
read(url: string): Promise<Buffer>;
|
||||
// (undocumented)
|
||||
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
|
||||
// (undocumented)
|
||||
search(url: string, options?: SearchOptions): Promise<SearchResponse>;
|
||||
// (undocumented)
|
||||
toString(): string;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "GitlabUrlReader_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const GitlabUrlReader: typeof GitlabUrlReader_2;
|
||||
|
||||
// @public
|
||||
export class HarnessUrlReader implements UrlReader {
|
||||
constructor(
|
||||
integration: HarnessIntegration,
|
||||
deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
);
|
||||
// (undocumented)
|
||||
static factory: ReaderFactory;
|
||||
// (undocumented)
|
||||
read(url: string): Promise<Buffer>;
|
||||
// (undocumented)
|
||||
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
|
||||
// (undocumented)
|
||||
search(): Promise<SearchResponse>;
|
||||
// (undocumented)
|
||||
toString(): string;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "HarnessUrlReader_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const HarnessUrlReader: typeof HarnessUrlReader_2;
|
||||
|
||||
// @public @deprecated
|
||||
export const HostDiscovery: typeof HostDiscovery_2;
|
||||
@@ -596,7 +415,7 @@ export const legacyPlugin: (
|
||||
permissions: PermissionsService;
|
||||
scheduler: SchedulerService;
|
||||
tokenManager: TokenManager;
|
||||
reader: UrlReader;
|
||||
reader: UrlReaderService;
|
||||
identity: IdentityService;
|
||||
},
|
||||
{
|
||||
@@ -677,73 +496,49 @@ export interface PullOptions {
|
||||
};
|
||||
}
|
||||
|
||||
// @public
|
||||
export type ReaderFactory = (options: {
|
||||
config: Config;
|
||||
logger: LoggerService;
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
}) => UrlReaderPredicateTuple[];
|
||||
// Warning: (ae-forgotten-export) The symbol "ReaderFactory_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export type ReaderFactory = ReaderFactory_2;
|
||||
|
||||
export { ReadTreeOptions };
|
||||
// @public @deprecated (undocumented)
|
||||
export type ReadTreeOptions = ReadTreeOptions_2;
|
||||
|
||||
export { ReadTreeResponse };
|
||||
// @public @deprecated (undocumented)
|
||||
export type ReadTreeResponse = ReadTreeResponse_2;
|
||||
|
||||
export { ReadTreeResponseDirOptions };
|
||||
// @public @deprecated (undocumented)
|
||||
export type ReadTreeResponseDirOptions = ReadTreeResponseDirOptions_2;
|
||||
|
||||
// @public
|
||||
export interface ReadTreeResponseFactory {
|
||||
// (undocumented)
|
||||
fromReadableArray(
|
||||
options: FromReadableArrayOptions,
|
||||
): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
fromTarArchive(
|
||||
options: ReadTreeResponseFactoryOptions & {
|
||||
stripFirstDirectory?: boolean;
|
||||
},
|
||||
): Promise<ReadTreeResponse>;
|
||||
// (undocumented)
|
||||
fromZipArchive(
|
||||
options: ReadTreeResponseFactoryOptions,
|
||||
): Promise<ReadTreeResponse>;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "ReadTreeResponseFactory_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export type ReadTreeResponseFactory = ReadTreeResponseFactory_2;
|
||||
|
||||
// @public
|
||||
export type ReadTreeResponseFactoryOptions = {
|
||||
stream: Readable;
|
||||
subpath?: string;
|
||||
etag: string;
|
||||
filter?: (
|
||||
path: string,
|
||||
info?: {
|
||||
size: number;
|
||||
},
|
||||
) => boolean;
|
||||
};
|
||||
// Warning: (ae-forgotten-export) The symbol "ReadTreeResponseFactoryOptions_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export type ReadTreeResponseFactoryOptions = ReadTreeResponseFactoryOptions_2;
|
||||
|
||||
export { ReadTreeResponseFile };
|
||||
// @public @deprecated (undocumented)
|
||||
export type ReadTreeResponseFile = ReadTreeResponseFile_2;
|
||||
|
||||
export { ReadUrlOptions };
|
||||
// @public @deprecated (undocumented)
|
||||
export type ReadUrlOptions = ReadUrlOptions_2;
|
||||
|
||||
export { ReadUrlResponse };
|
||||
// @public @deprecated (undocumented)
|
||||
export type ReadUrlResponse = ReadUrlResponse_2;
|
||||
|
||||
// @public
|
||||
export class ReadUrlResponseFactory {
|
||||
static fromNodeJSReadable(
|
||||
oldStyleStream: NodeJS.ReadableStream,
|
||||
options?: ReadUrlResponseFactoryFromStreamOptions,
|
||||
): Promise<ReadUrlResponse>;
|
||||
static fromReadable(
|
||||
stream: Readable,
|
||||
options?: ReadUrlResponseFactoryFromStreamOptions,
|
||||
): Promise<ReadUrlResponse>;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "ReadUrlResponseFactory_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const ReadUrlResponseFactory: typeof ReadUrlResponseFactory_2;
|
||||
|
||||
// @public
|
||||
export type ReadUrlResponseFactoryFromStreamOptions = {
|
||||
etag?: string;
|
||||
lastModifiedAt?: Date;
|
||||
};
|
||||
// Warning: (ae-forgotten-export) The symbol "ReadUrlResponseFactoryFromStreamOptions_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export type ReadUrlResponseFactoryFromStreamOptions =
|
||||
ReadUrlResponseFactoryFromStreamOptions_2;
|
||||
|
||||
// @public
|
||||
export function redactWinstonLogLine(
|
||||
@@ -778,11 +573,14 @@ export type RunContainerOptions = {
|
||||
pullOptions?: PullOptions;
|
||||
};
|
||||
|
||||
export { SearchOptions };
|
||||
// @public @deprecated (undocumented)
|
||||
export type SearchOptions = SearchOptions_2;
|
||||
|
||||
export { SearchResponse };
|
||||
// @public @deprecated (undocumented)
|
||||
export type SearchResponse = SearchResponse_2;
|
||||
|
||||
export { SearchResponseFile };
|
||||
// @public @deprecated (undocumented)
|
||||
export type SearchResponseFile = SearchResponseFile_2;
|
||||
|
||||
// @public
|
||||
export class ServerTokenManager implements TokenManager {
|
||||
@@ -861,26 +659,23 @@ export interface StatusCheckHandlerOptions {
|
||||
|
||||
export { TokenManager };
|
||||
|
||||
export { UrlReader };
|
||||
// @public @deprecated (undocumented)
|
||||
export type UrlReader = UrlReaderService;
|
||||
|
||||
// @public
|
||||
export type UrlReaderPredicateTuple = {
|
||||
predicate: (url: URL) => boolean;
|
||||
reader: UrlReader;
|
||||
};
|
||||
// Warning: (ae-forgotten-export) The symbol "UrlReaderPredicateTuple_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export type UrlReaderPredicateTuple = UrlReaderPredicateTuple_2;
|
||||
|
||||
// @public
|
||||
export class UrlReaders {
|
||||
static create(options: UrlReadersOptions): UrlReader;
|
||||
static default(options: UrlReadersOptions): UrlReader;
|
||||
}
|
||||
// Warning: (ae-forgotten-export) The symbol "UrlReaders_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export const UrlReaders: typeof UrlReaders_2;
|
||||
|
||||
// @public
|
||||
export type UrlReadersOptions = {
|
||||
config: Config;
|
||||
logger: LoggerService;
|
||||
factories?: ReaderFactory[];
|
||||
};
|
||||
// Warning: (ae-forgotten-export) The symbol "UrlReadersOptions_2" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// @public @deprecated (undocumented)
|
||||
export type UrlReadersOptions = UrlReadersOptions_2;
|
||||
|
||||
// @public @deprecated
|
||||
export function useHotCleanup(
|
||||
|
||||
@@ -1,611 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { JsonObject } from '@backstage/types';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
import { AwsCodeCommitUrlReader, parseUrl } from './AwsCodeCommitUrlReader';
|
||||
import { UrlReaderPredicateTuple } from './types';
|
||||
import path from 'path';
|
||||
import { NotModifiedError } from '@backstage/errors';
|
||||
import { mockClient } from 'aws-sdk-client-mock';
|
||||
import {
|
||||
CodeCommitClient,
|
||||
GetFileCommand,
|
||||
GetFolderCommand,
|
||||
} from '@aws-sdk/client-codecommit';
|
||||
import fs from 'fs';
|
||||
import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node';
|
||||
import {
|
||||
AwsCodeCommitIntegration,
|
||||
readAwsCodeCommitIntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
import { mockServices } from '@backstage/backend-test-utils';
|
||||
|
||||
const AMAZON_AWS_CODECOMMIT_HOST = 'console.aws.amazon.com';
|
||||
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
});
|
||||
|
||||
describe('parseUrl', () => {
|
||||
it('supports all formats (requireGitPath = true)', () => {
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml?region=eu-west-1',
|
||||
true,
|
||||
),
|
||||
).toEqual({
|
||||
path: 'catalog-info.yaml',
|
||||
repositoryName: 'my-repo',
|
||||
region: 'eu-west-1',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://us-east-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo2/browse/--/test1/test2/catalog-info.yaml',
|
||||
true,
|
||||
),
|
||||
).toEqual({
|
||||
path: 'test1/test2/catalog-info.yaml',
|
||||
repositoryName: 'my-repo2',
|
||||
region: 'us-east-1',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-repo/browse/refs/heads/develop/--/some-path/catalog-info.yaml',
|
||||
true,
|
||||
),
|
||||
).toEqual({
|
||||
path: 'some-path/catalog-info.yaml',
|
||||
repositoryName: 'my-test-repo',
|
||||
region: 'eu-west-1',
|
||||
commitSpecifier: 'refs/heads/develop',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-repo/browse/refs/tags/v1.0/--/some-path/catalog-info.yaml',
|
||||
true,
|
||||
),
|
||||
).toEqual({
|
||||
path: 'some-path/catalog-info.yaml',
|
||||
repositoryName: 'my-test-repo',
|
||||
region: 'eu-west-1',
|
||||
commitSpecifier: 'refs/tags/v1.0',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-repo/browse/c1234/--/some-path/catalog-info.yaml',
|
||||
true,
|
||||
),
|
||||
).toEqual({
|
||||
path: 'some-path/catalog-info.yaml',
|
||||
repositoryName: 'my-test-repo',
|
||||
region: 'eu-west-1',
|
||||
commitSpecifier: 'c1234',
|
||||
});
|
||||
});
|
||||
it('throw correct error when not providing full url to file (requireGitPath = true)', () => {
|
||||
expect(() =>
|
||||
parseUrl(
|
||||
`https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo`,
|
||||
true,
|
||||
),
|
||||
).toThrow('Please provide full path to yaml file from CodeCommit');
|
||||
});
|
||||
it('supports all formats (requireGitPath = false)', () => {
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-techdocs',
|
||||
),
|
||||
).toEqual({
|
||||
path: '/',
|
||||
repositoryName: 'my-test-techdocs',
|
||||
region: 'eu-west-1',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-techdocs/',
|
||||
),
|
||||
).toEqual({
|
||||
path: '/',
|
||||
repositoryName: 'my-test-techdocs',
|
||||
region: 'eu-west-1',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-techdocs/browse?region=eu-west-1',
|
||||
),
|
||||
).toEqual({
|
||||
path: '/',
|
||||
repositoryName: 'my-test-techdocs',
|
||||
region: 'eu-west-1',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-techdocs/browse/refs/heads/develop',
|
||||
),
|
||||
).toEqual({
|
||||
path: '/',
|
||||
repositoryName: 'my-test-techdocs',
|
||||
region: 'eu-west-1',
|
||||
commitSpecifier: 'refs/heads/develop',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-techdocs/browse/refs/tags/v1.0',
|
||||
),
|
||||
).toEqual({
|
||||
path: '/',
|
||||
repositoryName: 'my-test-techdocs',
|
||||
region: 'eu-west-1',
|
||||
commitSpecifier: 'refs/tags/v1.0',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-techdocs/browse/c1234',
|
||||
),
|
||||
).toEqual({
|
||||
path: '/',
|
||||
repositoryName: 'my-test-techdocs',
|
||||
region: 'eu-west-1',
|
||||
commitSpecifier: 'c1234',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-techdocs/browse/c1234/',
|
||||
),
|
||||
).toEqual({
|
||||
path: '/',
|
||||
repositoryName: 'my-test-techdocs',
|
||||
region: 'eu-west-1',
|
||||
commitSpecifier: 'c1234',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-techdocs/browse/c1234/--/folder/',
|
||||
),
|
||||
).toEqual({
|
||||
path: 'folder/',
|
||||
repositoryName: 'my-test-techdocs',
|
||||
region: 'eu-west-1',
|
||||
commitSpecifier: 'c1234',
|
||||
});
|
||||
});
|
||||
|
||||
it('throw correct error when providing edit url', () => {
|
||||
expect(() =>
|
||||
parseUrl(
|
||||
`https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/files/edit/refs/heads/develop/--/test/catalog-info.yaml`,
|
||||
),
|
||||
).toThrow(
|
||||
'Please provide the view url to yaml file from CodeCommit, not the edit url',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AwsCodeCommitUrlReader', () => {
|
||||
const codeCommitClient = mockClient(CodeCommitClient);
|
||||
|
||||
const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
|
||||
return AwsCodeCommitUrlReader.factory({
|
||||
config: new ConfigReader(config),
|
||||
logger: mockServices.logger.mock(),
|
||||
treeResponseFactory,
|
||||
});
|
||||
};
|
||||
|
||||
it('creates a sample reader without the awsCodeCommit field', () => {
|
||||
const entries = createReader({
|
||||
integrations: {},
|
||||
});
|
||||
|
||||
expect(entries).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('creates a reader with credentials correctly configured', () => {
|
||||
const awsCodeCommitIntegrations = [];
|
||||
awsCodeCommitIntegrations.push({
|
||||
host: AMAZON_AWS_CODECOMMIT_HOST,
|
||||
accessKeyId: 'fakekey',
|
||||
secretAccessKey: 'fakekey',
|
||||
region: 'fakeregion',
|
||||
});
|
||||
|
||||
const entries = createReader({
|
||||
integrations: {
|
||||
awsCodeCommit: awsCodeCommitIntegrations,
|
||||
},
|
||||
});
|
||||
|
||||
expect(entries).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('creates a reader with default credentials provider', () => {
|
||||
const awsCodeCommitIntegrations = [];
|
||||
awsCodeCommitIntegrations.push({
|
||||
host: AMAZON_AWS_CODECOMMIT_HOST,
|
||||
region: 'fakeregion',
|
||||
});
|
||||
|
||||
const entries = createReader({
|
||||
integrations: {
|
||||
awsCodeCommit: awsCodeCommitIntegrations,
|
||||
},
|
||||
});
|
||||
|
||||
expect(entries).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('creates a reader without a region', () => {
|
||||
const awsCodeCommitIntegrations: any[] = [];
|
||||
awsCodeCommitIntegrations.push({
|
||||
accessKeyId: 'fakekey',
|
||||
secretAccessKey: 'fakekey',
|
||||
});
|
||||
|
||||
expect(() => {
|
||||
createReader({
|
||||
integrations: {
|
||||
awsCodeCommit: awsCodeCommitIntegrations,
|
||||
},
|
||||
});
|
||||
}).toThrow(
|
||||
"Missing required config value at 'integrations.awsCodeCommit[0].region' in 'mock-config'",
|
||||
);
|
||||
});
|
||||
|
||||
describe('predicates', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
awsCodeCommit: [
|
||||
{
|
||||
region: 'fakeregion',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const predicate = readers[0].predicate;
|
||||
|
||||
it('returns true for the correct aws CodeCommit storage host', () => {
|
||||
expect(
|
||||
predicate(
|
||||
new URL(
|
||||
'https://fakeregion.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo',
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for a url with the full path and the correct host', () => {
|
||||
expect(
|
||||
predicate(
|
||||
new URL(
|
||||
'https://fakeregion.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml?region=eu-west-1',
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for a url with the full path and the correct host when the host overrules the region', () => {
|
||||
const predicateWithHost = createReader({
|
||||
integrations: {
|
||||
awsCodeCommit: [
|
||||
{
|
||||
host: 'fakehost',
|
||||
region: 'fakeregion',
|
||||
},
|
||||
],
|
||||
},
|
||||
})[0].predicate;
|
||||
|
||||
expect(
|
||||
predicateWithHost(
|
||||
new URL(
|
||||
'https://fakehost/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml?region=eu-west-1',
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for a url with the full path and the wrong host when the host overrules the region', () => {
|
||||
const predicateWithHost = createReader({
|
||||
integrations: {
|
||||
awsCodeCommit: [
|
||||
{
|
||||
host: 'fakehost',
|
||||
region: 'fakeregion',
|
||||
},
|
||||
],
|
||||
},
|
||||
})[0].predicate;
|
||||
|
||||
expect(
|
||||
predicateWithHost(
|
||||
new URL(
|
||||
'https://fakeregion.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml?region=eu-west-1',
|
||||
),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for an incorrect host', () => {
|
||||
expect(predicate(new URL('https://amazon.com'))).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for a completely different host', () => {
|
||||
expect(predicate(new URL('https://storage.cloud.google.com'))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('returns false for a path that starts with the wrong format', () => {
|
||||
expect(
|
||||
predicate(
|
||||
new URL(
|
||||
'https://s3.console.aws.amazon.com/s3/object/bucket?&bucketType=general&prefix=catalog-info.yaml',
|
||||
),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('read', () => {
|
||||
const [{ reader }] = createReader({
|
||||
integrations: {
|
||||
awsCodeCommit: [
|
||||
{
|
||||
host: 'amazonaws.com',
|
||||
accessKeyId: 'fake-access-key',
|
||||
secretAccessKey: 'fake-secret-key',
|
||||
region: 'fakeregion',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
codeCommitClient.reset();
|
||||
|
||||
codeCommitClient.on(GetFileCommand).resolves({
|
||||
fileContent: fs.readFileSync(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/awsCodeCommit/awsCodeCommit-mock-object.yaml',
|
||||
),
|
||||
),
|
||||
});
|
||||
});
|
||||
|
||||
it('returns contents of a file in a repository', async () => {
|
||||
const { buffer } = await reader.readUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml',
|
||||
);
|
||||
const response = await buffer();
|
||||
expect(response.toString().trim()).toBe('site_name: Test');
|
||||
});
|
||||
|
||||
it('rejects unknown targets', async () => {
|
||||
const url =
|
||||
'https://eu-west-1.console.aws.NOTamazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml';
|
||||
const expectedSnapshot = `[Error: Could not retrieve file from CodeCommit; caused by Error: Invalid AWS CodeCommit URL (unexpected host format): ${url}]`;
|
||||
await expect(reader.readUrl(url)).rejects.toMatchInlineSnapshot(
|
||||
expectedSnapshot,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readUrl', () => {
|
||||
const [{ reader }] = createReader({
|
||||
integrations: {
|
||||
awsCodeCommit: [
|
||||
{
|
||||
host: AMAZON_AWS_CODECOMMIT_HOST,
|
||||
accessKeyId: 'fake-access-key',
|
||||
secretAccessKey: 'fake-secret-key',
|
||||
region: 'fakeregion',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
codeCommitClient.reset();
|
||||
|
||||
codeCommitClient.on(GetFileCommand).resolves({
|
||||
fileContent: fs.readFileSync(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/awsCodeCommit/awsCodeCommit-mock-object.yaml',
|
||||
),
|
||||
),
|
||||
commitId: `123abc`,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns contents of a file in a repository via buffer', async () => {
|
||||
const { buffer, etag } = await reader.readUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml',
|
||||
);
|
||||
expect(etag).toBe('123abc');
|
||||
const response = await buffer();
|
||||
expect(response.toString().trim()).toBe('site_name: Test');
|
||||
});
|
||||
|
||||
it('returns contents of a file in a repository via stream', async () => {
|
||||
const { buffer, etag } = await reader.readUrl(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml',
|
||||
);
|
||||
expect(etag).toBe('123abc');
|
||||
const response = await buffer();
|
||||
expect(response.toString().trim()).toBe('site_name: Test');
|
||||
});
|
||||
|
||||
it('rejects unknown targets', async () => {
|
||||
const url =
|
||||
'https://eu-west-1.console.aws.NOTamazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml';
|
||||
const expectedSnapshot = `[Error: Could not retrieve file from CodeCommit; caused by Error: Invalid AWS CodeCommit URL (unexpected host format): ${url}]`;
|
||||
await expect(reader.readUrl!(url)).rejects.toMatchInlineSnapshot(
|
||||
expectedSnapshot,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readUrl with etag', () => {
|
||||
const [{ reader }] = createReader({
|
||||
integrations: {
|
||||
awsCodeCommit: [
|
||||
{
|
||||
host: AMAZON_AWS_CODECOMMIT_HOST,
|
||||
accessKeyId: 'fake-access-key',
|
||||
secretAccessKey: 'fake-secret-key',
|
||||
region: 'fakeregion',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
codeCommitClient.reset();
|
||||
|
||||
codeCommitClient.on(GetFileCommand).resolves({
|
||||
fileContent: fs.readFileSync(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/awsCodeCommit/awsCodeCommit-mock-object.yaml',
|
||||
),
|
||||
),
|
||||
commitId: `123abc`,
|
||||
blobId: '999',
|
||||
filePath: 'catalog.yaml',
|
||||
fileMode: 'EXECUTABLE',
|
||||
fileSize: 123,
|
||||
});
|
||||
});
|
||||
|
||||
it('Throw NotModifiedError on matching eTag', async () => {
|
||||
await expect(
|
||||
reader.readUrl!(
|
||||
'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml',
|
||||
{
|
||||
etag: '123abc',
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTree', () => {
|
||||
let awsCodeCommitUrlReader: AwsCodeCommitUrlReader;
|
||||
|
||||
beforeEach(() => {
|
||||
codeCommitClient.reset();
|
||||
|
||||
codeCommitClient
|
||||
.on(GetFolderCommand, {
|
||||
folderPath: `/`,
|
||||
repositoryName: `my-test-techdocs`,
|
||||
commitSpecifier: undefined,
|
||||
})
|
||||
.resolves({
|
||||
files: [
|
||||
{
|
||||
absolutePath: `awsCodeCommit-mock-object.yaml`,
|
||||
relativePath: `awsCodeCommit-mock-object.yaml`,
|
||||
},
|
||||
],
|
||||
subFolders: [
|
||||
{
|
||||
absolutePath: `subFolder`,
|
||||
relativePath: `subFolder`,
|
||||
},
|
||||
],
|
||||
});
|
||||
codeCommitClient
|
||||
.on(GetFolderCommand, {
|
||||
folderPath: `subFolder`,
|
||||
repositoryName: `my-test-techdocs`,
|
||||
commitSpecifier: undefined,
|
||||
})
|
||||
.resolves({
|
||||
files: [
|
||||
{
|
||||
absolutePath: `subFolder/awsCodeCommit-mock-object2.yaml`,
|
||||
relativePath: `subFolder/awsCodeCommit-mock-object2.yaml`,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
codeCommitClient
|
||||
.on(GetFileCommand, {
|
||||
filePath: `awsCodeCommit-mock-object.yaml`,
|
||||
commitSpecifier: undefined,
|
||||
repositoryName: `my-test-techdocs`,
|
||||
})
|
||||
.resolves({
|
||||
fileContent: fs.readFileSync(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/awsCodeCommit/awsCodeCommit-mock-object.yaml',
|
||||
),
|
||||
),
|
||||
});
|
||||
codeCommitClient
|
||||
.on(GetFileCommand, {
|
||||
filePath: `subFolder/awsCodeCommit-mock-object2.yaml`,
|
||||
commitSpecifier: undefined,
|
||||
repositoryName: `my-test-techdocs`,
|
||||
})
|
||||
.resolves({
|
||||
fileContent: fs.readFileSync(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/awsCodeCommit/subFolder/awsCodeCommit-mock-object2.yaml',
|
||||
),
|
||||
),
|
||||
});
|
||||
|
||||
const config = new ConfigReader({
|
||||
accessKeyId: 'fake-access-key',
|
||||
secretAccessKey: 'fake-secret-key',
|
||||
region: 'fakeregion',
|
||||
});
|
||||
|
||||
const credsManager = DefaultAwsCredentialsManager.fromConfig(config);
|
||||
|
||||
awsCodeCommitUrlReader = new AwsCodeCommitUrlReader(
|
||||
credsManager,
|
||||
new AwsCodeCommitIntegration(
|
||||
readAwsCodeCommitIntegrationConfig(config),
|
||||
),
|
||||
{ treeResponseFactory },
|
||||
);
|
||||
});
|
||||
|
||||
it('returns contents of a file in a repository', async () => {
|
||||
const response = await awsCodeCommitUrlReader.readTree(
|
||||
'https://fakeregion.console.aws.amazon.com/codesuite/codecommit/repositories/my-test-techdocs',
|
||||
);
|
||||
|
||||
const files = await response.files();
|
||||
const bodyRootFile = await files[0].content();
|
||||
const bodySubfolderFile = await files[1].content();
|
||||
|
||||
expect(files.length).toEqual(2);
|
||||
expect(bodyRootFile.toString().trim()).toBe('site_name: Test');
|
||||
expect(bodySubfolderFile.toString().trim()).toBe('site_name: Test2');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,393 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
ReaderFactory,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseFactory,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
} from './types';
|
||||
import {
|
||||
AwsCredentialsManager,
|
||||
DefaultAwsCredentialsManager,
|
||||
} from '@backstage/integration-aws-node';
|
||||
import {
|
||||
AwsCodeCommitIntegration,
|
||||
ScmIntegrations,
|
||||
} from '@backstage/integration';
|
||||
import { ForwardedError, NotModifiedError } from '@backstage/errors';
|
||||
import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
|
||||
import {
|
||||
CodeCommitClient,
|
||||
GetFileCommand,
|
||||
GetFileCommandInput,
|
||||
GetFileCommandOutput,
|
||||
GetFolderCommand,
|
||||
} from '@aws-sdk/client-codecommit';
|
||||
import { AwsCredentialIdentityProvider } from '@aws-sdk/types';
|
||||
import { Readable } from 'stream';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import { relative } from 'path/posix';
|
||||
import { AbortController } from '@aws-sdk/abort-controller';
|
||||
|
||||
export function parseUrl(
|
||||
url: string,
|
||||
requireGitPath: boolean = false,
|
||||
): {
|
||||
path: string;
|
||||
repositoryName: string;
|
||||
region: string;
|
||||
commitSpecifier?: string;
|
||||
} {
|
||||
const parsedUrl = new URL(url);
|
||||
|
||||
if (parsedUrl.pathname.includes('/files/edit/')) {
|
||||
throw new Error(
|
||||
'Please provide the view url to yaml file from CodeCommit, not the edit url',
|
||||
);
|
||||
}
|
||||
if (requireGitPath && !parsedUrl.pathname.includes('/browse/')) {
|
||||
throw new Error('Please provide full path to yaml file from CodeCommit');
|
||||
}
|
||||
|
||||
const hostMatch = parsedUrl.host.match(
|
||||
/^([^\.]+)\.console\.aws\.amazon\.com$/,
|
||||
);
|
||||
if (!hostMatch) {
|
||||
throw new Error(
|
||||
`Invalid AWS CodeCommit URL (unexpected host format): ${url}`,
|
||||
);
|
||||
}
|
||||
const [, region] = hostMatch;
|
||||
|
||||
const pathMatch = parsedUrl.pathname.match(
|
||||
/^\/codesuite\/codecommit\/repositories\/([^\/]+)\/browse\/((.*)\/)?--\/(.*)$/,
|
||||
);
|
||||
|
||||
if (!pathMatch) {
|
||||
if (!requireGitPath) {
|
||||
const pathname = parsedUrl.pathname
|
||||
.split('/--/')[0]
|
||||
.replace('/codesuite/codecommit/repositories/', '');
|
||||
const [repositoryName, commitSpecifier] = pathname.split('/browse');
|
||||
|
||||
return {
|
||||
region,
|
||||
repositoryName: repositoryName.replace(/^\/|\/$/g, ''),
|
||||
path: '/',
|
||||
commitSpecifier:
|
||||
commitSpecifier === ''
|
||||
? undefined
|
||||
: commitSpecifier?.replace(/^\/|\/$/g, ''),
|
||||
};
|
||||
}
|
||||
throw new Error(
|
||||
`Invalid AWS CodeCommit URL (unexpected path format): ${url}`,
|
||||
);
|
||||
}
|
||||
const [, repositoryName, , commitSpecifier, path] = pathMatch;
|
||||
|
||||
return {
|
||||
region,
|
||||
repositoryName,
|
||||
path,
|
||||
// the commitSpecifier is passed to AWS SDK which does not allow empty strings so replace empty string with undefined
|
||||
commitSpecifier: commitSpecifier === '' ? undefined : commitSpecifier,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for AWS CodeCommit.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class AwsCodeCommitUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
const credsManager = DefaultAwsCredentialsManager.fromConfig(config);
|
||||
|
||||
return integrations.awsCodeCommit.list().map(integration => {
|
||||
const reader = new AwsCodeCommitUrlReader(credsManager, integration, {
|
||||
treeResponseFactory,
|
||||
});
|
||||
const predicate = (url: URL) => {
|
||||
return (
|
||||
url.host.endsWith(integration.config.host) &&
|
||||
url.pathname.startsWith('/codesuite/codecommit')
|
||||
);
|
||||
};
|
||||
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly credsManager: AwsCredentialsManager,
|
||||
private readonly integration: AwsCodeCommitIntegration,
|
||||
private readonly deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
) {}
|
||||
|
||||
/**
|
||||
* If accessKeyId and secretAccessKey are missing, the standard credentials provider chain will be used:
|
||||
* https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
|
||||
*/
|
||||
private static buildStaticCredentials(
|
||||
accessKeyId: string,
|
||||
secretAccessKey: string,
|
||||
): AwsCredentialIdentityProvider {
|
||||
return async () => {
|
||||
return {
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
private static async buildCredentials(
|
||||
credsManager: AwsCredentialsManager,
|
||||
region: string,
|
||||
integration?: AwsCodeCommitIntegration,
|
||||
): Promise<AwsCredentialIdentityProvider> {
|
||||
// Fall back to the default credential chain if neither account ID
|
||||
// nor explicit credentials are provided
|
||||
if (!integration) {
|
||||
return (await credsManager.getCredentialProvider()).sdkCredentialProvider;
|
||||
}
|
||||
|
||||
const accessKeyId = integration.config.accessKeyId;
|
||||
const secretAccessKey = integration.config.secretAccessKey;
|
||||
let explicitCredentials: AwsCredentialIdentityProvider;
|
||||
if (accessKeyId && secretAccessKey) {
|
||||
explicitCredentials = AwsCodeCommitUrlReader.buildStaticCredentials(
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
);
|
||||
} else {
|
||||
explicitCredentials = (await credsManager.getCredentialProvider())
|
||||
.sdkCredentialProvider;
|
||||
}
|
||||
|
||||
const roleArn = integration.config.roleArn;
|
||||
if (roleArn) {
|
||||
return fromTemporaryCredentials({
|
||||
masterCredentials: explicitCredentials,
|
||||
params: {
|
||||
RoleSessionName: 'backstage-aws-code-commit-url-reader',
|
||||
RoleArn: roleArn,
|
||||
ExternalId: integration.config.externalId,
|
||||
},
|
||||
clientConfig: { region },
|
||||
});
|
||||
}
|
||||
|
||||
return explicitCredentials;
|
||||
}
|
||||
|
||||
private async buildCodeCommitClient(
|
||||
credsManager: AwsCredentialsManager,
|
||||
region: string,
|
||||
integration: AwsCodeCommitIntegration,
|
||||
): Promise<CodeCommitClient> {
|
||||
const credentials = await AwsCodeCommitUrlReader.buildCredentials(
|
||||
credsManager,
|
||||
region,
|
||||
integration,
|
||||
);
|
||||
|
||||
const codeCommit = new CodeCommitClient({
|
||||
region: region,
|
||||
credentials: credentials,
|
||||
});
|
||||
return codeCommit;
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
// etag and lastModifiedAfter are not supported by the CodeCommit API
|
||||
try {
|
||||
const { path, repositoryName, region, commitSpecifier } = parseUrl(
|
||||
url,
|
||||
true,
|
||||
);
|
||||
const codeCommitClient = await this.buildCodeCommitClient(
|
||||
this.credsManager,
|
||||
region,
|
||||
this.integration,
|
||||
);
|
||||
const abortController = new AbortController();
|
||||
|
||||
const input: GetFileCommandInput = {
|
||||
repositoryName: repositoryName,
|
||||
commitSpecifier: commitSpecifier,
|
||||
filePath: path,
|
||||
};
|
||||
|
||||
options?.signal?.addEventListener('abort', () => abortController.abort());
|
||||
const getObjectCommand = new GetFileCommand(input);
|
||||
const response: GetFileCommandOutput = await codeCommitClient.send(
|
||||
getObjectCommand,
|
||||
{
|
||||
abortSignal: abortController.signal,
|
||||
},
|
||||
);
|
||||
|
||||
if (options?.etag && options.etag === response.commitId) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
return ReadUrlResponseFactory.fromReadable(
|
||||
Readable.from([response?.fileContent] || []),
|
||||
{
|
||||
etag: response.commitId,
|
||||
},
|
||||
);
|
||||
} catch (e) {
|
||||
if (e.$metadata && e.$metadata.httpStatusCode === 304) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
if (e.name && e.name === 'NotModifiedError') {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
throw new ForwardedError('Could not retrieve file from CodeCommit', e);
|
||||
}
|
||||
}
|
||||
|
||||
async readTreePath(
|
||||
codeCommitClient: CodeCommitClient,
|
||||
abortSignal: any,
|
||||
path: string,
|
||||
repositoryName: string,
|
||||
commitSpecifier?: string,
|
||||
etag?: string,
|
||||
): Promise<string[]> {
|
||||
const getFolderCommand = new GetFolderCommand({
|
||||
folderPath: path,
|
||||
repositoryName: repositoryName,
|
||||
commitSpecifier: commitSpecifier,
|
||||
});
|
||||
const response = await codeCommitClient.send(getFolderCommand, {
|
||||
abortSignal: abortSignal,
|
||||
});
|
||||
|
||||
if (etag && etag === response.commitId) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
const output: string[] = [];
|
||||
if (response.files) {
|
||||
response.files.forEach(file => {
|
||||
if (file.absolutePath) {
|
||||
output.push(file.absolutePath);
|
||||
}
|
||||
});
|
||||
}
|
||||
if (!response.subFolders) {
|
||||
return output;
|
||||
}
|
||||
|
||||
for (const subFolder of response.subFolders) {
|
||||
if (subFolder.absolutePath) {
|
||||
output.push(
|
||||
...(await this.readTreePath(
|
||||
codeCommitClient,
|
||||
abortSignal,
|
||||
subFolder.absolutePath,
|
||||
repositoryName,
|
||||
commitSpecifier,
|
||||
etag,
|
||||
)),
|
||||
);
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
// url: https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/test-stijn-delete-techdocs/browse?region=eu-west-1
|
||||
try {
|
||||
const { path, repositoryName, region, commitSpecifier } = parseUrl(url);
|
||||
const codeCommitClient = await this.buildCodeCommitClient(
|
||||
this.credsManager,
|
||||
region,
|
||||
this.integration,
|
||||
);
|
||||
|
||||
const abortController = new AbortController();
|
||||
options?.signal?.addEventListener('abort', () => abortController.abort());
|
||||
|
||||
const allFiles: string[] = await this.readTreePath(
|
||||
codeCommitClient,
|
||||
abortController.signal,
|
||||
path,
|
||||
repositoryName,
|
||||
commitSpecifier,
|
||||
options?.etag,
|
||||
);
|
||||
const responses = [];
|
||||
|
||||
for (let i = 0; i < allFiles.length; i++) {
|
||||
const getFileCommand = new GetFileCommand({
|
||||
repositoryName: repositoryName,
|
||||
filePath: String(allFiles[i]),
|
||||
commitSpecifier: commitSpecifier,
|
||||
});
|
||||
const response = await codeCommitClient.send(getFileCommand);
|
||||
const objectData = await Readable.from([response?.fileContent] || []);
|
||||
|
||||
responses.push({
|
||||
data: objectData,
|
||||
path: relative(
|
||||
path.startsWith('/') ? path : `/${path}`,
|
||||
allFiles[i].startsWith('/') ? allFiles[i] : `/${allFiles[i]}`,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
return await this.deps.treeResponseFactory.fromReadableArray(responses);
|
||||
} catch (e) {
|
||||
if (e.name && e.name === 'NotModifiedError') {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
throw new ForwardedError(
|
||||
'Could not retrieve file tree from CodeCommit',
|
||||
e,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async search(): Promise<SearchResponse> {
|
||||
throw new Error('AwsCodeCommitReader does not implement search');
|
||||
}
|
||||
|
||||
toString() {
|
||||
const secretAccessKey = this.integration.config.secretAccessKey;
|
||||
return `awsCodeCommit{host=${this.integration.config.host},authed=${Boolean(
|
||||
secretAccessKey,
|
||||
)}}`;
|
||||
}
|
||||
}
|
||||
@@ -1,530 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { JsonObject } from '@backstage/types';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
import { DEFAULT_REGION, AwsS3UrlReader, parseUrl } from './AwsS3UrlReader';
|
||||
import {
|
||||
AwsS3Integration,
|
||||
readAwsS3IntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node';
|
||||
import { UrlReaderPredicateTuple } from './types';
|
||||
import path from 'path';
|
||||
import { NotModifiedError } from '@backstage/errors';
|
||||
import { mockClient } from 'aws-sdk-client-mock';
|
||||
import {
|
||||
S3Client,
|
||||
ListObjectsV2Command,
|
||||
ListObjectsV2Output,
|
||||
GetObjectCommand,
|
||||
S3ServiceException,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { sdkStreamMixin } from '@aws-sdk/util-stream-node';
|
||||
import fs from 'fs';
|
||||
import { mockServices } from '@backstage/backend-test-utils';
|
||||
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
});
|
||||
|
||||
describe('parseUrl', () => {
|
||||
it('supports all aws formats', () => {
|
||||
expect(
|
||||
parseUrl('https://s3.amazonaws.com/my.bucket-3/a/puppy.jpg', {
|
||||
host: 'amazonaws.com',
|
||||
}),
|
||||
).toEqual({
|
||||
path: 'a/puppy.jpg',
|
||||
bucket: 'my.bucket-3',
|
||||
region: 'us-east-1',
|
||||
});
|
||||
expect(
|
||||
parseUrl('https://s3.amazonaws.com.cn/my.bucket-3/a/puppy.jpg', {
|
||||
host: 'amazonaws.com',
|
||||
}),
|
||||
).toEqual({
|
||||
path: 'a/puppy.jpg',
|
||||
bucket: 'my.bucket-3',
|
||||
region: 'us-east-1',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://ec-backstage-staging.s3.cn-north-1.amazonaws.com.cn/payments-prod-oas30.json',
|
||||
{
|
||||
host: 'amazonaws.com',
|
||||
},
|
||||
),
|
||||
).toEqual({
|
||||
path: 'payments-prod-oas30.json',
|
||||
bucket: 'ec-backstage-staging',
|
||||
region: 'cn-north-1',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://ec-backstage-staging.s3.cn-north-1.amazonaws.com.cn/payments-prod-oas30.json',
|
||||
{
|
||||
host: 'amazonaws.com.cn',
|
||||
},
|
||||
),
|
||||
).toEqual({
|
||||
path: 'payments-prod-oas30.json',
|
||||
bucket: 'ec-backstage-staging',
|
||||
region: 'cn-north-1',
|
||||
});
|
||||
expect(
|
||||
parseUrl('https://s3.us-west-2.amazonaws.com/my.bucket-3/a/puppy.jpg', {
|
||||
host: 'amazonaws.com',
|
||||
}),
|
||||
).toEqual({
|
||||
path: 'a/puppy.jpg',
|
||||
bucket: 'my.bucket-3',
|
||||
region: 'us-west-2',
|
||||
});
|
||||
expect(
|
||||
parseUrl('https://s3-us-west-2.amazonaws.com/my.bucket-3/a/puppy.jpg', {
|
||||
host: 'amazonaws.com',
|
||||
}),
|
||||
).toEqual({
|
||||
path: 'a/puppy.jpg',
|
||||
bucket: 'my.bucket-3',
|
||||
region: 'us-west-2',
|
||||
});
|
||||
expect(
|
||||
parseUrl('https://my.bucket-3.s3.us-west-2.amazonaws.com/a/puppy.jpg', {
|
||||
host: 'amazonaws.com',
|
||||
}),
|
||||
).toEqual({
|
||||
path: 'a/puppy.jpg',
|
||||
bucket: 'my.bucket-3',
|
||||
region: 'us-west-2',
|
||||
});
|
||||
expect(
|
||||
parseUrl(
|
||||
'https://ignored.s3.us-west-2.amazonaws.com/my.bucket-3/a/puppy.jpg',
|
||||
{
|
||||
host: 'amazonaws.com',
|
||||
s3ForcePathStyle: true,
|
||||
},
|
||||
),
|
||||
).toEqual({
|
||||
path: 'a/puppy.jpg',
|
||||
bucket: 'my.bucket-3',
|
||||
region: 'us-west-2',
|
||||
});
|
||||
});
|
||||
|
||||
it('supports all non-aws formats', () => {
|
||||
expect(
|
||||
parseUrl('https://my-host.com/my.bucket-3/a/puppy.jpg', {
|
||||
host: 'my-host.com',
|
||||
}),
|
||||
).toEqual({
|
||||
path: 'a/puppy.jpg',
|
||||
bucket: 'my.bucket-3',
|
||||
region: DEFAULT_REGION,
|
||||
});
|
||||
expect(
|
||||
parseUrl('https://my.bucket-3.my-host.com/a/puppy.jpg', {
|
||||
host: 'my-host.com',
|
||||
}),
|
||||
).toEqual({
|
||||
path: 'a/puppy.jpg',
|
||||
bucket: 'my.bucket-3',
|
||||
region: DEFAULT_REGION,
|
||||
});
|
||||
expect(
|
||||
parseUrl('https://ignored.my-host.com/my.bucket-3/a/puppy.jpg', {
|
||||
host: 'my-host.com',
|
||||
s3ForcePathStyle: true,
|
||||
}),
|
||||
).toEqual({
|
||||
path: 'a/puppy.jpg',
|
||||
bucket: 'my.bucket-3',
|
||||
region: DEFAULT_REGION,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('AwsS3UrlReader', () => {
|
||||
const s3Client = mockClient(S3Client);
|
||||
|
||||
const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
|
||||
return AwsS3UrlReader.factory({
|
||||
config: new ConfigReader(config),
|
||||
logger: mockServices.logger.mock(),
|
||||
treeResponseFactory,
|
||||
});
|
||||
};
|
||||
|
||||
it('creates a sample reader without the awsS3 field', () => {
|
||||
const entries = createReader({
|
||||
integrations: {},
|
||||
});
|
||||
|
||||
expect(entries).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('creates a reader with credentials correctly configured', () => {
|
||||
const awsS3Integrations = [];
|
||||
awsS3Integrations.push({
|
||||
host: 'amazonaws.com',
|
||||
accessKeyId: 'fakekey',
|
||||
secretAccessKey: 'fakekey',
|
||||
});
|
||||
|
||||
const entries = createReader({
|
||||
integrations: {
|
||||
awsS3: awsS3Integrations,
|
||||
},
|
||||
});
|
||||
|
||||
expect(entries).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('creates a reader with default credentials provider', () => {
|
||||
const awsS3Integrations = [];
|
||||
awsS3Integrations.push({
|
||||
host: 'amazonaws.com',
|
||||
});
|
||||
|
||||
const entries = createReader({
|
||||
integrations: {
|
||||
awsS3: awsS3Integrations,
|
||||
},
|
||||
});
|
||||
|
||||
expect(entries).toHaveLength(1);
|
||||
});
|
||||
|
||||
describe('predicates', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
awsS3: [{}],
|
||||
},
|
||||
});
|
||||
const predicate = readers[0].predicate;
|
||||
|
||||
it('returns true for the correct aws s3 storage host', () => {
|
||||
expect(
|
||||
predicate(new URL('https://test-bucket.s3.us-east-2.amazonaws.com')),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for a url with the full path and the correct host', () => {
|
||||
expect(
|
||||
predicate(
|
||||
new URL(
|
||||
'https://test-bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml',
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for an incorrect host', () => {
|
||||
expect(predicate(new URL('https://amazon.com'))).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for a completely different host', () => {
|
||||
expect(predicate(new URL('https://storage.cloud.google.com'))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("returns true for a url with a bucket with '.'", () => {
|
||||
expect(
|
||||
predicate(
|
||||
new URL(
|
||||
'https://test.bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml',
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('read', () => {
|
||||
const [{ reader }] = createReader({
|
||||
integrations: {
|
||||
awsS3: [
|
||||
{
|
||||
host: 'amazonaws.com',
|
||||
accessKeyId: 'fake-access-key',
|
||||
secretAccessKey: 'fake-secret-key',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
s3Client.reset();
|
||||
|
||||
s3Client.on(GetObjectCommand).resolves({
|
||||
Body: sdkStreamMixin(
|
||||
fs.createReadStream(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/awsS3/awsS3-mock-object.yaml',
|
||||
),
|
||||
),
|
||||
),
|
||||
ETag: '123abc',
|
||||
});
|
||||
});
|
||||
|
||||
it('returns contents of an object in a bucket', async () => {
|
||||
const { buffer } = await reader.readUrl(
|
||||
'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
|
||||
);
|
||||
const response = await buffer();
|
||||
expect(response.toString().trim()).toBe('site_name: Test');
|
||||
});
|
||||
|
||||
it('rejects unknown targets', async () => {
|
||||
await expect(
|
||||
reader.readUrl(
|
||||
'https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml',
|
||||
),
|
||||
).rejects.toMatchInlineSnapshot(
|
||||
`[Error: Could not retrieve file from S3; caused by Error: Invalid AWS S3 URL https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml]`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readUrl', () => {
|
||||
const [{ reader }] = createReader({
|
||||
integrations: {
|
||||
awsS3: [
|
||||
{
|
||||
host: 'amazonaws.com',
|
||||
accessKeyId: 'fake-access-key',
|
||||
secretAccessKey: 'fake-secret-key',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
s3Client.reset();
|
||||
|
||||
s3Client.on(GetObjectCommand).resolves({
|
||||
Body: sdkStreamMixin(
|
||||
fs.createReadStream(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/awsS3/awsS3-mock-object.yaml',
|
||||
),
|
||||
),
|
||||
),
|
||||
ETag: '123abc',
|
||||
LastModified: new Date('2020-01-01T00:00:00Z'),
|
||||
});
|
||||
});
|
||||
|
||||
it('returns contents of an object in a bucket via buffer', async () => {
|
||||
const { buffer, etag, lastModifiedAt } = await reader.readUrl(
|
||||
'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
|
||||
);
|
||||
expect(etag).toBe('123abc');
|
||||
expect(lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
|
||||
const response = await buffer();
|
||||
expect(response.toString().trim()).toBe('site_name: Test');
|
||||
});
|
||||
|
||||
it('returns contents of an object in a bucket via stream', async () => {
|
||||
const { buffer, etag, lastModifiedAt } = await reader.readUrl(
|
||||
'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
|
||||
);
|
||||
expect(etag).toBe('123abc');
|
||||
expect(lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
|
||||
const response = await buffer();
|
||||
expect(response.toString().trim()).toBe('site_name: Test');
|
||||
});
|
||||
|
||||
it('rejects unknown targets', async () => {
|
||||
await expect(
|
||||
reader.readUrl!(
|
||||
'https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml',
|
||||
),
|
||||
).rejects.toMatchInlineSnapshot(
|
||||
`[Error: Could not retrieve file from S3; caused by Error: Invalid AWS S3 URL https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml]`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readUrl towards custom host', () => {
|
||||
const [{ reader }] = createReader({
|
||||
integrations: {
|
||||
awsS3: [
|
||||
{
|
||||
host: 'localhost:4566',
|
||||
accessKeyId: 'fake-access-key',
|
||||
secretAccessKey: 'fake-secret-key',
|
||||
endpoint: 'http://localhost:4566',
|
||||
s3ForcePathStyle: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
s3Client.on(GetObjectCommand).resolves({
|
||||
Body: sdkStreamMixin(
|
||||
fs.createReadStream(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/awsS3/awsS3-mock-object.yaml',
|
||||
),
|
||||
),
|
||||
),
|
||||
ETag: '123abc',
|
||||
});
|
||||
});
|
||||
|
||||
it('returns contents of an object in a bucket via buffer', async () => {
|
||||
const { buffer, etag } = await reader.readUrl(
|
||||
'http://localhost:4566/test-bucket/awsS3-mock-object.yaml',
|
||||
);
|
||||
expect(etag).toBe('123abc');
|
||||
const response = await buffer();
|
||||
expect(response.toString().trim()).toBe('site_name: Test');
|
||||
});
|
||||
});
|
||||
|
||||
describe('readUrl with etag', () => {
|
||||
const [{ reader }] = createReader({
|
||||
integrations: {
|
||||
awsS3: [
|
||||
{
|
||||
host: 'amazonaws.com',
|
||||
accessKeyId: 'fake-access-key',
|
||||
secretAccessKey: 'fake-secret-key',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
s3Client.reset();
|
||||
const t = new S3ServiceException({
|
||||
name: '304',
|
||||
$fault: 'client',
|
||||
$metadata: { httpStatusCode: 304 },
|
||||
});
|
||||
s3Client.on(GetObjectCommand).rejects(t);
|
||||
});
|
||||
|
||||
it('returns contents of an object in a bucket', async () => {
|
||||
await expect(
|
||||
reader.readUrl!(
|
||||
'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
|
||||
{
|
||||
etag: '123abc',
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readUrl with lastModifiedAfter', () => {
|
||||
const [{ reader }] = createReader({
|
||||
integrations: {
|
||||
awsS3: [
|
||||
{
|
||||
host: 'amazonaws.com',
|
||||
accessKeyId: 'fake-access-key',
|
||||
secretAccessKey: 'fake-secret-key',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
s3Client.reset();
|
||||
const t = new S3ServiceException({
|
||||
name: '304',
|
||||
$fault: 'client',
|
||||
$metadata: { httpStatusCode: 304 },
|
||||
});
|
||||
s3Client.on(GetObjectCommand).rejects(t);
|
||||
});
|
||||
|
||||
it('returns contents of an object in a bucket', async () => {
|
||||
await expect(
|
||||
reader.readUrl!(
|
||||
'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
|
||||
{
|
||||
lastModifiedAfter: new Date('2020-01-01T00:00:00Z'),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTree', () => {
|
||||
let awsS3UrlReader: AwsS3UrlReader;
|
||||
|
||||
beforeEach(() => {
|
||||
s3Client.reset();
|
||||
|
||||
const object: Object = {
|
||||
Key: 'awsS3-mock-object.yaml',
|
||||
};
|
||||
|
||||
const objectList: Object[] = [object];
|
||||
const output: ListObjectsV2Output = {
|
||||
Contents: objectList,
|
||||
};
|
||||
|
||||
s3Client.on(ListObjectsV2Command).resolves(output);
|
||||
|
||||
s3Client.on(GetObjectCommand).resolves({
|
||||
Body: sdkStreamMixin(
|
||||
fs.createReadStream(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/awsS3/awsS3-mock-object.yaml',
|
||||
),
|
||||
),
|
||||
),
|
||||
});
|
||||
|
||||
const config = new ConfigReader({
|
||||
host: '.amazonaws.com',
|
||||
accessKeyId: 'fake-access-key',
|
||||
secretAccessKey: 'fake-secret-key',
|
||||
});
|
||||
|
||||
const credsManager = DefaultAwsCredentialsManager.fromConfig(config);
|
||||
|
||||
awsS3UrlReader = new AwsS3UrlReader(
|
||||
credsManager,
|
||||
new AwsS3Integration(readAwsS3IntegrationConfig(config)),
|
||||
{ treeResponseFactory },
|
||||
);
|
||||
});
|
||||
|
||||
it('returns contents of an object in a bucket', async () => {
|
||||
const response = await awsS3UrlReader.readTree(
|
||||
'https://test.s3.us-east-2.amazonaws.com',
|
||||
);
|
||||
const files = await response.files();
|
||||
const body = await files[0].content();
|
||||
|
||||
expect(body.toString().trim()).toBe('site_name: Test');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,370 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
ReaderFactory,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseFactory,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
} from './types';
|
||||
import {
|
||||
AwsCredentialsManager,
|
||||
DefaultAwsCredentialsManager,
|
||||
} from '@backstage/integration-aws-node';
|
||||
import {
|
||||
AwsS3Integration,
|
||||
ScmIntegrations,
|
||||
AwsS3IntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
import { ForwardedError, NotModifiedError } from '@backstage/errors';
|
||||
import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
|
||||
import { AwsCredentialIdentityProvider } from '@aws-sdk/types';
|
||||
import {
|
||||
S3Client,
|
||||
ListObjectsV2Command,
|
||||
ListObjectsV2CommandOutput,
|
||||
GetObjectCommand,
|
||||
GetObjectCommandInput,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { AbortController } from '@aws-sdk/abort-controller';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import { Readable } from 'stream';
|
||||
import { relative } from 'path/posix';
|
||||
|
||||
export const DEFAULT_REGION = 'us-east-1';
|
||||
|
||||
/**
|
||||
* Path style URLs: https://s3.(region).amazonaws.com/(bucket)/(key)
|
||||
* The region can also be on the old form: https://s3-(region).amazonaws.com/(bucket)/(key)
|
||||
* Virtual hosted style URLs: https://(bucket).s3.(region).amazonaws.com/(key)
|
||||
* See https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access
|
||||
*/
|
||||
export function parseUrl(
|
||||
url: string,
|
||||
config: AwsS3IntegrationConfig,
|
||||
): { path: string; bucket: string; region: string } {
|
||||
const parsedUrl = new URL(url);
|
||||
|
||||
/**
|
||||
* Removes the leading '/' from the pathname to be processed
|
||||
* as a parameter by AWS S3 SDK getObject method.
|
||||
*/
|
||||
const pathname = parsedUrl.pathname.substring(1);
|
||||
const host = parsedUrl.host;
|
||||
|
||||
// Treat Amazon hosted separately because it has special region logic
|
||||
if (config.host === 'amazonaws.com' || config.host === 'amazonaws.com.cn') {
|
||||
const match = host.match(
|
||||
/^(?:([a-z0-9.-]+)\.)?s3(?:[.-]([a-z0-9-]+))?\.amazonaws\.com(\.cn)?$/,
|
||||
);
|
||||
if (!match) {
|
||||
throw new Error(`Invalid AWS S3 URL ${url}`);
|
||||
}
|
||||
|
||||
const [, hostBucket, hostRegion] = match;
|
||||
|
||||
if (config.s3ForcePathStyle || !hostBucket) {
|
||||
const slashIndex = pathname.indexOf('/');
|
||||
if (slashIndex < 0) {
|
||||
throw new Error(
|
||||
`Invalid path-style AWS S3 URL ${url}, does not contain bucket in the path`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
path: pathname.substring(slashIndex + 1),
|
||||
bucket: pathname.substring(0, slashIndex),
|
||||
region: hostRegion ?? DEFAULT_REGION,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
path: pathname,
|
||||
bucket: hostBucket,
|
||||
region: hostRegion ?? DEFAULT_REGION,
|
||||
};
|
||||
}
|
||||
|
||||
const usePathStyle =
|
||||
config.s3ForcePathStyle || host.length === config.host.length;
|
||||
|
||||
if (usePathStyle) {
|
||||
const slashIndex = pathname.indexOf('/');
|
||||
if (slashIndex < 0) {
|
||||
throw new Error(
|
||||
`Invalid path-style AWS S3 URL ${url}, does not contain bucket in the path`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
path: pathname.substring(slashIndex + 1),
|
||||
bucket: pathname.substring(0, slashIndex),
|
||||
region: DEFAULT_REGION,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
path: pathname,
|
||||
bucket: host.substring(0, host.length - config.host.length - 1),
|
||||
region: DEFAULT_REGION,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for AWS S3 buckets.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class AwsS3UrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
const credsManager = DefaultAwsCredentialsManager.fromConfig(config);
|
||||
|
||||
return integrations.awsS3.list().map(integration => {
|
||||
const reader = new AwsS3UrlReader(credsManager, integration, {
|
||||
treeResponseFactory,
|
||||
});
|
||||
const predicate = (url: URL) =>
|
||||
url.host.endsWith(integration.config.host);
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly credsManager: AwsCredentialsManager,
|
||||
private readonly integration: AwsS3Integration,
|
||||
private readonly deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
) {}
|
||||
|
||||
/**
|
||||
* If accessKeyId and secretAccessKey are missing, the standard credentials provider chain will be used:
|
||||
* https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
|
||||
*/
|
||||
private static buildStaticCredentials(
|
||||
accessKeyId: string,
|
||||
secretAccessKey: string,
|
||||
): AwsCredentialIdentityProvider {
|
||||
return async () => {
|
||||
return {
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
private static async buildCredentials(
|
||||
credsManager: AwsCredentialsManager,
|
||||
region: string,
|
||||
integration?: AwsS3Integration,
|
||||
): Promise<AwsCredentialIdentityProvider> {
|
||||
// Fall back to the default credential chain if neither account ID
|
||||
// nor explicit credentials are provided
|
||||
if (!integration) {
|
||||
return (await credsManager.getCredentialProvider()).sdkCredentialProvider;
|
||||
}
|
||||
|
||||
const accessKeyId = integration.config.accessKeyId;
|
||||
const secretAccessKey = integration.config.secretAccessKey;
|
||||
let explicitCredentials: AwsCredentialIdentityProvider;
|
||||
if (accessKeyId && secretAccessKey) {
|
||||
explicitCredentials = AwsS3UrlReader.buildStaticCredentials(
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
);
|
||||
} else {
|
||||
explicitCredentials = (await credsManager.getCredentialProvider())
|
||||
.sdkCredentialProvider;
|
||||
}
|
||||
|
||||
const roleArn = integration.config.roleArn;
|
||||
if (roleArn) {
|
||||
return fromTemporaryCredentials({
|
||||
masterCredentials: explicitCredentials,
|
||||
params: {
|
||||
RoleSessionName: 'backstage-aws-s3-url-reader',
|
||||
RoleArn: roleArn,
|
||||
ExternalId: integration.config.externalId,
|
||||
},
|
||||
clientConfig: { region },
|
||||
});
|
||||
}
|
||||
|
||||
return explicitCredentials;
|
||||
}
|
||||
|
||||
private async buildS3Client(
|
||||
credsManager: AwsCredentialsManager,
|
||||
region: string,
|
||||
integration: AwsS3Integration,
|
||||
): Promise<S3Client> {
|
||||
const credentials = await AwsS3UrlReader.buildCredentials(
|
||||
credsManager,
|
||||
region,
|
||||
integration,
|
||||
);
|
||||
|
||||
const s3 = new S3Client({
|
||||
region: region,
|
||||
credentials: credentials,
|
||||
endpoint: integration.config.endpoint,
|
||||
forcePathStyle: integration.config.s3ForcePathStyle,
|
||||
});
|
||||
return s3;
|
||||
}
|
||||
|
||||
private async retrieveS3ObjectData(stream: Readable): Promise<Readable> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
const chunks: any[] = [];
|
||||
stream.on('data', chunk => chunks.push(chunk));
|
||||
stream.on('error', (e: Error) =>
|
||||
reject(new ForwardedError('Unable to read stream', e)),
|
||||
);
|
||||
stream.on('end', () => resolve(Readable.from(Buffer.concat(chunks))));
|
||||
} catch (e) {
|
||||
throw new ForwardedError('Unable to parse the response data', e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
const { etag, lastModifiedAfter } = options ?? {};
|
||||
|
||||
try {
|
||||
const { path, bucket, region } = parseUrl(url, this.integration.config);
|
||||
const s3Client = await this.buildS3Client(
|
||||
this.credsManager,
|
||||
region,
|
||||
this.integration,
|
||||
);
|
||||
const abortController = new AbortController();
|
||||
|
||||
const params: GetObjectCommandInput = {
|
||||
Bucket: bucket,
|
||||
Key: path,
|
||||
...(etag && { IfNoneMatch: etag }),
|
||||
...(lastModifiedAfter && {
|
||||
IfModifiedSince: lastModifiedAfter,
|
||||
}),
|
||||
};
|
||||
|
||||
options?.signal?.addEventListener('abort', () => abortController.abort());
|
||||
const getObjectCommand = new GetObjectCommand(params);
|
||||
const response = await s3Client.send(getObjectCommand, {
|
||||
abortSignal: abortController.signal,
|
||||
});
|
||||
|
||||
const s3ObjectData = await this.retrieveS3ObjectData(
|
||||
response.Body as Readable,
|
||||
);
|
||||
|
||||
return ReadUrlResponseFactory.fromReadable(s3ObjectData, {
|
||||
etag: response.ETag,
|
||||
lastModifiedAt: response.LastModified,
|
||||
});
|
||||
} catch (e) {
|
||||
if (e.$metadata && e.$metadata.httpStatusCode === 304) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
throw new ForwardedError('Could not retrieve file from S3', e);
|
||||
}
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
try {
|
||||
const { path, bucket, region } = parseUrl(url, this.integration.config);
|
||||
const s3Client = await this.buildS3Client(
|
||||
this.credsManager,
|
||||
region,
|
||||
this.integration,
|
||||
);
|
||||
const abortController = new AbortController();
|
||||
const allObjects: String[] = [];
|
||||
const responses = [];
|
||||
let continuationToken: string | undefined;
|
||||
let output: ListObjectsV2CommandOutput;
|
||||
do {
|
||||
const listObjectsV2Command = new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
ContinuationToken: continuationToken,
|
||||
Prefix: path,
|
||||
});
|
||||
options?.signal?.addEventListener('abort', () =>
|
||||
abortController.abort(),
|
||||
);
|
||||
output = await s3Client.send(listObjectsV2Command, {
|
||||
abortSignal: abortController.signal,
|
||||
});
|
||||
if (output.Contents) {
|
||||
output.Contents.forEach(contents => {
|
||||
allObjects.push(contents.Key!);
|
||||
});
|
||||
}
|
||||
continuationToken = output.NextContinuationToken;
|
||||
} while (continuationToken);
|
||||
|
||||
for (let i = 0; i < allObjects.length; i++) {
|
||||
const getObjectCommand = new GetObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: String(allObjects[i]),
|
||||
});
|
||||
const response = await s3Client.send(getObjectCommand);
|
||||
const s3ObjectData = await this.retrieveS3ObjectData(
|
||||
response.Body as Readable,
|
||||
);
|
||||
|
||||
responses.push({
|
||||
data: s3ObjectData,
|
||||
path: relative(path, String(allObjects[i])),
|
||||
lastModifiedAt: response?.LastModified ?? undefined,
|
||||
});
|
||||
}
|
||||
|
||||
return await this.deps.treeResponseFactory.fromReadableArray(responses);
|
||||
} catch (e) {
|
||||
throw new ForwardedError('Could not retrieve file tree from S3', e);
|
||||
}
|
||||
}
|
||||
|
||||
async search(): Promise<SearchResponse> {
|
||||
throw new Error('AwsS3Reader does not implement search');
|
||||
}
|
||||
|
||||
toString() {
|
||||
const secretAccessKey = this.integration.config.secretAccessKey;
|
||||
return `awsS3{host=${this.integration.config.host},authed=${Boolean(
|
||||
secretAccessKey,
|
||||
)}}`;
|
||||
}
|
||||
}
|
||||
@@ -1,377 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import {
|
||||
AzureIntegration,
|
||||
DefaultAzureDevOpsCredentialsProvider,
|
||||
readAzureIntegrationConfig,
|
||||
ScmIntegrations,
|
||||
AzureDevOpsCredentialLike,
|
||||
AzureIntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
import {
|
||||
createMockDirectory,
|
||||
mockServices,
|
||||
setupRequestMockHandlers,
|
||||
} from '@backstage/backend-test-utils';
|
||||
import fs from 'fs-extra';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import path from 'path';
|
||||
import { NotModifiedError } from '@backstage/errors';
|
||||
import { AzureUrlReader } from './AzureUrlReader';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
|
||||
type AzureIntegrationConfigLike = Partial<
|
||||
Omit<AzureIntegrationConfig, 'credential' | 'credentials'>
|
||||
> & {
|
||||
credentials?: Partial<AzureDevOpsCredentialLike>[];
|
||||
};
|
||||
|
||||
const logger = mockServices.logger.mock();
|
||||
|
||||
const mockDir = createMockDirectory({ mockOsTmpDir: true });
|
||||
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
});
|
||||
|
||||
const urlReaderFactory = (azureIntegration: AzureIntegrationConfigLike) => {
|
||||
const credentialsProvider =
|
||||
DefaultAzureDevOpsCredentialsProvider.fromIntegrations(
|
||||
ScmIntegrations.fromConfig(
|
||||
new ConfigReader({
|
||||
integrations: {
|
||||
azure: [azureIntegration],
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
return new AzureUrlReader(
|
||||
new AzureIntegration(
|
||||
readAzureIntegrationConfig(new ConfigReader(azureIntegration)),
|
||||
),
|
||||
{
|
||||
treeResponseFactory,
|
||||
credentialsProvider,
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
describe('AzureUrlReader', () => {
|
||||
beforeEach(mockDir.clear);
|
||||
|
||||
const worker = setupServer();
|
||||
setupRequestMockHandlers(worker);
|
||||
|
||||
describe('read', () => {
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get('*', (req, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
url: req.url.toString(),
|
||||
headers: req.headers.all(),
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
const createConfig = (token?: string) => {
|
||||
let credentials: AzureDevOpsCredentialLike[] | undefined = undefined;
|
||||
if (token !== undefined) {
|
||||
credentials = [
|
||||
{
|
||||
personalAccessToken: token,
|
||||
},
|
||||
];
|
||||
}
|
||||
return new ConfigReader(
|
||||
{
|
||||
integrations: {
|
||||
azure: [
|
||||
{
|
||||
host: 'dev.azure.com',
|
||||
credentials: credentials,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
'test-config',
|
||||
);
|
||||
};
|
||||
|
||||
it.each([
|
||||
{
|
||||
url: 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
|
||||
config: createConfig(),
|
||||
response: expect.objectContaining({
|
||||
url: 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?api-version=6.0&path=my-template.yaml&version=master',
|
||||
}),
|
||||
},
|
||||
{
|
||||
url: 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml',
|
||||
config: createConfig(),
|
||||
response: expect.objectContaining({
|
||||
url: 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?api-version=6.0&path=my-template.yaml',
|
||||
}),
|
||||
},
|
||||
{
|
||||
url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
|
||||
config: createConfig('0123456789'),
|
||||
response: expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
authorization: 'Basic OjAxMjM0NTY3ODk=',
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
|
||||
config: createConfig(undefined),
|
||||
response: expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
authorization: expect.stringMatching(/^Bearer /),
|
||||
}),
|
||||
}),
|
||||
},
|
||||
])('should handle happy path %#', async ({ url, config, response }) => {
|
||||
const [{ reader }] = AzureUrlReader.factory({
|
||||
config,
|
||||
logger,
|
||||
treeResponseFactory,
|
||||
});
|
||||
|
||||
const { buffer } = await reader.readUrl(url);
|
||||
const fromStream = await buffer();
|
||||
const res = await JSON.parse(fromStream.toString());
|
||||
expect(res).toEqual(response);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
url: 'https://api.com/a/b/blob/master/path/to/c.yaml',
|
||||
config: createConfig(),
|
||||
error: 'Azure URL must point to a git repository',
|
||||
},
|
||||
{
|
||||
url: 'com/a/b/blob/master/path/to/c.yaml',
|
||||
config: createConfig(),
|
||||
error: 'Invalid URL',
|
||||
},
|
||||
{
|
||||
url: '',
|
||||
config: createConfig(''),
|
||||
error:
|
||||
"Invalid type in config for key 'integrations.azure[0].credentials[0].personalAccessToken' in 'test-config', got empty-string, wanted string",
|
||||
},
|
||||
])('should handle error path %#', async ({ url, config, error }) => {
|
||||
await expect(async () => {
|
||||
const [{ reader }] = AzureUrlReader.factory({
|
||||
config,
|
||||
logger,
|
||||
treeResponseFactory,
|
||||
});
|
||||
await reader.readUrl(url);
|
||||
}).rejects.toThrow(error);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTree', () => {
|
||||
const repoBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/mock-main.zip'),
|
||||
);
|
||||
|
||||
const processor = urlReaderFactory({
|
||||
host: 'dev.azure.com',
|
||||
credentials: [
|
||||
{
|
||||
personalAccessToken: 'my-pat',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://dev.azure.com/organization/project/_apis/git/repositories/repository/items',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.body(repoBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
// https://docs.microsoft.com/en-us/rest/api/azure/devops/git/commits/get%20commits?view=azure-devops-rest-6.0#on-a-branch
|
||||
'https://dev.azure.com/organization/project/_apis/git/repositories/repository/commits',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
count: 2,
|
||||
value: [
|
||||
{
|
||||
commitId: '123abc2',
|
||||
comment: 'second commit',
|
||||
},
|
||||
{
|
||||
commitId: '123abc1',
|
||||
comment: 'first commit',
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns the wanted files from an archive', async () => {
|
||||
const response = await processor.readTree(
|
||||
'https://dev.azure.com/organization/project/_git/repository',
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('123abc2');
|
||||
|
||||
const files = await response.files();
|
||||
|
||||
expect(files.length).toBe(2);
|
||||
const mkDocsFile = await files[0].content();
|
||||
const indexMarkdownFile = await files[1].content();
|
||||
|
||||
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('creates a directory with the wanted files', async () => {
|
||||
const response = await processor.readTree(
|
||||
'https://dev.azure.com/organization/project/_git/repository',
|
||||
);
|
||||
|
||||
const dir = await response.dir({ targetDir: mockDir.path });
|
||||
|
||||
await expect(
|
||||
fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'),
|
||||
).resolves.toBe('site_name: Test\n');
|
||||
await expect(
|
||||
fs.readFile(path.join(dir, 'docs', 'index.md'), 'utf8'),
|
||||
).resolves.toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('throws a NotModifiedError when given a etag in options', async () => {
|
||||
const fnAzure = async () => {
|
||||
await processor.readTree(
|
||||
'https://dev.azure.com/organization/project/_git/repository',
|
||||
{ etag: '123abc2' },
|
||||
);
|
||||
};
|
||||
|
||||
await expect(fnAzure).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should not throw a NotModifiedError when given an outdated etag in options', async () => {
|
||||
const response = await processor.readTree(
|
||||
'https://dev.azure.com/organization/project/_git/repository',
|
||||
{ etag: 'outdated123abc' },
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('123abc2');
|
||||
const files = await response.files();
|
||||
|
||||
expect(files.length).toBe(2);
|
||||
const mkDocsFile = await files[0].content();
|
||||
const indexMarkdownFile = await files[1].content();
|
||||
|
||||
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
});
|
||||
|
||||
describe('search', () => {
|
||||
const repoBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/mock-main.zip'),
|
||||
);
|
||||
|
||||
const processor = urlReaderFactory({
|
||||
host: 'dev.azure.com',
|
||||
credentials: [
|
||||
{
|
||||
personalAccessToken: 'my-pat',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.body(repoBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
// https://docs.microsoft.com/en-us/rest/api/azure/devops/git/commits/get%20commits?view=azure-devops-rest-6.0#on-a-branch
|
||||
'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/commits',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
count: 2,
|
||||
value: [
|
||||
{
|
||||
commitId: '123abc2',
|
||||
comment: 'second commit',
|
||||
},
|
||||
{
|
||||
commitId: '123abc1',
|
||||
comment: 'first commit',
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('works for the naive case', async () => {
|
||||
const result = await processor.search(
|
||||
'https://dev.azure.com/org-name/project-name/_git/repo-name?path=%2F**%2Findex.*&version=GBmaster',
|
||||
);
|
||||
expect(result.etag).toBe('123abc2');
|
||||
expect(result.files.length).toBe(1);
|
||||
expect(result.files[0].url).toBe(
|
||||
'https://dev.azure.com/org-name/project-name/_git/repo-name?path=%2Fdocs%2Findex.md&version=GBmaster',
|
||||
);
|
||||
await expect(result.files[0].content()).resolves.toEqual(
|
||||
Buffer.from('# Test\n'),
|
||||
);
|
||||
});
|
||||
|
||||
it('throws NotModifiedError when same etag', async () => {
|
||||
await expect(
|
||||
processor.search(
|
||||
'https://dev.azure.com/org-name/project-name/_git/repo-name?path=**/index.*&version=GBmaster',
|
||||
{ etag: '123abc2' },
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,221 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
getAzureCommitsUrl,
|
||||
getAzureDownloadUrl,
|
||||
getAzureFileFetchUrl,
|
||||
AzureDevOpsCredentialsProvider,
|
||||
DefaultAzureDevOpsCredentialsProvider,
|
||||
ScmIntegrations,
|
||||
AzureIntegration,
|
||||
} from '@backstage/integration';
|
||||
import fetch, { Response } from 'node-fetch';
|
||||
import { Minimatch } from 'minimatch';
|
||||
import { Readable } from 'stream';
|
||||
import { NotFoundError, NotModifiedError } from '@backstage/errors';
|
||||
import {
|
||||
ReadTreeResponseFactory,
|
||||
ReaderFactory,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
SearchOptions,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
} from './types';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for Azure repos.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class AzureUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
const credentialProvider =
|
||||
DefaultAzureDevOpsCredentialsProvider.fromIntegrations(integrations);
|
||||
return integrations.azure.list().map(integration => {
|
||||
const reader = new AzureUrlReader(integration, {
|
||||
treeResponseFactory,
|
||||
credentialsProvider: credentialProvider,
|
||||
});
|
||||
const predicate = (url: URL) => url.host === integration.config.host;
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly integration: AzureIntegration,
|
||||
private readonly deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
credentialsProvider: AzureDevOpsCredentialsProvider;
|
||||
},
|
||||
) {}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
// TODO: etag is not implemented yet.
|
||||
const { signal } = options ?? {};
|
||||
|
||||
const builtUrl = getAzureFileFetchUrl(url);
|
||||
let response: Response;
|
||||
try {
|
||||
const credentials = await this.deps.credentialsProvider.getCredentials({
|
||||
url: builtUrl,
|
||||
});
|
||||
response = await fetch(builtUrl, {
|
||||
headers: credentials?.headers,
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can
|
||||
// be removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
...(signal && { signal: signal as any }),
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${url}, ${e}`);
|
||||
}
|
||||
|
||||
// for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html
|
||||
if (response.ok && response.status !== 203) {
|
||||
return ReadUrlResponseFactory.fromNodeJSReadable(response.body);
|
||||
}
|
||||
|
||||
const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`;
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
const { etag, filter, signal } = options ?? {};
|
||||
|
||||
// TODO: Support filepath based reading tree feature like other providers
|
||||
|
||||
// Get latest commit SHA
|
||||
|
||||
const credentials = await this.deps.credentialsProvider.getCredentials({
|
||||
url: url,
|
||||
});
|
||||
|
||||
const commitsAzureResponse = await fetch(getAzureCommitsUrl(url), {
|
||||
headers: credentials?.headers,
|
||||
});
|
||||
if (!commitsAzureResponse.ok) {
|
||||
const message = `Failed to read tree from ${url}, ${commitsAzureResponse.status} ${commitsAzureResponse.statusText}`;
|
||||
if (commitsAzureResponse.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
const commitSha = (await commitsAzureResponse.json()).value[0].commitId;
|
||||
if (etag && etag === commitSha) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
const archiveAzureResponse = await fetch(getAzureDownloadUrl(url), {
|
||||
headers: {
|
||||
...credentials?.headers,
|
||||
Accept: 'application/zip',
|
||||
},
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can be
|
||||
// removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
...(signal && { signal: signal as any }),
|
||||
});
|
||||
if (!archiveAzureResponse.ok) {
|
||||
const message = `Failed to read tree from ${url}, ${archiveAzureResponse.status} ${archiveAzureResponse.statusText}`;
|
||||
if (archiveAzureResponse.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
// When downloading a zip archive from azure on a subpath we get an extra directory
|
||||
// layer added at the top. With for example the file /a/b/c.txt and a download of
|
||||
// /a/b, we'll see /b/c.txt in the zip archive. This picks out /b so that we can remove it.
|
||||
let subpath;
|
||||
const path = new URL(url).searchParams.get('path');
|
||||
if (path) {
|
||||
subpath = path.split('/').filter(Boolean).slice(-1)[0];
|
||||
}
|
||||
|
||||
return await this.deps.treeResponseFactory.fromZipArchive({
|
||||
stream: Readable.from(archiveAzureResponse.body),
|
||||
etag: commitSha,
|
||||
filter,
|
||||
subpath,
|
||||
});
|
||||
}
|
||||
|
||||
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
|
||||
const treeUrl = new URL(url);
|
||||
|
||||
const path = treeUrl.searchParams.get('path');
|
||||
const matcher = path && new Minimatch(path.replace(/^\/+/, ''));
|
||||
|
||||
// TODO(freben): For now, read the entire repo and filter through that. In
|
||||
// a future improvement, we could be smart and try to deduce that non-glob
|
||||
// prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used
|
||||
// to get just that part of the repo.
|
||||
treeUrl.searchParams.delete('path');
|
||||
|
||||
const tree = await this.readTree(treeUrl.toString(), {
|
||||
etag: options?.etag,
|
||||
signal: options?.signal,
|
||||
filter: p => (matcher ? matcher.match(p) : true),
|
||||
});
|
||||
const files = await tree.files();
|
||||
|
||||
return {
|
||||
etag: tree.etag,
|
||||
files: files.map(file => ({
|
||||
url: this.integration.resolveUrl({
|
||||
url: `/${file.path}`,
|
||||
base: url,
|
||||
}),
|
||||
content: file.content,
|
||||
lastModifiedAt: file.lastModifiedAt,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
toString() {
|
||||
const { host, credentials } = this.integration.config;
|
||||
return `azure{host=${host},authed=${Boolean(
|
||||
credentials !== undefined && credentials.length > 0,
|
||||
)}}`;
|
||||
}
|
||||
}
|
||||
@@ -1,455 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import {
|
||||
BitbucketCloudIntegration,
|
||||
readBitbucketCloudIntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
import {
|
||||
createMockDirectory,
|
||||
setupRequestMockHandlers,
|
||||
} from '@backstage/backend-test-utils';
|
||||
import fs from 'fs-extra';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import path from 'path';
|
||||
import { NotModifiedError } from '@backstage/errors';
|
||||
import { BitbucketCloudUrlReader } from './BitbucketCloudUrlReader';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
import getRawBody from 'raw-body';
|
||||
|
||||
const mockDir = createMockDirectory({ mockOsTmpDir: true });
|
||||
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
});
|
||||
|
||||
const reader = new BitbucketCloudUrlReader(
|
||||
new BitbucketCloudIntegration(
|
||||
readBitbucketCloudIntegrationConfig(
|
||||
new ConfigReader({
|
||||
host: 'bitbucket.org',
|
||||
apiBaseUrl: 'https://api.bitbucket.org/2.0',
|
||||
username: 'username',
|
||||
appPassword: 'password',
|
||||
}),
|
||||
),
|
||||
),
|
||||
{ treeResponseFactory },
|
||||
);
|
||||
|
||||
describe('BitbucketCloudUrlReader', () => {
|
||||
beforeEach(mockDir.clear);
|
||||
|
||||
const worker = setupServer();
|
||||
setupRequestMockHandlers(worker);
|
||||
|
||||
describe('readUrl', () => {
|
||||
it('should be able to readUrl via buffer without ETag', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-None-Match')).toBeNull();
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body('foo'),
|
||||
ctx.set('ETag', 'etag-value'),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await reader.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
);
|
||||
const buffer = await result.buffer();
|
||||
expect(buffer.toString()).toBe('foo');
|
||||
});
|
||||
|
||||
it('should be able to readUrl via stream without ETag', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-None-Match')).toBeNull();
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body('foo'),
|
||||
ctx.set('ETag', 'etag-value'),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await reader.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
);
|
||||
const fromStream = await getRawBody(result.stream!());
|
||||
expect(fromStream.toString()).toBe('foo');
|
||||
});
|
||||
|
||||
it('should be able to readUrl with matching ETag', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-None-Match')).toBe(
|
||||
'matching-etag-value',
|
||||
);
|
||||
return res(ctx.status(304));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
reader.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
{ etag: 'matching-etag-value' },
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should be able to readUrl without matching ETag', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-None-Match')).toBe(
|
||||
'previous-etag-value',
|
||||
);
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body('foo'),
|
||||
ctx.set('ETag', 'new-etag-value'),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await reader.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
{ etag: 'previous-etag-value' },
|
||||
);
|
||||
const buffer = await result.buffer();
|
||||
expect(buffer.toString()).toBe('foo');
|
||||
expect(result.etag).toBe('new-etag-value');
|
||||
});
|
||||
|
||||
it('should be able to readUrl via buffer without If-Modified-Since', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-None-Match')).toBeNull();
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body('foo'),
|
||||
ctx.set('ETag', 'etag-value'),
|
||||
ctx.set(
|
||||
'Last-Modified',
|
||||
new Date('2020-01-01T00:00:00Z').toUTCString(),
|
||||
),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await reader.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
);
|
||||
const buffer = await result.buffer();
|
||||
expect(result.lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
|
||||
expect(buffer.toString()).toBe('foo');
|
||||
});
|
||||
|
||||
it('should be throw not modified when If-Modified-Since returns a 304', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-Modified-Since')).toBe(
|
||||
new Date('1999 12 31 23:59:59 GMT').toUTCString(),
|
||||
);
|
||||
return res(ctx.status(304));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
reader.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
{ lastModifiedAfter: new Date('1999 12 31 23:59:59 GMT') },
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should be able to readUrl when If-Modified-Since is before Last-Modified', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-Modified-Since')).toBe(
|
||||
new Date('1999 12 31 23:59:59 GMT').toUTCString(),
|
||||
);
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set(
|
||||
'Last-Modified',
|
||||
new Date('2020-01-01T00:00:00Z').toUTCString(),
|
||||
),
|
||||
ctx.body('foo'),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await reader.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
{ lastModifiedAfter: new Date('1999 12 31 23:59:59 GMT') },
|
||||
);
|
||||
const buffer = await result.buffer();
|
||||
expect(buffer.toString()).toBe('foo');
|
||||
expect(result.lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('read', () => {
|
||||
it('rejects unknown targets', async () => {
|
||||
await expect(
|
||||
reader.readUrl('https://not.bitbucket.com/apa'),
|
||||
).rejects.toThrow(
|
||||
'Incorrect URL: https://not.bitbucket.com/apa, Error: Invalid Bitbucket Cloud URL or file path',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTree', () => {
|
||||
const repoBuffer = fs.readFileSync(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/bitbucket-repo-with-commit-hash.tar.gz',
|
||||
),
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage/mock',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
mainbranch: {
|
||||
type: 'branch',
|
||||
name: 'master',
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://bitbucket.org/backstage/mock/get/master.tar.gz',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz',
|
||||
),
|
||||
ctx.body(repoBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns the wanted files from an archive', async () => {
|
||||
const response = await reader.readTree(
|
||||
'https://bitbucket.org/backstage/mock/src/master',
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('12ab34cd56ef');
|
||||
|
||||
const files = await response.files();
|
||||
|
||||
expect(files.length).toBe(2);
|
||||
const mkDocsFile = await files[0].content();
|
||||
const indexMarkdownFile = await files[1].content();
|
||||
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
|
||||
});
|
||||
|
||||
it('creates a directory with the wanted files', async () => {
|
||||
const response = await reader.readTree(
|
||||
'https://bitbucket.org/backstage/mock',
|
||||
);
|
||||
|
||||
const dir = await response.dir({ targetDir: mockDir.path });
|
||||
|
||||
await expect(
|
||||
fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'),
|
||||
).resolves.toBe('site_name: Test\n');
|
||||
await expect(
|
||||
fs.readFile(path.join(dir, 'docs', 'index.md'), 'utf8'),
|
||||
).resolves.toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('returns the wanted files from an archive with a subpath', async () => {
|
||||
const response = await reader.readTree(
|
||||
'https://bitbucket.org/backstage/mock/src/master/docs',
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('12ab34cd56ef');
|
||||
|
||||
const files = await response.files();
|
||||
|
||||
expect(files.length).toBe(1);
|
||||
const indexMarkdownFile = await files[0].content();
|
||||
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('creates a directory with the wanted files with a subpath', async () => {
|
||||
const response = await reader.readTree(
|
||||
'https://bitbucket.org/backstage/mock/src/master/docs',
|
||||
);
|
||||
|
||||
const dir = await response.dir({ targetDir: mockDir.path });
|
||||
|
||||
await expect(
|
||||
fs.readFile(path.join(dir, 'index.md'), 'utf8'),
|
||||
).resolves.toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('throws a NotModifiedError when given a etag in options', async () => {
|
||||
const fnBitbucket = async () => {
|
||||
await reader.readTree('https://bitbucket.org/backstage/mock', {
|
||||
etag: '12ab34cd56ef',
|
||||
});
|
||||
};
|
||||
|
||||
await expect(fnBitbucket).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should not throw a NotModifiedError when given an outdated etag in options', async () => {
|
||||
const response = await reader.readTree(
|
||||
'https://bitbucket.org/backstage/mock',
|
||||
{ etag: 'outdatedetag123abc' },
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('12ab34cd56ef');
|
||||
});
|
||||
});
|
||||
|
||||
describe('search hosted', () => {
|
||||
const repoBuffer = fs.readFileSync(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/bitbucket-repo-with-commit-hash.tar.gz',
|
||||
),
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage/mock',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
mainbranch: {
|
||||
type: 'branch',
|
||||
name: 'master',
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://bitbucket.org/backstage/mock/get/master.tar.gz',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz',
|
||||
),
|
||||
ctx.body(repoBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('works for the naive case', async () => {
|
||||
const result = await reader.search(
|
||||
'https://bitbucket.org/backstage/mock/src/master/**/index.*',
|
||||
);
|
||||
expect(result.etag).toBe('12ab34cd56ef');
|
||||
expect(result.files.length).toBe(1);
|
||||
expect(result.files[0].url).toBe(
|
||||
'https://bitbucket.org/backstage/mock/src/master/docs/index.md',
|
||||
);
|
||||
await expect(result.files[0].content()).resolves.toEqual(
|
||||
Buffer.from('# Test\n'),
|
||||
);
|
||||
});
|
||||
|
||||
it('works in nested folders', async () => {
|
||||
const result = await reader.search(
|
||||
'https://bitbucket.org/backstage/mock/src/master/docs/index.*',
|
||||
);
|
||||
expect(result.etag).toBe('12ab34cd56ef');
|
||||
expect(result.files.length).toBe(1);
|
||||
expect(result.files[0].url).toBe(
|
||||
'https://bitbucket.org/backstage/mock/src/master/docs/index.md',
|
||||
);
|
||||
await expect(result.files[0].content()).resolves.toEqual(
|
||||
Buffer.from('# Test\n'),
|
||||
);
|
||||
});
|
||||
|
||||
it('throws NotModifiedError when same etag', async () => {
|
||||
await expect(
|
||||
reader.search(
|
||||
'https://bitbucket.org/backstage/mock/src/master/**/index.*',
|
||||
{ etag: '12ab34cd56ef' },
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,242 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { NotFoundError, NotModifiedError } from '@backstage/errors';
|
||||
import {
|
||||
BitbucketCloudIntegration,
|
||||
getBitbucketCloudDefaultBranch,
|
||||
getBitbucketCloudDownloadUrl,
|
||||
getBitbucketCloudFileFetchUrl,
|
||||
getBitbucketCloudRequestOptions,
|
||||
ScmIntegrations,
|
||||
} from '@backstage/integration';
|
||||
import fetch, { Response } from 'node-fetch';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { trimEnd } from 'lodash';
|
||||
import { Minimatch } from 'minimatch';
|
||||
import { Readable } from 'stream';
|
||||
import {
|
||||
ReaderFactory,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseFactory,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
SearchOptions,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
} from './types';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import { parseLastModified } from './util';
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Cloud.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class BitbucketCloudUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
return integrations.bitbucketCloud.list().map(integration => {
|
||||
const reader = new BitbucketCloudUrlReader(integration, {
|
||||
treeResponseFactory,
|
||||
});
|
||||
const predicate = (url: URL) => url.host === integration.config.host;
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly integration: BitbucketCloudIntegration,
|
||||
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
|
||||
) {
|
||||
const { host, username, appPassword } = integration.config;
|
||||
|
||||
if (username && !appPassword) {
|
||||
throw new Error(
|
||||
`Bitbucket Cloud integration for '${host}' has configured a username but is missing a required appPassword.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
const { etag, lastModifiedAfter, signal } = options ?? {};
|
||||
const bitbucketUrl = getBitbucketCloudFileFetchUrl(
|
||||
url,
|
||||
this.integration.config,
|
||||
);
|
||||
const requestOptions = getBitbucketCloudRequestOptions(
|
||||
this.integration.config,
|
||||
);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(bitbucketUrl.toString(), {
|
||||
headers: {
|
||||
...requestOptions.headers,
|
||||
...(etag && { 'If-None-Match': etag }),
|
||||
...(lastModifiedAfter && {
|
||||
'If-Modified-Since': lastModifiedAfter.toUTCString(),
|
||||
}),
|
||||
},
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can be
|
||||
// removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
...(signal && { signal: signal as any }),
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${url}, ${e}`);
|
||||
}
|
||||
|
||||
if (response.status === 304) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
if (response.ok) {
|
||||
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
|
||||
etag: response.headers.get('ETag') ?? undefined,
|
||||
lastModifiedAt: parseLastModified(
|
||||
response.headers.get('Last-Modified'),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`;
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
const { filepath } = parseGitUrl(url);
|
||||
|
||||
const lastCommitShortHash = await this.getLastCommitShortHash(url);
|
||||
if (options?.etag && options.etag === lastCommitShortHash) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
const downloadUrl = await getBitbucketCloudDownloadUrl(
|
||||
url,
|
||||
this.integration.config,
|
||||
);
|
||||
const archiveResponse = await fetch(
|
||||
downloadUrl,
|
||||
getBitbucketCloudRequestOptions(this.integration.config),
|
||||
);
|
||||
if (!archiveResponse.ok) {
|
||||
const message = `Failed to read tree from ${url}, ${archiveResponse.status} ${archiveResponse.statusText}`;
|
||||
if (archiveResponse.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
return await this.deps.treeResponseFactory.fromTarArchive({
|
||||
stream: Readable.from(archiveResponse.body),
|
||||
subpath: filepath,
|
||||
etag: lastCommitShortHash,
|
||||
filter: options?.filter,
|
||||
});
|
||||
}
|
||||
|
||||
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
|
||||
const { filepath } = parseGitUrl(url);
|
||||
const matcher = new Minimatch(filepath);
|
||||
|
||||
// TODO(freben): For now, read the entire repo and filter through that. In
|
||||
// a future improvement, we could be smart and try to deduce that non-glob
|
||||
// prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used
|
||||
// to get just that part of the repo.
|
||||
const treeUrl = trimEnd(url.replace(filepath, ''), '/');
|
||||
|
||||
const tree = await this.readTree(treeUrl, {
|
||||
etag: options?.etag,
|
||||
filter: path => matcher.match(path),
|
||||
});
|
||||
const files = await tree.files();
|
||||
|
||||
return {
|
||||
etag: tree.etag,
|
||||
files: files.map(file => ({
|
||||
url: this.integration.resolveUrl({
|
||||
url: `/${file.path}`,
|
||||
base: url,
|
||||
}),
|
||||
content: file.content,
|
||||
lastModifiedAt: file.lastModifiedAt,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
toString() {
|
||||
const { host, username, appPassword } = this.integration.config;
|
||||
const authed = Boolean(username && appPassword);
|
||||
return `bitbucketCloud{host=${host},authed=${authed}}`;
|
||||
}
|
||||
|
||||
private async getLastCommitShortHash(url: string): Promise<string> {
|
||||
const { name: repoName, owner: project, ref } = parseGitUrl(url);
|
||||
|
||||
let branch = ref;
|
||||
if (!branch) {
|
||||
branch = await getBitbucketCloudDefaultBranch(
|
||||
url,
|
||||
this.integration.config,
|
||||
);
|
||||
}
|
||||
|
||||
const commitsApiUrl = `${this.integration.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`;
|
||||
|
||||
const commitsResponse = await fetch(
|
||||
commitsApiUrl,
|
||||
getBitbucketCloudRequestOptions(this.integration.config),
|
||||
);
|
||||
if (!commitsResponse.ok) {
|
||||
const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`;
|
||||
if (commitsResponse.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
const commits = await commitsResponse.json();
|
||||
if (
|
||||
commits &&
|
||||
commits.values &&
|
||||
commits.values.length > 0 &&
|
||||
commits.values[0].hash
|
||||
) {
|
||||
return commits.values[0].hash.substring(0, 12);
|
||||
}
|
||||
|
||||
throw new Error(`Failed to read response from ${commitsApiUrl}`);
|
||||
}
|
||||
}
|
||||
@@ -1,281 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import {
|
||||
BitbucketServerIntegration,
|
||||
readBitbucketServerIntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
import {
|
||||
createMockDirectory,
|
||||
setupRequestMockHandlers,
|
||||
} from '@backstage/backend-test-utils';
|
||||
import fs from 'fs-extra';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import path from 'path';
|
||||
import { NotModifiedError } from '@backstage/errors';
|
||||
import { BitbucketServerUrlReader } from './BitbucketServerUrlReader';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
|
||||
createMockDirectory({ mockOsTmpDir: true });
|
||||
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
});
|
||||
|
||||
const reader = new BitbucketServerUrlReader(
|
||||
new BitbucketServerIntegration(
|
||||
readBitbucketServerIntegrationConfig(
|
||||
new ConfigReader({
|
||||
host: 'bitbucket.mycompany.net',
|
||||
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
|
||||
}),
|
||||
),
|
||||
),
|
||||
{ treeResponseFactory },
|
||||
);
|
||||
|
||||
describe('BitbucketServerUrlReader', () => {
|
||||
const worker = setupServer();
|
||||
setupRequestMockHandlers(worker);
|
||||
|
||||
describe('readTree', () => {
|
||||
const repoBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/bitbucket-server-repo.tar.gz'),
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=backstage-mock.tgz',
|
||||
),
|
||||
ctx.body(repoBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
size: 2,
|
||||
values: [
|
||||
{
|
||||
displayId: 'some-branch-that-should-be-ignored',
|
||||
latestCommit: 'bogus hash',
|
||||
},
|
||||
{
|
||||
displayId: 'some-branch',
|
||||
latestCommit: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st',
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches/default',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
id: 'refs/heads/master',
|
||||
displayId: 'master',
|
||||
type: 'BRANCH',
|
||||
latestCommit: '3bdd5457286abdf920db4b77bf2fef79a06190c2',
|
||||
latestChangeset: '3bdd5457286abdf920db4b77bf2fef79a06190c2',
|
||||
isDefault: true,
|
||||
}),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/default-branch',
|
||||
(_, res, ctx) => res(ctx.status(404)),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('uses private bitbucket host', async () => {
|
||||
const response = await reader.readTree(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch',
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('12ab34cd56ef');
|
||||
|
||||
const files = await response.files();
|
||||
|
||||
expect(files.length).toBe(1);
|
||||
const indexMarkdownFile = await files[0].content();
|
||||
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('uses default branch when no branch is provided', async () => {
|
||||
const response = await reader.readTree(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/src',
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('3bdd5457286a');
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTree without branch', () => {
|
||||
const repoBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/bitbucket-server-repo.tar.gz'),
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=backstage-mock.tgz',
|
||||
),
|
||||
ctx.body(repoBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
size: 2,
|
||||
values: [
|
||||
{
|
||||
displayId: 'some-branch-that-should-be-ignored',
|
||||
latestCommit: 'bogus hash',
|
||||
},
|
||||
{
|
||||
displayId: 'some-branch',
|
||||
latestCommit: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st',
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('uses private bitbucket host', async () => {
|
||||
const response = await reader.readTree(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch',
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('12ab34cd56ef');
|
||||
|
||||
const files = await response.files();
|
||||
|
||||
expect(files.length).toBe(1);
|
||||
const indexMarkdownFile = await files[0].content();
|
||||
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
});
|
||||
|
||||
describe('search private', () => {
|
||||
const repoBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/bitbucket-server-repo.tar.gz'),
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=backstage-mock.tgz',
|
||||
),
|
||||
ctx.body(repoBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
size: 2,
|
||||
values: [
|
||||
{
|
||||
displayId: 'master-of-none',
|
||||
latestCommit: 'bogus hash',
|
||||
},
|
||||
{
|
||||
displayId: 'master',
|
||||
latestCommit: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st',
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('works for the naive case', async () => {
|
||||
const result = await reader.search(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master',
|
||||
);
|
||||
expect(result.etag).toBe('12ab34cd56ef');
|
||||
expect(result.files.length).toBe(1);
|
||||
expect(result.files[0].url).toBe(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master',
|
||||
);
|
||||
await expect(result.files[0].content()).resolves.toEqual(
|
||||
Buffer.from('# Test\n'),
|
||||
);
|
||||
});
|
||||
|
||||
it('works in nested folders', async () => {
|
||||
const result = await reader.search(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.*?at=master',
|
||||
);
|
||||
expect(result.etag).toBe('12ab34cd56ef');
|
||||
expect(result.files.length).toBe(1);
|
||||
expect(result.files[0].url).toBe(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master',
|
||||
);
|
||||
await expect(result.files[0].content()).resolves.toEqual(
|
||||
Buffer.from('# Test\n'),
|
||||
);
|
||||
});
|
||||
|
||||
it('throws NotModifiedError when same etag', async () => {
|
||||
await expect(
|
||||
reader.search(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master',
|
||||
{ etag: '12ab34cd56ef' },
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,240 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { NotFoundError, NotModifiedError } from '@backstage/errors';
|
||||
import {
|
||||
BitbucketServerIntegration,
|
||||
getBitbucketServerDownloadUrl,
|
||||
getBitbucketServerFileFetchUrl,
|
||||
getBitbucketServerRequestOptions,
|
||||
ScmIntegrations,
|
||||
} from '@backstage/integration';
|
||||
import fetch, { Response } from 'node-fetch';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { trimEnd } from 'lodash';
|
||||
import { Minimatch } from 'minimatch';
|
||||
import { Readable } from 'stream';
|
||||
import {
|
||||
ReaderFactory,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseFactory,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
SearchOptions,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
} from './types';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import { parseLastModified } from './util';
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Server APIs.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class BitbucketServerUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
return integrations.bitbucketServer.list().map(integration => {
|
||||
const reader = new BitbucketServerUrlReader(integration, {
|
||||
treeResponseFactory,
|
||||
});
|
||||
const predicate = (url: URL) => url.host === integration.config.host;
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly integration: BitbucketServerIntegration,
|
||||
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
|
||||
) {}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
const { etag, lastModifiedAfter, signal } = options ?? {};
|
||||
const bitbucketUrl = getBitbucketServerFileFetchUrl(
|
||||
url,
|
||||
this.integration.config,
|
||||
);
|
||||
const requestOptions = getBitbucketServerRequestOptions(
|
||||
this.integration.config,
|
||||
);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(bitbucketUrl.toString(), {
|
||||
headers: {
|
||||
...requestOptions.headers,
|
||||
...(etag && { 'If-None-Match': etag }),
|
||||
...(lastModifiedAfter && {
|
||||
'If-Modified-Since': lastModifiedAfter.toUTCString(),
|
||||
}),
|
||||
},
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can be
|
||||
// removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
...(signal && { signal: signal as any }),
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${url}, ${e}`);
|
||||
}
|
||||
|
||||
if (response.status === 304) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
if (response.ok) {
|
||||
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
|
||||
etag: response.headers.get('ETag') ?? undefined,
|
||||
lastModifiedAt: parseLastModified(
|
||||
response.headers.get('Last-Modified'),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`;
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
const { filepath } = parseGitUrl(url);
|
||||
|
||||
const lastCommitShortHash = await this.getLastCommitShortHash(url);
|
||||
if (options?.etag && options.etag === lastCommitShortHash) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
const downloadUrl = await getBitbucketServerDownloadUrl(
|
||||
url,
|
||||
this.integration.config,
|
||||
);
|
||||
const archiveResponse = await fetch(
|
||||
downloadUrl,
|
||||
getBitbucketServerRequestOptions(this.integration.config),
|
||||
);
|
||||
if (!archiveResponse.ok) {
|
||||
const message = `Failed to read tree from ${url}, ${archiveResponse.status} ${archiveResponse.statusText}`;
|
||||
if (archiveResponse.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
return await this.deps.treeResponseFactory.fromTarArchive({
|
||||
stream: Readable.from(archiveResponse.body),
|
||||
subpath: filepath,
|
||||
etag: lastCommitShortHash,
|
||||
filter: options?.filter,
|
||||
});
|
||||
}
|
||||
|
||||
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
|
||||
const { filepath } = parseGitUrl(url);
|
||||
const matcher = new Minimatch(filepath);
|
||||
|
||||
// TODO(freben): For now, read the entire repo and filter through that. In
|
||||
// a future improvement, we could be smart and try to deduce that non-glob
|
||||
// prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used
|
||||
// to get just that part of the repo.
|
||||
const treeUrl = trimEnd(url.replace(filepath, ''), '/');
|
||||
|
||||
const tree = await this.readTree(treeUrl, {
|
||||
etag: options?.etag,
|
||||
filter: path => matcher.match(path),
|
||||
});
|
||||
const files = await tree.files();
|
||||
|
||||
return {
|
||||
etag: tree.etag,
|
||||
files: files.map(file => ({
|
||||
url: this.integration.resolveUrl({
|
||||
url: `/${file.path}`,
|
||||
base: url,
|
||||
}),
|
||||
content: file.content,
|
||||
lastModifiedAt: file.lastModifiedAt,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
toString() {
|
||||
const { host, token } = this.integration.config;
|
||||
const authed = Boolean(token);
|
||||
return `bitbucketServer{host=${host},authed=${authed}}`;
|
||||
}
|
||||
|
||||
private async getLastCommitShortHash(url: string): Promise<string> {
|
||||
const { name: repoName, owner: project, ref: branch } = parseGitUrl(url);
|
||||
|
||||
// If a branch is provided use that otherwise fall back to the default branch
|
||||
const branchParameter = branch
|
||||
? `?filterText=${encodeURIComponent(branch)}`
|
||||
: '/default';
|
||||
|
||||
// https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp211 (branches docs)
|
||||
const branchListUrl = `${this.integration.config.apiBaseUrl}/projects/${project}/repos/${repoName}/branches${branchParameter}`;
|
||||
|
||||
const branchListResponse = await fetch(
|
||||
branchListUrl,
|
||||
getBitbucketServerRequestOptions(this.integration.config),
|
||||
);
|
||||
if (!branchListResponse.ok) {
|
||||
const message = `Failed to retrieve branch list from ${branchListUrl}, ${branchListResponse.status} ${branchListResponse.statusText}`;
|
||||
if (branchListResponse.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
const branchMatches = await branchListResponse.json();
|
||||
|
||||
if (branchMatches && branchMatches.size > 0) {
|
||||
const exactBranchMatch = branchMatches.values.filter(
|
||||
(branchDetails: { displayId: string }) =>
|
||||
branchDetails.displayId === branch,
|
||||
)[0];
|
||||
return exactBranchMatch.latestCommit.substring(0, 12);
|
||||
}
|
||||
|
||||
// Handle when no branch is provided using the default as the fallback
|
||||
if (!branch && branchMatches) {
|
||||
return branchMatches.latestCommit.substring(0, 12);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Failed to find Last Commit using ${
|
||||
branch ? `branch "${branch}"` : 'default branch'
|
||||
} in response from ${branchListUrl}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,617 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import {
|
||||
BitbucketIntegration,
|
||||
readBitbucketIntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
import {
|
||||
createMockDirectory,
|
||||
mockServices,
|
||||
setupRequestMockHandlers,
|
||||
} from '@backstage/backend-test-utils';
|
||||
import fs from 'fs-extra';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import path from 'path';
|
||||
import { NotModifiedError } from '@backstage/errors';
|
||||
import { BitbucketUrlReader } from './BitbucketUrlReader';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
import getRawBody from 'raw-body';
|
||||
|
||||
const logger = mockServices.logger.mock();
|
||||
|
||||
describe('BitbucketUrlReader.factory', () => {
|
||||
it('only apply integration configs not inherited from bitbucketCloud or bitbucketServer', () => {
|
||||
const config = new ConfigReader({
|
||||
integrations: {
|
||||
bitbucket: [],
|
||||
bitbucketCloud: [
|
||||
{
|
||||
username: 'username',
|
||||
appPassword: 'password',
|
||||
},
|
||||
],
|
||||
bitbucketServer: [
|
||||
{
|
||||
host: 'bitbucket-server.local',
|
||||
token: 'test-token',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config: config,
|
||||
});
|
||||
|
||||
const tuples = BitbucketUrlReader.factory({
|
||||
config,
|
||||
logger,
|
||||
treeResponseFactory,
|
||||
});
|
||||
|
||||
expect(tuples).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('BitbucketUrlReader', () => {
|
||||
const mockDir = createMockDirectory({ mockOsTmpDir: true });
|
||||
|
||||
beforeEach(mockDir.clear);
|
||||
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
});
|
||||
|
||||
const bitbucketProcessor = new BitbucketUrlReader(
|
||||
new BitbucketIntegration(
|
||||
readBitbucketIntegrationConfig(
|
||||
new ConfigReader({
|
||||
host: 'bitbucket.org',
|
||||
apiBaseUrl: 'https://api.bitbucket.org/2.0',
|
||||
}),
|
||||
),
|
||||
),
|
||||
logger,
|
||||
{ treeResponseFactory },
|
||||
);
|
||||
|
||||
const hostedBitbucketProcessor = new BitbucketUrlReader(
|
||||
new BitbucketIntegration(
|
||||
readBitbucketIntegrationConfig(
|
||||
new ConfigReader({
|
||||
host: 'bitbucket.mycompany.net',
|
||||
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
|
||||
}),
|
||||
),
|
||||
),
|
||||
logger,
|
||||
{ treeResponseFactory },
|
||||
);
|
||||
|
||||
const worker = setupServer();
|
||||
setupRequestMockHandlers(worker);
|
||||
|
||||
describe('readUrl', () => {
|
||||
it('should be able to readUrl via buffer without ETag', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-None-Match')).toBeNull();
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body('foo'),
|
||||
ctx.set('ETag', 'etag-value'),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await bitbucketProcessor.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
);
|
||||
const buffer = await result.buffer();
|
||||
expect(buffer.toString()).toBe('foo');
|
||||
});
|
||||
|
||||
it('should be able to readUrl via stream without ETag', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-None-Match')).toBeNull();
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body('foo'),
|
||||
ctx.set('ETag', 'etag-value'),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await bitbucketProcessor.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
);
|
||||
const fromStream = await getRawBody(result.stream!());
|
||||
expect(fromStream.toString()).toBe('foo');
|
||||
});
|
||||
|
||||
it('should be able to readUrl with matching ETag', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-None-Match')).toBe(
|
||||
'matching-etag-value',
|
||||
);
|
||||
return res(ctx.status(304));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
bitbucketProcessor.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
{ etag: 'matching-etag-value' },
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should be able to readUrl without matching ETag', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-None-Match')).toBe(
|
||||
'previous-etag-value',
|
||||
);
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body('foo'),
|
||||
ctx.set('ETag', 'new-etag-value'),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await bitbucketProcessor.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
{ etag: 'previous-etag-value' },
|
||||
);
|
||||
const buffer = await result.buffer();
|
||||
expect(buffer.toString()).toBe('foo');
|
||||
expect(result.etag).toBe('new-etag-value');
|
||||
});
|
||||
|
||||
it('should be able to readUrl via buffer without If-Modified-Since', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-None-Match')).toBeNull();
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body('foo'),
|
||||
ctx.set('ETag', 'etag-value'),
|
||||
ctx.set(
|
||||
'Last-Modified',
|
||||
new Date('2020-01-01T00:00:00Z').toUTCString(),
|
||||
),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await bitbucketProcessor.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
);
|
||||
const buffer = await result.buffer();
|
||||
expect(result.lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
|
||||
expect(buffer.toString()).toBe('foo');
|
||||
});
|
||||
|
||||
it('should be throw not modified when If-Modified-Since returns a 304', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-Modified-Since')).toBe(
|
||||
new Date('1999 12 31 23:59:59 GMT').toUTCString(),
|
||||
);
|
||||
return res(ctx.status(304));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
bitbucketProcessor.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
{ lastModifiedAfter: new Date('1999 12 31 23:59:59 GMT') },
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should be able to readUrl when If-Modified-Since is before Last-Modified', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('If-Modified-Since')).toBe(
|
||||
new Date('1999 12 31 23:59:59 GMT').toUTCString(),
|
||||
);
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set(
|
||||
'Last-Modified',
|
||||
new Date('2020-01-01T00:00:00Z').toUTCString(),
|
||||
),
|
||||
ctx.body('foo'),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await bitbucketProcessor.readUrl(
|
||||
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
|
||||
{ lastModifiedAfter: new Date('1999 12 31 23:59:59 GMT') },
|
||||
);
|
||||
const buffer = await result.buffer();
|
||||
expect(buffer.toString()).toBe('foo');
|
||||
expect(result.lastModifiedAt).toEqual(new Date('2020-01-01T00:00:00Z'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('read', () => {
|
||||
it('rejects unknown targets', async () => {
|
||||
await expect(
|
||||
bitbucketProcessor.read('https://not.bitbucket.com/apa'),
|
||||
).rejects.toThrow(
|
||||
'Incorrect URL: https://not.bitbucket.com/apa, Error: Invalid Bitbucket URL or file path',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTree', () => {
|
||||
const repoBuffer = fs.readFileSync(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/bitbucket-repo-with-commit-hash.tar.gz',
|
||||
),
|
||||
);
|
||||
|
||||
const privateBitbucketRepoBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/bitbucket-server-repo.tar.gz'),
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage/mock',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
mainbranch: {
|
||||
type: 'branch',
|
||||
name: 'master',
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://bitbucket.org/backstage/mock/get/master.tar.gz',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz',
|
||||
),
|
||||
ctx.body(repoBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
|
||||
}),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=backstage-mock.tgz',
|
||||
),
|
||||
ctx.body(privateBitbucketRepoBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/commits',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
values: [{ id: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns the wanted files from an archive', async () => {
|
||||
const response = await bitbucketProcessor.readTree(
|
||||
'https://bitbucket.org/backstage/mock/src/master',
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('12ab34cd56ef');
|
||||
|
||||
const files = await response.files();
|
||||
|
||||
expect(files.length).toBe(2);
|
||||
const mkDocsFile = await files[0].content();
|
||||
const indexMarkdownFile = await files[1].content();
|
||||
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
|
||||
});
|
||||
|
||||
it('creates a directory with the wanted files', async () => {
|
||||
const response = await bitbucketProcessor.readTree(
|
||||
'https://bitbucket.org/backstage/mock',
|
||||
);
|
||||
|
||||
const dir = await response.dir({ targetDir: mockDir.path });
|
||||
|
||||
await expect(
|
||||
fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'),
|
||||
).resolves.toBe('site_name: Test\n');
|
||||
await expect(
|
||||
fs.readFile(path.join(dir, 'docs', 'index.md'), 'utf8'),
|
||||
).resolves.toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('uses private bitbucket host', async () => {
|
||||
const response = await hostedBitbucketProcessor.readTree(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch',
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('12ab34cd56ef');
|
||||
|
||||
const files = await response.files();
|
||||
|
||||
expect(files.length).toBe(1);
|
||||
const indexMarkdownFile = await files[0].content();
|
||||
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('returns the wanted files from an archive with a subpath', async () => {
|
||||
const response = await bitbucketProcessor.readTree(
|
||||
'https://bitbucket.org/backstage/mock/src/master/docs',
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('12ab34cd56ef');
|
||||
|
||||
const files = await response.files();
|
||||
|
||||
expect(files.length).toBe(1);
|
||||
const indexMarkdownFile = await files[0].content();
|
||||
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('creates a directory with the wanted files with a subpath', async () => {
|
||||
const response = await bitbucketProcessor.readTree(
|
||||
'https://bitbucket.org/backstage/mock/src/master/docs',
|
||||
);
|
||||
|
||||
const dir = await response.dir({ targetDir: mockDir.path });
|
||||
|
||||
await expect(
|
||||
fs.readFile(path.join(dir, 'index.md'), 'utf8'),
|
||||
).resolves.toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('throws a NotModifiedError when given a etag in options', async () => {
|
||||
const fnBitbucket = async () => {
|
||||
await bitbucketProcessor.readTree(
|
||||
'https://bitbucket.org/backstage/mock',
|
||||
{ etag: '12ab34cd56ef' },
|
||||
);
|
||||
};
|
||||
|
||||
await expect(fnBitbucket).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should not throw a NotModifiedError when given an outdated etag in options', async () => {
|
||||
const response = await bitbucketProcessor.readTree(
|
||||
'https://bitbucket.org/backstage/mock',
|
||||
{ etag: 'outdatedetag123abc' },
|
||||
);
|
||||
|
||||
expect(response.etag).toBe('12ab34cd56ef');
|
||||
});
|
||||
});
|
||||
|
||||
describe('search hosted', () => {
|
||||
const repoBuffer = fs.readFileSync(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'__fixtures__/bitbucket-repo-with-commit-hash.tar.gz',
|
||||
),
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage/mock',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
mainbranch: {
|
||||
type: 'branch',
|
||||
name: 'master',
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://bitbucket.org/backstage/mock/get/master.tar.gz',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz',
|
||||
),
|
||||
ctx.body(repoBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('works for the naive case', async () => {
|
||||
const result = await bitbucketProcessor.search(
|
||||
'https://bitbucket.org/backstage/mock/src/master/**/index.*',
|
||||
);
|
||||
expect(result.etag).toBe('12ab34cd56ef');
|
||||
expect(result.files.length).toBe(1);
|
||||
expect(result.files[0].url).toBe(
|
||||
'https://bitbucket.org/backstage/mock/src/master/docs/index.md',
|
||||
);
|
||||
await expect(result.files[0].content()).resolves.toEqual(
|
||||
Buffer.from('# Test\n'),
|
||||
);
|
||||
});
|
||||
|
||||
it('works in nested folders', async () => {
|
||||
const result = await bitbucketProcessor.search(
|
||||
'https://bitbucket.org/backstage/mock/src/master/docs/index.*',
|
||||
);
|
||||
expect(result.etag).toBe('12ab34cd56ef');
|
||||
expect(result.files.length).toBe(1);
|
||||
expect(result.files[0].url).toBe(
|
||||
'https://bitbucket.org/backstage/mock/src/master/docs/index.md',
|
||||
);
|
||||
await expect(result.files[0].content()).resolves.toEqual(
|
||||
Buffer.from('# Test\n'),
|
||||
);
|
||||
});
|
||||
|
||||
it('throws NotModifiedError when same etag', async () => {
|
||||
await expect(
|
||||
bitbucketProcessor.search(
|
||||
'https://bitbucket.org/backstage/mock/src/master/**/index.*',
|
||||
{ etag: '12ab34cd56ef' },
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('search private', () => {
|
||||
const privateBitbucketRepoBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/bitbucket-server-repo.tar.gz'),
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=backstage-mock.tgz',
|
||||
),
|
||||
ctx.body(privateBitbucketRepoBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/commits',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
values: [{ id: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('works for the naive case', async () => {
|
||||
const result = await hostedBitbucketProcessor.search(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master',
|
||||
);
|
||||
expect(result.etag).toBe('12ab34cd56ef');
|
||||
expect(result.files.length).toBe(1);
|
||||
expect(result.files[0].url).toBe(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master',
|
||||
);
|
||||
await expect(result.files[0].content()).resolves.toEqual(
|
||||
Buffer.from('# Test\n'),
|
||||
);
|
||||
});
|
||||
|
||||
it('works in nested folders', async () => {
|
||||
const result = await hostedBitbucketProcessor.search(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.*?at=master',
|
||||
);
|
||||
expect(result.etag).toBe('12ab34cd56ef');
|
||||
expect(result.files.length).toBe(1);
|
||||
expect(result.files[0].url).toBe(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master',
|
||||
);
|
||||
await expect(result.files[0].content()).resolves.toEqual(
|
||||
Buffer.from('# Test\n'),
|
||||
);
|
||||
});
|
||||
|
||||
it('throws NotModifiedError when same etag', async () => {
|
||||
await expect(
|
||||
hostedBitbucketProcessor.search(
|
||||
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master',
|
||||
{ etag: '12ab34cd56ef' },
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,268 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { NotFoundError, NotModifiedError } from '@backstage/errors';
|
||||
import {
|
||||
BitbucketIntegration,
|
||||
getBitbucketDefaultBranch,
|
||||
getBitbucketDownloadUrl,
|
||||
getBitbucketFileFetchUrl,
|
||||
getBitbucketRequestOptions,
|
||||
ScmIntegrations,
|
||||
} from '@backstage/integration';
|
||||
import fetch, { Response } from 'node-fetch';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { trimEnd } from 'lodash';
|
||||
import { Minimatch } from 'minimatch';
|
||||
import { Readable } from 'stream';
|
||||
import { LoggerService } from '@backstage/backend-plugin-api';
|
||||
import {
|
||||
ReaderFactory,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseFactory,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
SearchOptions,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
} from './types';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import { parseLastModified } from './util';
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket v1 and v2 APIs, such
|
||||
* as the one exposed by Bitbucket Cloud itself.
|
||||
*
|
||||
* @public
|
||||
* @deprecated in favor of BitbucketCloudUrlReader and BitbucketServerUrlReader
|
||||
*/
|
||||
export class BitbucketUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, logger, treeResponseFactory }) => {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
return integrations.bitbucket
|
||||
.list()
|
||||
.filter(
|
||||
item =>
|
||||
!integrations.bitbucketCloud.byHost(item.config.host) &&
|
||||
!integrations.bitbucketServer.byHost(item.config.host),
|
||||
)
|
||||
.map(integration => {
|
||||
const reader = new BitbucketUrlReader(integration, logger, {
|
||||
treeResponseFactory,
|
||||
});
|
||||
const predicate = (url: URL) => url.host === integration.config.host;
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly integration: BitbucketIntegration,
|
||||
logger: LoggerService,
|
||||
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
|
||||
) {
|
||||
const { host, token, username, appPassword } = integration.config;
|
||||
const replacement =
|
||||
host === 'bitbucket.org' ? 'bitbucketCloud' : 'bitbucketServer';
|
||||
logger.warn(
|
||||
`[Deprecated] Please migrate from "integrations.bitbucket" to "integrations.${replacement}".`,
|
||||
);
|
||||
|
||||
if (!token && username && !appPassword) {
|
||||
throw new Error(
|
||||
`Bitbucket integration for '${host}' has configured a username but is missing a required appPassword.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
const { etag, lastModifiedAfter, signal } = options ?? {};
|
||||
const bitbucketUrl = getBitbucketFileFetchUrl(url, this.integration.config);
|
||||
const requestOptions = getBitbucketRequestOptions(this.integration.config);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(bitbucketUrl.toString(), {
|
||||
headers: {
|
||||
...requestOptions.headers,
|
||||
...(etag && { 'If-None-Match': etag }),
|
||||
...(lastModifiedAfter && {
|
||||
'If-Modified-Since': lastModifiedAfter.toUTCString(),
|
||||
}),
|
||||
},
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can be
|
||||
// removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
...(signal && { signal: signal as any }),
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${url}, ${e}`);
|
||||
}
|
||||
|
||||
if (response.status === 304) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
if (response.ok) {
|
||||
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
|
||||
etag: response.headers.get('ETag') ?? undefined,
|
||||
lastModifiedAt: parseLastModified(
|
||||
response.headers.get('Last-Modified'),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`;
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
const { filepath } = parseGitUrl(url);
|
||||
|
||||
const lastCommitShortHash = await this.getLastCommitShortHash(url);
|
||||
if (options?.etag && options.etag === lastCommitShortHash) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
const downloadUrl = await getBitbucketDownloadUrl(
|
||||
url,
|
||||
this.integration.config,
|
||||
);
|
||||
const archiveBitbucketResponse = await fetch(
|
||||
downloadUrl,
|
||||
getBitbucketRequestOptions(this.integration.config),
|
||||
);
|
||||
if (!archiveBitbucketResponse.ok) {
|
||||
const message = `Failed to read tree from ${url}, ${archiveBitbucketResponse.status} ${archiveBitbucketResponse.statusText}`;
|
||||
if (archiveBitbucketResponse.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
return await this.deps.treeResponseFactory.fromTarArchive({
|
||||
stream: Readable.from(archiveBitbucketResponse.body),
|
||||
subpath: filepath,
|
||||
etag: lastCommitShortHash,
|
||||
filter: options?.filter,
|
||||
});
|
||||
}
|
||||
|
||||
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
|
||||
const { filepath } = parseGitUrl(url);
|
||||
const matcher = new Minimatch(filepath);
|
||||
|
||||
// TODO(freben): For now, read the entire repo and filter through that. In
|
||||
// a future improvement, we could be smart and try to deduce that non-glob
|
||||
// prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used
|
||||
// to get just that part of the repo.
|
||||
const treeUrl = trimEnd(url.replace(filepath, ''), '/');
|
||||
|
||||
const tree = await this.readTree(treeUrl, {
|
||||
etag: options?.etag,
|
||||
filter: path => matcher.match(path),
|
||||
});
|
||||
const files = await tree.files();
|
||||
|
||||
return {
|
||||
etag: tree.etag,
|
||||
files: files.map(file => ({
|
||||
url: this.integration.resolveUrl({
|
||||
url: `/${file.path}`,
|
||||
base: url,
|
||||
}),
|
||||
content: file.content,
|
||||
lastModifiedAt: file.lastModifiedAt,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
toString() {
|
||||
const { host, token, username, appPassword } = this.integration.config;
|
||||
let authed = Boolean(token);
|
||||
if (!authed) {
|
||||
authed = Boolean(username && appPassword);
|
||||
}
|
||||
return `bitbucket{host=${host},authed=${authed}}`;
|
||||
}
|
||||
|
||||
private async getLastCommitShortHash(url: string): Promise<string> {
|
||||
const { resource, name: repoName, owner: project, ref } = parseGitUrl(url);
|
||||
|
||||
let branch = ref;
|
||||
if (!branch) {
|
||||
branch = await getBitbucketDefaultBranch(url, this.integration.config);
|
||||
}
|
||||
|
||||
const isHosted = resource === 'bitbucket.org';
|
||||
// Bitbucket Server https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp222
|
||||
const commitsApiUrl = isHosted
|
||||
? `${this.integration.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`
|
||||
: `${this.integration.config.apiBaseUrl}/projects/${project}/repos/${repoName}/commits`;
|
||||
|
||||
const commitsResponse = await fetch(
|
||||
commitsApiUrl,
|
||||
getBitbucketRequestOptions(this.integration.config),
|
||||
);
|
||||
if (!commitsResponse.ok) {
|
||||
const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`;
|
||||
if (commitsResponse.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
const commits = await commitsResponse.json();
|
||||
if (isHosted) {
|
||||
if (
|
||||
commits &&
|
||||
commits.values &&
|
||||
commits.values.length > 0 &&
|
||||
commits.values[0].hash
|
||||
) {
|
||||
return commits.values[0].hash.substring(0, 12);
|
||||
}
|
||||
} else {
|
||||
if (
|
||||
commits &&
|
||||
commits.values &&
|
||||
commits.values.length > 0 &&
|
||||
commits.values[0].id
|
||||
) {
|
||||
return commits.values[0].id.substring(0, 12);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Failed to read response from ${commitsApiUrl}`);
|
||||
}
|
||||
}
|
||||
@@ -1,269 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { NotFoundError, NotModifiedError } from '@backstage/errors';
|
||||
import {
|
||||
mockServices,
|
||||
setupRequestMockHandlers,
|
||||
} from '@backstage/backend-test-utils';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { FetchUrlReader } from './FetchUrlReader';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
|
||||
const fetchUrlReader = new FetchUrlReader();
|
||||
|
||||
describe('FetchUrlReader', () => {
|
||||
const worker = setupServer();
|
||||
|
||||
setupRequestMockHandlers(worker);
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get('https://backstage.io/some-resource', (req, res, ctx) => {
|
||||
if (req.headers.get('if-none-match') === 'foo') {
|
||||
return res(
|
||||
ctx.status(304),
|
||||
ctx.set('Content-Type', 'text/plain'),
|
||||
ctx.set('etag', 'foo'),
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
req.headers.get('if-modified-since') &&
|
||||
new Date(req.headers.get('if-modified-since') ?? '') <
|
||||
new Date('2021-01-01T00:00:00Z')
|
||||
) {
|
||||
return res(
|
||||
ctx.status(304),
|
||||
ctx.set('Content-Type', 'text/plain'),
|
||||
ctx.set(
|
||||
'last-modified',
|
||||
new Date('2021-01-01T00:00:00Z').toUTCString(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'text/plain'),
|
||||
ctx.set('etag', 'foo'),
|
||||
ctx.body('content foo'),
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
worker.use(
|
||||
rest.get('https://backstage.io/not-exists', (_req, res, ctx) => {
|
||||
return res(ctx.status(404));
|
||||
}),
|
||||
);
|
||||
|
||||
worker.use(
|
||||
rest.get('https://backstage.io/error', (_req, res, ctx) => {
|
||||
return res(ctx.status(500), ctx.body('An internal error occurred'));
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('factory should create a single entry with a predicate that matches config', async () => {
|
||||
const entries = FetchUrlReader.factory({
|
||||
config: new ConfigReader({
|
||||
backend: {
|
||||
reading: {
|
||||
allow: [
|
||||
{ host: 'example.com' },
|
||||
{ host: 'example.com:100-200' },
|
||||
{ host: 'example.com:700' },
|
||||
{ host: '*.examples.org' },
|
||||
{ host: '*.examples.org:700' },
|
||||
{ host: '*.examples.org:900-1000' },
|
||||
{ host: '*.examples.org:900-1000' },
|
||||
{ host: 'https.org:443' },
|
||||
{ host: 'http.org:80' },
|
||||
{
|
||||
host: 'foobar.org',
|
||||
paths: ['/dir1/'],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}),
|
||||
logger: mockServices.logger.mock(),
|
||||
treeResponseFactory: DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
}),
|
||||
});
|
||||
|
||||
expect(entries.length).toBe(1);
|
||||
const [{ predicate }] = entries;
|
||||
|
||||
expect(predicate(new URL('https://example.com/test'))).toBe(true);
|
||||
expect(predicate(new URL('https://a.example.com/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://example.com:600/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://a.example.com:600/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://example.com:700/test'))).toBe(true);
|
||||
expect(predicate(new URL('https://a.example.com:700/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://other.com/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://examples.org/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://a.examples.org/test'))).toBe(true);
|
||||
expect(predicate(new URL('https://a.b.examples.org/test'))).toBe(true);
|
||||
expect(predicate(new URL('https://examples.org:600/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://a.examples.org:600/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://a.b.examples.org:600/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://examples.org:700/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://a.examples.org:700/test'))).toBe(true);
|
||||
expect(predicate(new URL('https://a.b.examples.org:700/test'))).toBe(true);
|
||||
expect(predicate(new URL('https://foobar.org/dir1/subpath'))).toBe(true);
|
||||
expect(predicate(new URL('https://foobar.org/dir12'))).toBe(false);
|
||||
expect(predicate(new URL('https://foobar.org/'))).toBe(false);
|
||||
expect(predicate(new URL('https://a.examples.org:900/test'))).toBe(true);
|
||||
expect(predicate(new URL('https://a.examples.org:1000/test'))).toBe(true);
|
||||
expect(predicate(new URL('https://a.examples.org:950/test'))).toBe(true);
|
||||
expect(predicate(new URL('https://a.examples.org:1050/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://example.com:150/test'))).toBe(true);
|
||||
expect(predicate(new URL('https://example.com:4000/test'))).toBe(false);
|
||||
expect(predicate(new URL('https://https.org'))).toBe(true);
|
||||
expect(predicate(new URL('http://https.org'))).toBe(false);
|
||||
expect(predicate(new URL('http://http.org'))).toBe(true);
|
||||
expect(predicate(new URL('https://http.org'))).toBe(false);
|
||||
});
|
||||
|
||||
it('factory should throw for malformed uri', async () => {
|
||||
const buildFactory = (hosts: string[]) => {
|
||||
return FetchUrlReader.factory({
|
||||
config: new ConfigReader({
|
||||
backend: {
|
||||
reading: {
|
||||
allow: hosts.map(host => ({ host })),
|
||||
},
|
||||
},
|
||||
}),
|
||||
logger: mockServices.logger.mock(),
|
||||
treeResponseFactory: DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
}),
|
||||
});
|
||||
};
|
||||
expect(() =>
|
||||
buildFactory(['example.com:100-200', 'example.com:100-']),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
buildFactory(['example.com:100-200', 'example.com:-']),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
buildFactory(['example.com:100-200', 'example.com:500-']),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
buildFactory(['example.com:100-200', 'example.com:100-50']),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
buildFactory(['example.com:100-200', 'example.com:-330-']),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
buildFactory(['example.com:100-200', 'example.com:-100-300']),
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
buildFactory(['example.com:100-200', 'example.com:nb-300']),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
describe('read', () => {
|
||||
it('should return etag from the response', async () => {
|
||||
const { buffer } = await fetchUrlReader.readUrl(
|
||||
'https://backstage.io/some-resource',
|
||||
);
|
||||
const response = await buffer();
|
||||
expect(response.toString()).toBe('content foo');
|
||||
});
|
||||
|
||||
it('should throw NotFound if server responds with 404', async () => {
|
||||
await expect(
|
||||
fetchUrlReader.readUrl('https://backstage.io/not-exists'),
|
||||
).rejects.toThrow(NotFoundError);
|
||||
});
|
||||
|
||||
it('should throw Error if server responds with 500', async () => {
|
||||
await expect(
|
||||
fetchUrlReader.readUrl('https://backstage.io/error'),
|
||||
).rejects.toThrow(Error);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readUrl', () => {
|
||||
it('should throw NotModified if server responds with 304 from etag', async () => {
|
||||
await expect(
|
||||
fetchUrlReader.readUrl('https://backstage.io/some-resource', {
|
||||
etag: 'foo',
|
||||
}),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should throw NotModified if server responds with 304 from lastModifiedAfter', async () => {
|
||||
await expect(
|
||||
fetchUrlReader.readUrl('https://backstage.io/some-resource', {
|
||||
lastModifiedAfter: new Date('2020-01-01T00:00:00Z'),
|
||||
}),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should send Authorization header if token is provided', async () => {
|
||||
expect.assertions(1);
|
||||
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://backstage.io/requires-authentication',
|
||||
(req, res, ctx) => {
|
||||
expect(req.headers.get('authorization')).toBe('Bearer mytoken');
|
||||
return res(ctx.status(200));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await fetchUrlReader.readUrl(
|
||||
'https://backstage.io/requires-authentication',
|
||||
{
|
||||
token: 'mytoken',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('should return etag from the response', async () => {
|
||||
const response = await fetchUrlReader.readUrl(
|
||||
'https://backstage.io/some-resource',
|
||||
);
|
||||
expect(response.etag).toBe('foo');
|
||||
expect((await response.buffer()).toString()).toEqual('content foo');
|
||||
});
|
||||
|
||||
it('should throw NotFound if server responds with 404', async () => {
|
||||
await expect(
|
||||
fetchUrlReader.readUrl('https://backstage.io/not-exists'),
|
||||
).rejects.toThrow(NotFoundError);
|
||||
});
|
||||
|
||||
it('should throw Error if server responds with 500', async () => {
|
||||
await expect(
|
||||
fetchUrlReader.readUrl('https://backstage.io/error'),
|
||||
).rejects.toThrow(Error);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,179 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { NotFoundError, NotModifiedError } from '@backstage/errors';
|
||||
import fetch, { Response } from 'node-fetch';
|
||||
import {
|
||||
ReaderFactory,
|
||||
ReadTreeResponse,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
} from './types';
|
||||
import path from 'path';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import { parseLastModified } from './util';
|
||||
|
||||
const isInRange = (num: number, [start, end]: [number, number]) => {
|
||||
return num >= start && num <= end;
|
||||
};
|
||||
|
||||
const parsePortRange = (port: string): [number, number] => {
|
||||
const isRange = port.includes('-');
|
||||
if (isRange) {
|
||||
const range = port
|
||||
.split('-')
|
||||
.map(v => parseInt(v, 10))
|
||||
.filter(Boolean) as [number, number];
|
||||
if (range.length !== 2) throw new Error(`Port range is not valid: ${port}`);
|
||||
const [start, end] = range;
|
||||
if (start <= 0 || end <= 0 || start > end)
|
||||
throw new Error(`Port range is not valid: [${start}, ${end}]`);
|
||||
return range;
|
||||
}
|
||||
const parsedPort = parseInt(port, 10);
|
||||
return [parsedPort, parsedPort];
|
||||
};
|
||||
|
||||
const parsePortPredicate = (port: string | undefined) => {
|
||||
if (port) {
|
||||
const range = parsePortRange(port);
|
||||
return (url: URL) => {
|
||||
if (url.port) return isInRange(parseInt(url.port, 10), range);
|
||||
|
||||
if (url.protocol === 'http:') return isInRange(80, range);
|
||||
if (url.protocol === 'https:') return isInRange(443, range);
|
||||
return false;
|
||||
};
|
||||
}
|
||||
return (url: URL) => !url.port;
|
||||
};
|
||||
|
||||
/**
|
||||
* A {@link @backstage/backend-plugin-api#UrlReaderService} that does a plain fetch of the URL.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class FetchUrlReader implements UrlReader {
|
||||
/**
|
||||
* The factory creates a single reader that will be used for reading any URL that's listed
|
||||
* in configuration at `backend.reading.allow`. The allow list contains a list of objects describing
|
||||
* targets to allow, containing the following fields:
|
||||
*
|
||||
* `host`:
|
||||
* Either full hostnames to match, or subdomain wildcard matchers with a leading '*'.
|
||||
* For example 'example.com' and '*.example.com' are valid values, 'prod.*.example.com' is not.
|
||||
*
|
||||
* `paths`:
|
||||
* An optional list of paths which are allowed. If the list is omitted all paths are allowed.
|
||||
*/
|
||||
static factory: ReaderFactory = ({ config }) => {
|
||||
const predicates =
|
||||
config
|
||||
.getOptionalConfigArray('backend.reading.allow')
|
||||
?.map(allowConfig => {
|
||||
const paths = allowConfig.getOptionalStringArray('paths');
|
||||
const checkPath = paths
|
||||
? (url: URL) => {
|
||||
const targetPath = path.posix.normalize(url.pathname);
|
||||
return paths.some(allowedPath =>
|
||||
targetPath.startsWith(allowedPath),
|
||||
);
|
||||
}
|
||||
: (_url: URL) => true;
|
||||
const host = allowConfig.getString('host');
|
||||
const [hostname, port] = host.split(':');
|
||||
|
||||
const checkPort = parsePortPredicate(port);
|
||||
|
||||
if (hostname.startsWith('*.')) {
|
||||
const suffix = hostname.slice(1);
|
||||
return (url: URL) =>
|
||||
url.hostname.endsWith(suffix) && checkPath(url) && checkPort(url);
|
||||
}
|
||||
return (url: URL) =>
|
||||
url.hostname === hostname && checkPath(url) && checkPort(url);
|
||||
}) ?? [];
|
||||
|
||||
const reader = new FetchUrlReader();
|
||||
const predicate = (url: URL) => predicates.some(p => p(url));
|
||||
return [{ reader, predicate }];
|
||||
};
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(url, {
|
||||
headers: {
|
||||
...(options?.etag && { 'If-None-Match': options.etag }),
|
||||
...(options?.lastModifiedAfter && {
|
||||
'If-Modified-Since': options.lastModifiedAfter.toUTCString(),
|
||||
}),
|
||||
...(options?.token && { Authorization: `Bearer ${options.token}` }),
|
||||
},
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can
|
||||
// be removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
signal: options?.signal as any,
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${url}, ${e}`);
|
||||
}
|
||||
|
||||
if (response.status === 304) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
if (response.ok) {
|
||||
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
|
||||
etag: response.headers.get('ETag') ?? undefined,
|
||||
lastModifiedAt: parseLastModified(
|
||||
response.headers.get('Last-Modified'),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
const message = `could not read ${url}, ${response.status} ${response.statusText}`;
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
async readTree(): Promise<ReadTreeResponse> {
|
||||
throw new Error('FetchUrlReader does not implement readTree');
|
||||
}
|
||||
|
||||
async search(): Promise<SearchResponse> {
|
||||
throw new Error('FetchUrlReader does not implement search');
|
||||
}
|
||||
|
||||
toString() {
|
||||
return 'fetch{}';
|
||||
}
|
||||
}
|
||||
@@ -1,456 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
createMockDirectory,
|
||||
mockServices,
|
||||
setupRequestMockHandlers,
|
||||
} from '@backstage/backend-test-utils';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { NotModifiedError, NotFoundError } from '@backstage/errors';
|
||||
import {
|
||||
GerritIntegration,
|
||||
readGerritIntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
import { JsonObject } from '@backstage/types';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import fs from 'fs-extra';
|
||||
import path from 'path';
|
||||
import { UrlReaderPredicateTuple } from './types';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
import {
|
||||
GITILES_BASE_URL_DEPRECATION_MESSSAGE,
|
||||
GerritUrlReader,
|
||||
} from './GerritUrlReader';
|
||||
import getRawBody from 'raw-body';
|
||||
|
||||
const mockDir = createMockDirectory({ mockOsTmpDir: true });
|
||||
const env = process.env;
|
||||
process.env = { ...env, DISABLE_GERRIT_GITILES_REQUIREMENT: '1' };
|
||||
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
});
|
||||
|
||||
const cloneMock = jest.fn(() => Promise.resolve());
|
||||
jest.mock('./git', () => ({
|
||||
Git: {
|
||||
fromAuth: () => ({
|
||||
clone: cloneMock,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
// Gerrit processor without a gitilesBaseUrl configured
|
||||
const gerritProcessor = new GerritUrlReader(
|
||||
new GerritIntegration(
|
||||
readGerritIntegrationConfig(
|
||||
new ConfigReader({
|
||||
host: 'gerrit.com',
|
||||
}),
|
||||
),
|
||||
),
|
||||
{ treeResponseFactory },
|
||||
'/tmp',
|
||||
);
|
||||
|
||||
// Gerrit processor with a gitilesBaseUrl configured.
|
||||
// Use to test readTree with Gitiles archive download.
|
||||
const gerritProcessorWithGitiles = new GerritUrlReader(
|
||||
new GerritIntegration(
|
||||
readGerritIntegrationConfig(
|
||||
new ConfigReader({
|
||||
host: 'gerrit.com',
|
||||
gitilesBaseUrl: 'https://gerrit.com/gitiles',
|
||||
}),
|
||||
),
|
||||
),
|
||||
{ treeResponseFactory },
|
||||
'/tmp',
|
||||
);
|
||||
|
||||
const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
|
||||
return GerritUrlReader.factory({
|
||||
config: new ConfigReader(config),
|
||||
logger: mockServices.logger.mock(),
|
||||
treeResponseFactory,
|
||||
});
|
||||
};
|
||||
|
||||
// TODO(Rugvip): These tests seem to be a direct or indirect cause of the TaskWorker test flakiness
|
||||
// We're not sure why at this point, but while investigating these tests are disabled
|
||||
// eslint-disable-next-line jest/no-disabled-tests
|
||||
describe.skip('GerritUrlReader', () => {
|
||||
const worker = setupServer();
|
||||
setupRequestMockHandlers(worker);
|
||||
|
||||
beforeEach(() => {
|
||||
mockDir.clear();
|
||||
process.env = { ...env, DISABLE_GERRIT_GITILES_REQUIREMENT: '1' };
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
jest.clearAllMocks();
|
||||
process.env = env;
|
||||
});
|
||||
|
||||
describe('reader factory', () => {
|
||||
it('creates a reader.', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
gerrit: [{ host: 'gerrit.com' }],
|
||||
},
|
||||
});
|
||||
expect(readers).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should not create a default entry.', () => {
|
||||
const readers = createReader({
|
||||
integrations: {},
|
||||
});
|
||||
expect(readers).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handle optional gitilesBaseUrl deprecation', () => {
|
||||
it('should throw if gitilesBaseUrl is not set.', () => {
|
||||
process.env = env;
|
||||
expect(() =>
|
||||
createReader({
|
||||
integrations: {
|
||||
gerrit: [{ host: 'gerrit.com' }],
|
||||
},
|
||||
}),
|
||||
).toThrow(GITILES_BASE_URL_DEPRECATION_MESSSAGE);
|
||||
});
|
||||
it('should not throw if gitilesBaseUrl requirement is overridden.', () => {
|
||||
process.env = { ...env, DISABLE_GERRIT_GITILES_REQUIREMENT: '1' };
|
||||
expect(() =>
|
||||
createReader({
|
||||
integrations: {
|
||||
gerrit: [{ host: 'gerrit.com' }],
|
||||
},
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('predicates without Gitiles', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
gerrit: [{ host: 'gerrit.com' }],
|
||||
},
|
||||
});
|
||||
const predicate = readers[0].predicate;
|
||||
|
||||
it('returns true for the configured host', () => {
|
||||
expect(predicate(new URL('https://gerrit.com/path'))).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for a different host.', () => {
|
||||
expect(predicate(new URL('https://github.com/path'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('predicates with gitilesBaseUrl set.', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
gerrit: [
|
||||
{ host: 'gerrit-review.com', gitilesBaseUrl: 'https://gerrit.com' },
|
||||
],
|
||||
},
|
||||
});
|
||||
const predicate = readers[0].predicate;
|
||||
|
||||
it('returns false since gitilesBaseUrl is set to the api host.', () => {
|
||||
expect(predicate(new URL('https://gerrit-review.com/path'))).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for host.', () => {
|
||||
expect(predicate(new URL('https://gerrit.com/path'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readUrl', () => {
|
||||
const responseBuffer = Buffer.from('Apache License');
|
||||
it('should be able to read file contents as buffer', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content',
|
||||
(_, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body(responseBuffer.toString('base64')),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await gerritProcessor.readUrl(
|
||||
'https://gerrit.com/web/project/+/refs/heads/master/LICENSE',
|
||||
);
|
||||
const buffer = await result.buffer();
|
||||
expect(buffer.toString()).toBe(responseBuffer.toString());
|
||||
});
|
||||
|
||||
it('should be able to read file contents as stream', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content',
|
||||
(_, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body(responseBuffer.toString('base64')),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await gerritProcessor.readUrl(
|
||||
'https://gerrit.com/web/project/+/refs/heads/master/LICENSE',
|
||||
);
|
||||
const fromStream = await getRawBody(result.stream!());
|
||||
expect(fromStream.toString()).toBe(responseBuffer.toString());
|
||||
});
|
||||
|
||||
it('should raise NotFoundError on 404.', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content',
|
||||
(_, res, ctx) => {
|
||||
return res(ctx.status(404, 'File not found.'));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
gerritProcessor.readUrl(
|
||||
'https://gerrit.com/web/project/+/refs/heads/master/LICENSE',
|
||||
),
|
||||
).rejects.toThrow(
|
||||
'File https://gerrit.com/web/project/+/refs/heads/master/LICENSE not found.',
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw an error on non 404 errors.', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content',
|
||||
(_, res, ctx) => {
|
||||
return res(ctx.status(500, 'Error!!!'));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
gerritProcessor.readUrl(
|
||||
'https://gerrit.com/web/project/+/refs/heads/master/LICENSE',
|
||||
),
|
||||
).rejects.toThrow(
|
||||
'https://gerrit.com/web/project/+/refs/heads/master/LICENSE' +
|
||||
' could not be read as https://gerrit.com/projects/web%2Fproject' +
|
||||
'/branches/master/files/LICENSE/content, 500 Error!!!',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTree', () => {
|
||||
const branchAPIUrl =
|
||||
'https://gerrit.com/projects/app%2Fweb/branches/master';
|
||||
const branchAPIresponse = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/gerrit/branch-info-response.txt'),
|
||||
);
|
||||
const treeUrl = 'https://gerrit.com/app/web/+/refs/heads/master/';
|
||||
const treeUrlGitiles =
|
||||
'https://gerrit.com/gitiles/app/web/+/refs/heads/master/';
|
||||
const etag = '52432507a70b677b5674b019c9a46b2e9f29d0a1';
|
||||
const mkdocsContent = 'a repo fetched using git clone';
|
||||
const mdContent = 'doc';
|
||||
const repoArchiveBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/gerrit/gerrit-master.tar.gz'),
|
||||
);
|
||||
const repoArchiveDocsBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/gerrit/gerrit-master-docs.tar.gz'),
|
||||
);
|
||||
|
||||
beforeEach(async () => {
|
||||
mockDir.setContent({
|
||||
'repo/mkdocs.yml': mkdocsContent,
|
||||
'repo/docs/first.md': mdContent,
|
||||
});
|
||||
const spy = jest.spyOn(fs, 'mkdtemp');
|
||||
spy.mockImplementation(() => mockDir.path);
|
||||
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gerrit.com/gitiles/app/web/\\+archive/refs/heads/master.tar.gz',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/x-gzip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=web-refs/heads/master.tar.gz',
|
||||
),
|
||||
ctx.body(repoArchiveBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://gerrit.com/gitiles/app/web/\\+archive/refs/heads/master/docs.tar.gz',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/x-gzip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=web-refs/heads/master-docs.tar.gz',
|
||||
),
|
||||
ctx.body(repoArchiveDocsBuffer),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('reads the wanted files correctly using gitiles.', async () => {
|
||||
worker.use(
|
||||
rest.get(branchAPIUrl, (_, res, ctx) => {
|
||||
return res(ctx.status(200), ctx.body(branchAPIresponse));
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await gerritProcessorWithGitiles.readTree(
|
||||
treeUrlGitiles,
|
||||
);
|
||||
|
||||
expect(response.etag).toBe(etag);
|
||||
|
||||
const files = await response.files();
|
||||
expect(files.length).toBe(2);
|
||||
|
||||
const docsYaml = await files[0].content();
|
||||
expect(docsYaml.toString()).toBe('# Test\n');
|
||||
|
||||
const mdFile = await files[1].content();
|
||||
expect(mdFile.toString()).toBe('site_name: Test\n');
|
||||
|
||||
expect(cloneMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reads the wanted files correctly using git clone.', async () => {
|
||||
worker.use(
|
||||
rest.get(branchAPIUrl, (_, res, ctx) => {
|
||||
return res(ctx.status(200), ctx.body(branchAPIresponse));
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await gerritProcessor.readTree(treeUrl);
|
||||
|
||||
expect(response.etag).toBe(etag);
|
||||
|
||||
const files = await response.files();
|
||||
expect(files.length).toBe(2);
|
||||
|
||||
const docsYaml = await files[0].content();
|
||||
expect(docsYaml.toString()).toBe(mkdocsContent);
|
||||
|
||||
const mdFile = await files[1].content();
|
||||
expect(mdFile.toString()).toBe(mdContent);
|
||||
|
||||
expect(cloneMock).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('throws NotModifiedError for matching etags.', async () => {
|
||||
worker.use(
|
||||
rest.get(branchAPIUrl, (_, res, ctx) => {
|
||||
return res(ctx.status(200), ctx.body(branchAPIresponse));
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(gerritProcessor.readTree(treeUrl, { etag })).rejects.toThrow(
|
||||
NotModifiedError,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws NotFoundError if branch info not found.', async () => {
|
||||
worker.use(
|
||||
rest.get(branchAPIUrl, (_, res, ctx) => {
|
||||
return res(ctx.status(404, 'Not found.'));
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(gerritProcessor.readTree(treeUrl)).rejects.toThrow(
|
||||
NotFoundError,
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw on failures while getting branch info.', async () => {
|
||||
worker.use(
|
||||
rest.get(branchAPIUrl, (_, res, ctx) => {
|
||||
return res(ctx.status(500, 'Error'));
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(gerritProcessor.readTree(treeUrl)).rejects.toThrow(Error);
|
||||
});
|
||||
|
||||
it('should returns wanted files with a subpath using gitiles', async () => {
|
||||
worker.use(
|
||||
rest.get(branchAPIUrl, (_, res, ctx) => {
|
||||
return res(ctx.status(200), ctx.body(branchAPIresponse));
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await gerritProcessorWithGitiles.readTree(
|
||||
`${treeUrlGitiles}/docs`,
|
||||
);
|
||||
|
||||
expect(response.etag).toBe(etag);
|
||||
|
||||
const files = await response.files();
|
||||
expect(files.length).toBe(1);
|
||||
|
||||
const mdFile = await files[0].content();
|
||||
expect(mdFile.toString()).toBe('# Test\n');
|
||||
|
||||
expect(cloneMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should returns wanted files with a subpath using git clone', async () => {
|
||||
worker.use(
|
||||
rest.get(branchAPIUrl, (_, res, ctx) => {
|
||||
return res(ctx.status(200), ctx.body(branchAPIresponse));
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await gerritProcessor.readTree(`${treeUrl}/docs`);
|
||||
|
||||
expect(response.etag).toBe(etag);
|
||||
|
||||
const files = await response.files();
|
||||
expect(files.length).toBe(1);
|
||||
|
||||
const mdFile = await files[0].content();
|
||||
expect(mdFile.toString()).toBe(mdContent);
|
||||
|
||||
expect(cloneMock).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,301 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Base64Decode } from 'base64-stream';
|
||||
import concatStream from 'concat-stream';
|
||||
import fs from 'fs-extra';
|
||||
import fetch, { Response } from 'node-fetch';
|
||||
import os from 'os';
|
||||
import { join as joinPath } from 'path';
|
||||
import { Readable, pipeline as pipelineCb } from 'stream';
|
||||
import tar from 'tar';
|
||||
import { promisify } from 'util';
|
||||
import {
|
||||
GerritIntegration,
|
||||
ScmIntegrations,
|
||||
buildGerritGitilesArchiveUrl,
|
||||
getGerritBranchApiUrl,
|
||||
getGerritCloneRepoUrl,
|
||||
getGerritFileContentsApiUrl,
|
||||
getGerritRequestOptions,
|
||||
parseGerritGitilesUrl,
|
||||
parseGerritJsonResponse,
|
||||
} from '@backstage/integration';
|
||||
import { NotFoundError, NotModifiedError } from '@backstage/errors';
|
||||
import {
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseFactory,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
ReaderFactory,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
} from './types';
|
||||
import { Git } from './git';
|
||||
|
||||
const pipeline = promisify(pipelineCb);
|
||||
|
||||
export const GITILES_BASE_URL_DEPRECATION_MESSSAGE = `A gitilesBaseUrl must be provided \
|
||||
for the gerrit integration to work. You can disable this check by setting \
|
||||
DISABLE_GERRIT_GITILES_REQUIREMENT=1 but this will be removed in a future release. If you \
|
||||
are not able to use the gitiles gerrit plugin, please open an issue towards \
|
||||
https://github.com/backstage/backstage`;
|
||||
|
||||
const createTemporaryDirectory = async (workDir: string): Promise<string> =>
|
||||
await fs.mkdtemp(joinPath(workDir, '/gerrit-clone-'));
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files in Gerrit.
|
||||
*
|
||||
* @remarks
|
||||
* To be able to link to Git contents for Gerrit providers in a user friendly
|
||||
* way we are depending on that there is a Gitiles installation somewhere
|
||||
* that we can link to. It is perfectly possible to integrate Gerrit with
|
||||
* Backstage without Gitiles since all API calls goes directly to Gerrit.
|
||||
* However if Gitiles is configured, readTree will use it to fetch
|
||||
* an archive instead of cloning the repository.
|
||||
*
|
||||
* The "host" variable in the config is the Gerrit host. The address where
|
||||
* Gitiles is installed may be on the same host but it could be on a
|
||||
* separate host. For example a Gerrit instance could be hosted on
|
||||
* "gerrit-review.company.com" but the repos could be browsable on a separate
|
||||
* host, e.g. "gerrit.company.com" and the human readable URL would then
|
||||
* not point to the API host.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class GerritUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
if (!integrations.gerrit) {
|
||||
return [];
|
||||
}
|
||||
const workDir =
|
||||
config.getOptionalString('backend.workingDirectory') ?? os.tmpdir();
|
||||
return integrations.gerrit.list().map(integration => {
|
||||
if (
|
||||
integration.config.gitilesBaseUrl === integration.config.baseUrl &&
|
||||
process.env.DISABLE_GERRIT_GITILES_REQUIREMENT === undefined
|
||||
) {
|
||||
throw new Error(GITILES_BASE_URL_DEPRECATION_MESSSAGE);
|
||||
}
|
||||
const reader = new GerritUrlReader(
|
||||
integration,
|
||||
{ treeResponseFactory },
|
||||
workDir,
|
||||
);
|
||||
const predicate = (url: URL) => {
|
||||
const gitilesUrl = new URL(integration.config.gitilesBaseUrl!);
|
||||
// If gitilesUrl is not specified it will default to
|
||||
// "integration.config.host".
|
||||
return url.host === gitilesUrl.host;
|
||||
};
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly integration: GerritIntegration,
|
||||
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
|
||||
private readonly workDir: string,
|
||||
) {}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
const apiUrl = getGerritFileContentsApiUrl(this.integration.config, url);
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(apiUrl, {
|
||||
method: 'GET',
|
||||
...getGerritRequestOptions(this.integration.config),
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can
|
||||
// be removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
signal: options?.signal as any,
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read gerrit file ${url}, ${e}`);
|
||||
}
|
||||
|
||||
if (response.ok) {
|
||||
let responseBody: string;
|
||||
return {
|
||||
buffer: async () => {
|
||||
if (responseBody === undefined) {
|
||||
responseBody = await response.text();
|
||||
}
|
||||
return Buffer.from(responseBody, 'base64');
|
||||
},
|
||||
stream: () => {
|
||||
const readable = Readable.from(response.body);
|
||||
return readable.pipe(new Base64Decode());
|
||||
},
|
||||
};
|
||||
}
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(`File ${url} not found.`);
|
||||
}
|
||||
throw new Error(
|
||||
`${url} could not be read as ${apiUrl}, ${response.status} ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
const apiUrl = getGerritBranchApiUrl(this.integration.config, url);
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(apiUrl, {
|
||||
method: 'GET',
|
||||
...getGerritRequestOptions(this.integration.config),
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read branch state ${url}, ${e}`);
|
||||
}
|
||||
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(`Not found: ${url}`);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`${url} could not be read as ${apiUrl}, ${response.status} ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
const branchInfo = (await parseGerritJsonResponse(response as any)) as {
|
||||
revision: string;
|
||||
};
|
||||
if (options?.etag === branchInfo.revision) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
if (
|
||||
this.integration.config.gitilesBaseUrl !== this.integration.config.baseUrl
|
||||
) {
|
||||
return this.readTreeFromGitiles(url, branchInfo.revision, options);
|
||||
}
|
||||
return this.readTreeFromGitClone(url, branchInfo.revision, options);
|
||||
}
|
||||
|
||||
async search(): Promise<SearchResponse> {
|
||||
throw new Error('GerritReader does not implement search');
|
||||
}
|
||||
|
||||
toString() {
|
||||
const { host, password } = this.integration.config;
|
||||
return `gerrit{host=${host},authed=${Boolean(password)}}`;
|
||||
}
|
||||
|
||||
private async readTreeFromGitClone(
|
||||
url: string,
|
||||
revision: string,
|
||||
options?: ReadTreeOptions,
|
||||
) {
|
||||
const { filePath } = parseGerritGitilesUrl(this.integration.config, url);
|
||||
|
||||
const git = Git.fromAuth({
|
||||
username: this.integration.config.username,
|
||||
password: this.integration.config.password,
|
||||
});
|
||||
const tempDir = await createTemporaryDirectory(this.workDir);
|
||||
const cloneUrl = getGerritCloneRepoUrl(this.integration.config, url);
|
||||
try {
|
||||
// The "fromTarArchive" function will strip the top level directory so
|
||||
// an additional directory level is created when we clone.
|
||||
await git.clone({
|
||||
url: cloneUrl,
|
||||
dir: joinPath(tempDir, 'repo'),
|
||||
ref: revision,
|
||||
depth: 1,
|
||||
});
|
||||
|
||||
const data = await new Promise<Buffer>(async resolve => {
|
||||
await pipeline(
|
||||
tar.create({ cwd: tempDir }, ['']),
|
||||
concatStream(resolve),
|
||||
);
|
||||
});
|
||||
const tarArchive = Readable.from(data);
|
||||
return await this.deps.treeResponseFactory.fromTarArchive({
|
||||
stream: tarArchive,
|
||||
subpath: filePath === '/' ? undefined : filePath,
|
||||
etag: revision,
|
||||
filter: options?.filter,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(`Could not clone ${cloneUrl}: ${error}`);
|
||||
} finally {
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
private async readTreeFromGitiles(
|
||||
url: string,
|
||||
revision: string,
|
||||
options?: ReadTreeOptions,
|
||||
) {
|
||||
const { branch, filePath, project } = parseGerritGitilesUrl(
|
||||
this.integration.config,
|
||||
url,
|
||||
);
|
||||
const archiveUrl = buildGerritGitilesArchiveUrl(
|
||||
this.integration.config,
|
||||
project,
|
||||
branch,
|
||||
filePath,
|
||||
);
|
||||
const archiveResponse = await fetch(archiveUrl, {
|
||||
...getGerritRequestOptions(this.integration.config),
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can
|
||||
// be removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
signal: options?.signal as any,
|
||||
});
|
||||
|
||||
if (archiveResponse.status === 404) {
|
||||
throw new NotFoundError(`Not found: ${archiveUrl}`);
|
||||
}
|
||||
|
||||
if (!archiveResponse.ok) {
|
||||
throw new Error(
|
||||
`${url} could not be read as ${archiveUrl}, ${archiveResponse.status} ${archiveResponse.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
return await this.deps.treeResponseFactory.fromTarArchive({
|
||||
stream: archiveResponse.body as unknown as Readable,
|
||||
etag: revision,
|
||||
filter: options?.filter,
|
||||
stripFirstDirectory: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,333 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
mockServices,
|
||||
setupRequestMockHandlers,
|
||||
} from '@backstage/backend-test-utils';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { GiteaIntegration, readGiteaConfig } from '@backstage/integration';
|
||||
import { JsonObject } from '@backstage/types';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { UrlReaderPredicateTuple } from './types';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
import getRawBody from 'raw-body';
|
||||
import { GiteaUrlReader } from './GiteaUrlReader';
|
||||
import { NotFoundError, NotModifiedError } from '@backstage/errors';
|
||||
import fs from 'fs-extra';
|
||||
import path from 'path';
|
||||
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
});
|
||||
|
||||
jest.mock('./git', () => ({
|
||||
Git: {
|
||||
fromAuth: () => ({
|
||||
clone: jest.fn(() => Promise.resolve({})),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
const giteaProcessor = new GiteaUrlReader(
|
||||
new GiteaIntegration(
|
||||
readGiteaConfig(
|
||||
new ConfigReader({
|
||||
host: 'gitea.com',
|
||||
}),
|
||||
),
|
||||
),
|
||||
{ treeResponseFactory },
|
||||
);
|
||||
|
||||
const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
|
||||
return GiteaUrlReader.factory({
|
||||
config: new ConfigReader(config),
|
||||
logger: mockServices.logger.mock(),
|
||||
treeResponseFactory,
|
||||
});
|
||||
};
|
||||
|
||||
describe('GiteaUrlReader', () => {
|
||||
const worker = setupServer();
|
||||
setupRequestMockHandlers(worker);
|
||||
|
||||
afterAll(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('reader factory', () => {
|
||||
it('creates a reader.', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
gitea: [{ host: 'gitea.com' }],
|
||||
},
|
||||
});
|
||||
expect(readers).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should not create a default entry.', () => {
|
||||
const readers = createReader({
|
||||
integrations: {},
|
||||
});
|
||||
expect(readers).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('predicates', () => {
|
||||
it('returns true for the configured host', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
gitea: [{ host: 'gitea.com' }],
|
||||
},
|
||||
});
|
||||
const predicate = readers[0].predicate;
|
||||
|
||||
expect(predicate(new URL('https://gitea.com/path'))).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for a different host.', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
gitea: [{ host: 'gitea.com' }],
|
||||
},
|
||||
});
|
||||
const predicate = readers[0].predicate;
|
||||
|
||||
expect(predicate(new URL('https://github.com/path'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readUrl', () => {
|
||||
const responseBuffer = Buffer.from('Apache License');
|
||||
const giteaApiResponse = (content: any) => {
|
||||
return JSON.stringify({
|
||||
encoding: 'base64',
|
||||
content: Buffer.from(content).toString('base64'),
|
||||
});
|
||||
};
|
||||
|
||||
it('should be able to read file contents as buffer', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
|
||||
(req, res, ctx) => {
|
||||
// Test utils prefers matching URL directly but it is part of Gitea's API
|
||||
if (req.url.searchParams.get('ref') === 'branch2') {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body(giteaApiResponse(responseBuffer.toString())),
|
||||
);
|
||||
}
|
||||
|
||||
return res(ctx.status(500));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await giteaProcessor.readUrl(
|
||||
'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
|
||||
);
|
||||
const buffer = await result.buffer();
|
||||
expect(buffer.toString()).toBe(responseBuffer.toString());
|
||||
});
|
||||
|
||||
it('should be able to read file contents as stream', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
|
||||
(req, res, ctx) => {
|
||||
if (req.url.searchParams.get('ref') === 'branch2') {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body(giteaApiResponse(responseBuffer.toString())),
|
||||
);
|
||||
}
|
||||
|
||||
return res(ctx.status(500));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await giteaProcessor.readUrl(
|
||||
'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
|
||||
);
|
||||
const fromStream = await getRawBody(result.stream!());
|
||||
expect(fromStream.toString()).toBe(responseBuffer.toString());
|
||||
});
|
||||
|
||||
it('should raise NotFoundError on 404.', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
|
||||
(_, res, ctx) => {
|
||||
return res(ctx.status(404, 'File not found.'));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
giteaProcessor.readUrl(
|
||||
'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
|
||||
),
|
||||
).rejects.toThrow(NotFoundError);
|
||||
});
|
||||
|
||||
it('should throw an error on non 404 errors.', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
|
||||
(_, res, ctx) => {
|
||||
return res(ctx.status(500, 'Error!!!'));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
giteaProcessor.readUrl(
|
||||
'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
|
||||
),
|
||||
).rejects.toThrow(
|
||||
'https://gitea.com/owner/project/src/branch/branch2/LICENSE could not be read as https://gitea.com/api/v1/repos/owner/project/contents/LICENSE?ref=branch2, 500 Error!!!',
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw NotModified if server responds with 304 from etag', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
|
||||
(_, res, ctx) => {
|
||||
return res(ctx.set('ETag', 'foo'), ctx.status(304, 'Error!!!'));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
giteaProcessor.readUrl(
|
||||
'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
|
||||
{
|
||||
etag: 'foo',
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should throw NotModified if server responds with 304 from lastModifiedAfter', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
|
||||
(_, res, ctx) => {
|
||||
return res(
|
||||
ctx.set(
|
||||
'Last-Modified',
|
||||
new Date('2020-01-01T00:00:00Z').toUTCString(),
|
||||
),
|
||||
ctx.status(304, 'Error!!!'),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
giteaProcessor.readUrl(
|
||||
'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
|
||||
{
|
||||
lastModifiedAfter: new Date('2020-01-01T00:00:00Z'),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTree', () => {
|
||||
const commitHash = '3bdd5457286abdf920db4b77bf2fef79a06190c2';
|
||||
|
||||
const repoBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/mock-main.tar.gz'),
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitea.com/api/v1/repos/owner/project/git/commits/branch2',
|
||||
(_, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json({ sha: commitHash }),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('should be able to get archive', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitea.com/api/v1/repos/owner/project/archive/branch2.tar.gz',
|
||||
(_, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/gzip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=backstage-mock.tar.gz',
|
||||
),
|
||||
ctx.body(repoBuffer),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const response = await giteaProcessor.readTree(
|
||||
'https://gitea.com/owner/project/src/branch/branch2',
|
||||
);
|
||||
expect(response.etag).toBe(commitHash);
|
||||
|
||||
const files = await response.files();
|
||||
expect(files.length).toBe(2);
|
||||
});
|
||||
|
||||
it('should return not modified', async () => {
|
||||
await expect(
|
||||
giteaProcessor.readTree(
|
||||
'https://gitea.com/owner/project/src/branch/branch2',
|
||||
{
|
||||
etag: commitHash,
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should return not found', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitea.com/api/v1/repos/owner/project/git/commits/branch3',
|
||||
(_, res, ctx) => {
|
||||
return res(ctx.status(404));
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
giteaProcessor.readTree(
|
||||
'https://gitea.com/owner/project/src/branch/branch3',
|
||||
),
|
||||
).rejects.toThrow(NotFoundError);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,187 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import {
|
||||
getGiteaFileContentsUrl,
|
||||
getGiteaArchiveUrl,
|
||||
getGiteaLatestCommitUrl,
|
||||
parseGiteaUrl,
|
||||
getGiteaRequestOptions,
|
||||
GiteaIntegration,
|
||||
ScmIntegrations,
|
||||
} from '@backstage/integration';
|
||||
import {
|
||||
ReaderFactory,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseFactory,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
} from './types';
|
||||
import fetch, { Response } from 'node-fetch';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import {
|
||||
AuthenticationError,
|
||||
NotFoundError,
|
||||
NotModifiedError,
|
||||
} from '@backstage/errors';
|
||||
import { Readable } from 'stream';
|
||||
import { parseLastModified } from './util';
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for the Gitea v1 api.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class GiteaUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
return ScmIntegrations.fromConfig(config)
|
||||
.gitea.list()
|
||||
.map(integration => {
|
||||
const reader = new GiteaUrlReader(integration, { treeResponseFactory });
|
||||
const predicate = (url: URL) => {
|
||||
return url.host === integration.config.host;
|
||||
};
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly integration: GiteaIntegration,
|
||||
private readonly deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
) {}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
let response: Response;
|
||||
const blobUrl = getGiteaFileContentsUrl(this.integration.config, url);
|
||||
|
||||
try {
|
||||
response = await fetch(blobUrl, {
|
||||
method: 'GET',
|
||||
...getGiteaRequestOptions(this.integration.config),
|
||||
signal: options?.signal as any,
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${blobUrl}, ${e}`);
|
||||
}
|
||||
|
||||
if (response.ok) {
|
||||
// Gitea returns an object with the file contents encoded, not the file itself
|
||||
const { encoding, content } = await response.json();
|
||||
|
||||
if (encoding === 'base64') {
|
||||
return ReadUrlResponseFactory.fromReadable(
|
||||
Readable.from(Buffer.from(content, 'base64')),
|
||||
{
|
||||
etag: response.headers.get('ETag') ?? undefined,
|
||||
lastModifiedAt: parseLastModified(
|
||||
response.headers.get('Last-Modified'),
|
||||
),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(`Unknown encoding: ${encoding}`);
|
||||
}
|
||||
|
||||
const message = `${url} could not be read as ${blobUrl}, ${response.status} ${response.statusText}`;
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
|
||||
if (response.status === 304) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
if (response.status === 403) {
|
||||
throw new AuthenticationError();
|
||||
}
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
const lastCommitHash = await this.getLastCommitHash(url);
|
||||
if (options?.etag && options.etag === lastCommitHash) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
const archiveUri = getGiteaArchiveUrl(this.integration.config, url);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(archiveUri, {
|
||||
method: 'GET',
|
||||
...getGiteaRequestOptions(this.integration.config),
|
||||
signal: options?.signal as any,
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${archiveUri}, ${e}`);
|
||||
}
|
||||
|
||||
const parsedUri = parseGiteaUrl(this.integration.config, url);
|
||||
|
||||
return this.deps.treeResponseFactory.fromTarArchive({
|
||||
stream: Readable.from(response.body),
|
||||
subpath: parsedUri.path,
|
||||
etag: lastCommitHash,
|
||||
filter: options?.filter,
|
||||
});
|
||||
}
|
||||
|
||||
search(): Promise<SearchResponse> {
|
||||
throw new Error('GiteaUrlReader search not implemented.');
|
||||
}
|
||||
|
||||
toString() {
|
||||
const { host } = this.integration.config;
|
||||
return `gitea{host=${host},authed=${Boolean(
|
||||
this.integration.config.password,
|
||||
)}}`;
|
||||
}
|
||||
|
||||
private async getLastCommitHash(url: string): Promise<string> {
|
||||
const commitUri = getGiteaLatestCommitUrl(this.integration.config, url);
|
||||
|
||||
const response = await fetch(
|
||||
commitUri,
|
||||
getGiteaRequestOptions(this.integration.config),
|
||||
);
|
||||
if (!response.ok) {
|
||||
const message = `Failed to retrieve latest commit information from ${commitUri}, ${response.status} ${response.statusText}`;
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
return (await response.json()).sha;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,364 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
getGithubFileFetchUrl,
|
||||
DefaultGithubCredentialsProvider,
|
||||
GithubCredentialsProvider,
|
||||
GithubIntegration,
|
||||
ScmIntegrations,
|
||||
GithubCredentials,
|
||||
} from '@backstage/integration';
|
||||
import { RestEndpointMethodTypes } from '@octokit/rest';
|
||||
import fetch, { RequestInit, Response } from 'node-fetch';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { Minimatch } from 'minimatch';
|
||||
import { Readable } from 'stream';
|
||||
import { NotFoundError, NotModifiedError } from '@backstage/errors';
|
||||
import {
|
||||
ReadTreeResponseFactory,
|
||||
ReaderFactory,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
SearchOptions,
|
||||
SearchResponse,
|
||||
SearchResponseFile,
|
||||
UrlReader,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
} from './types';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import { parseLastModified } from './util';
|
||||
|
||||
export type GhRepoResponse =
|
||||
RestEndpointMethodTypes['repos']['get']['response']['data'];
|
||||
export type GhCombinedCommitStatusResponse =
|
||||
RestEndpointMethodTypes['repos']['getCombinedStatusForRef']['response']['data'];
|
||||
export type GhTreeResponse =
|
||||
RestEndpointMethodTypes['git']['getTree']['response']['data'];
|
||||
export type GhBlobResponse =
|
||||
RestEndpointMethodTypes['git']['getBlob']['response']['data'];
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files through the GitHub v3 APIs, such as
|
||||
* the one exposed by GitHub itself.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class GithubUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
const credentialsProvider =
|
||||
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
|
||||
return integrations.github.list().map(integration => {
|
||||
const reader = new GithubUrlReader(integration, {
|
||||
treeResponseFactory,
|
||||
credentialsProvider,
|
||||
});
|
||||
const predicate = (url: URL) => url.host === integration.config.host;
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly integration: GithubIntegration,
|
||||
private readonly deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
credentialsProvider: GithubCredentialsProvider;
|
||||
},
|
||||
) {
|
||||
if (!integration.config.apiBaseUrl && !integration.config.rawBaseUrl) {
|
||||
throw new Error(
|
||||
`GitHub integration '${integration.title}' must configure an explicit apiBaseUrl or rawBaseUrl`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
private getCredentials = async (
|
||||
url: string,
|
||||
options?: { token?: string },
|
||||
): Promise<GithubCredentials> => {
|
||||
if (options?.token) {
|
||||
return {
|
||||
headers: {
|
||||
Authorization: `Bearer ${options.token}`,
|
||||
},
|
||||
type: 'token',
|
||||
token: options.token,
|
||||
};
|
||||
}
|
||||
|
||||
return await this.deps.credentialsProvider.getCredentials({
|
||||
url,
|
||||
});
|
||||
};
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
const credentials = await this.getCredentials(url, options);
|
||||
|
||||
const ghUrl = getGithubFileFetchUrl(
|
||||
url,
|
||||
this.integration.config,
|
||||
credentials,
|
||||
);
|
||||
|
||||
const response = await this.fetchResponse(ghUrl, {
|
||||
headers: {
|
||||
...credentials?.headers,
|
||||
...(options?.etag && { 'If-None-Match': options.etag }),
|
||||
...(options?.lastModifiedAfter && {
|
||||
'If-Modified-Since': options.lastModifiedAfter.toUTCString(),
|
||||
}),
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
},
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can
|
||||
// be removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
signal: options?.signal as any,
|
||||
});
|
||||
|
||||
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
|
||||
etag: response.headers.get('ETag') ?? undefined,
|
||||
lastModifiedAt: parseLastModified(response.headers.get('Last-Modified')),
|
||||
});
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
const repoDetails = await this.getRepoDetails(url);
|
||||
const commitSha = repoDetails.commitSha;
|
||||
|
||||
if (options?.etag && options.etag === commitSha) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
const { filepath } = parseGitUrl(url);
|
||||
const { headers } = await this.getCredentials(url, options);
|
||||
|
||||
return this.doReadTree(
|
||||
repoDetails.repo.archive_url,
|
||||
commitSha,
|
||||
filepath,
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can be
|
||||
// removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
{ headers, signal: options?.signal as any },
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
|
||||
const repoDetails = await this.getRepoDetails(url);
|
||||
const commitSha = repoDetails.commitSha;
|
||||
|
||||
if (options?.etag && options.etag === commitSha) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
const { filepath } = parseGitUrl(url);
|
||||
const { headers } = await this.getCredentials(url, options);
|
||||
|
||||
const files = await this.doSearch(
|
||||
url,
|
||||
repoDetails.repo.trees_url,
|
||||
repoDetails.repo.archive_url,
|
||||
commitSha,
|
||||
filepath,
|
||||
{ headers, signal: options?.signal as any },
|
||||
);
|
||||
|
||||
return { files, etag: commitSha };
|
||||
}
|
||||
|
||||
toString() {
|
||||
const { host, token } = this.integration.config;
|
||||
return `github{host=${host},authed=${Boolean(token)}}`;
|
||||
}
|
||||
|
||||
private async doReadTree(
|
||||
archiveUrl: string,
|
||||
sha: string,
|
||||
subpath: string,
|
||||
init: RequestInit,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
// archive_url looks like "https://api.github.com/repos/owner/repo/{archive_format}{/ref}"
|
||||
const archive = await this.fetchResponse(
|
||||
archiveUrl
|
||||
.replace('{archive_format}', 'tarball')
|
||||
.replace('{/ref}', `/${sha}`),
|
||||
init,
|
||||
);
|
||||
|
||||
return await this.deps.treeResponseFactory.fromTarArchive({
|
||||
// TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want
|
||||
// to stick to using that in exclusively backend code.
|
||||
stream: Readable.from(archive.body),
|
||||
subpath,
|
||||
etag: sha,
|
||||
filter: options?.filter,
|
||||
});
|
||||
}
|
||||
|
||||
private async doSearch(
|
||||
url: string,
|
||||
treesUrl: string,
|
||||
archiveUrl: string,
|
||||
sha: string,
|
||||
query: string,
|
||||
init: RequestInit,
|
||||
): Promise<SearchResponseFile[]> {
|
||||
function pathToUrl(path: string): string {
|
||||
// TODO(freben): Use the integration package facility for this instead
|
||||
// pathname starts as /backstage/backstage/blob/master/<path>
|
||||
const updated = new URL(url);
|
||||
const base = updated.pathname.split('/').slice(1, 5).join('/');
|
||||
updated.pathname = `${base}/${path}`;
|
||||
return updated.toString();
|
||||
}
|
||||
|
||||
const matcher = new Minimatch(query.replace(/^\/+/, ''));
|
||||
|
||||
// trees_url looks like "https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}"
|
||||
const recursiveTree: GhTreeResponse = await this.fetchJson(
|
||||
treesUrl.replace('{/sha}', `/${sha}?recursive=true`),
|
||||
init,
|
||||
);
|
||||
|
||||
// The simple case is that we got the entire tree in a single operation.
|
||||
if (!recursiveTree.truncated) {
|
||||
const matching = recursiveTree.tree.filter(
|
||||
item =>
|
||||
item.type === 'blob' &&
|
||||
item.path &&
|
||||
item.url &&
|
||||
matcher.match(item.path),
|
||||
);
|
||||
|
||||
return matching.map(item => ({
|
||||
url: pathToUrl(item.path!),
|
||||
content: async () => {
|
||||
const blob: GhBlobResponse = await this.fetchJson(item.url!, init);
|
||||
return Buffer.from(blob.content, 'base64');
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
// For larger repos, we leverage readTree and filter through that instead
|
||||
const tree = await this.doReadTree(archiveUrl, sha, '', init, {
|
||||
filter: path => matcher.match(path),
|
||||
});
|
||||
const files = await tree.files();
|
||||
|
||||
return files.map(file => ({
|
||||
url: pathToUrl(file.path),
|
||||
content: file.content,
|
||||
lastModifiedAt: file.lastModifiedAt,
|
||||
}));
|
||||
}
|
||||
|
||||
private async getRepoDetails(url: string): Promise<{
|
||||
commitSha: string;
|
||||
repo: {
|
||||
archive_url: string;
|
||||
trees_url: string;
|
||||
};
|
||||
}> {
|
||||
const parsed = parseGitUrl(url);
|
||||
const { ref, full_name } = parsed;
|
||||
|
||||
const credentials = await this.deps.credentialsProvider.getCredentials({
|
||||
url,
|
||||
});
|
||||
const { headers } = credentials;
|
||||
|
||||
const commitStatus: GhCombinedCommitStatusResponse = await this.fetchJson(
|
||||
`${this.integration.config.apiBaseUrl}/repos/${full_name}/commits/${
|
||||
ref || (await this.getDefaultBranch(full_name, credentials))
|
||||
}/status?per_page=0`,
|
||||
{ headers },
|
||||
);
|
||||
|
||||
return {
|
||||
commitSha: commitStatus.sha,
|
||||
repo: commitStatus.repository,
|
||||
};
|
||||
}
|
||||
|
||||
private async getDefaultBranch(
|
||||
repoFullName: string,
|
||||
credentials: GithubCredentials,
|
||||
): Promise<string> {
|
||||
const repo: GhRepoResponse = await this.fetchJson(
|
||||
`${this.integration.config.apiBaseUrl}/repos/${repoFullName}`,
|
||||
{ headers: credentials.headers },
|
||||
);
|
||||
|
||||
return repo.default_branch;
|
||||
}
|
||||
|
||||
private async fetchResponse(
|
||||
url: string | URL,
|
||||
init: RequestInit,
|
||||
): Promise<Response> {
|
||||
const urlAsString = url.toString();
|
||||
const response = await fetch(urlAsString, init);
|
||||
|
||||
if (!response.ok) {
|
||||
let message = `Request failed for ${urlAsString}, ${response.status} ${response.statusText}`;
|
||||
|
||||
if (response.status === 304) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
|
||||
// GitHub returns a 403 response with a couple of headers indicating rate
|
||||
// limit status. See more in the GitHub docs:
|
||||
// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
|
||||
if (this.integration.parseRateLimitInfo(response).isRateLimited) {
|
||||
message += ' (rate limit exceeded)';
|
||||
}
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
private async fetchJson(url: string | URL, init: RequestInit): Promise<any> {
|
||||
const response = await this.fetchResponse(url, init);
|
||||
return await response.json();
|
||||
}
|
||||
}
|
||||
@@ -1,766 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import {
|
||||
createMockDirectory,
|
||||
mockServices,
|
||||
setupRequestMockHandlers,
|
||||
} from '@backstage/backend-test-utils';
|
||||
import fs from 'fs-extra';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import path from 'path';
|
||||
import { GitlabUrlReader } from './GitlabUrlReader';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
import { NotModifiedError, NotFoundError } from '@backstage/errors';
|
||||
import {
|
||||
GitLabIntegration,
|
||||
readGitLabIntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
|
||||
const logger = mockServices.logger.mock();
|
||||
|
||||
const mockDir = createMockDirectory({ mockOsTmpDir: true });
|
||||
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
});
|
||||
|
||||
const gitlabProcessor = new GitlabUrlReader(
|
||||
new GitLabIntegration(
|
||||
readGitLabIntegrationConfig(
|
||||
new ConfigReader({
|
||||
host: 'gitlab.com',
|
||||
apiBaseUrl: 'https://gitlab.com/api/v4',
|
||||
baseUrl: 'https://gitlab.com',
|
||||
token: 'gl-dummy-token',
|
||||
}),
|
||||
),
|
||||
),
|
||||
{ treeResponseFactory },
|
||||
);
|
||||
|
||||
const hostedGitlabProcessor = new GitlabUrlReader(
|
||||
new GitLabIntegration(
|
||||
readGitLabIntegrationConfig(
|
||||
new ConfigReader({
|
||||
host: 'gitlab.mycompany.com',
|
||||
apiBaseUrl: 'https://gitlab.mycompany.com/api/v4',
|
||||
baseUrl: 'https://gitlab.mycompany.com',
|
||||
token: 'gl-dummy-token',
|
||||
}),
|
||||
),
|
||||
),
|
||||
{ treeResponseFactory },
|
||||
);
|
||||
|
||||
describe('GitlabUrlReader', () => {
|
||||
beforeEach(mockDir.clear);
|
||||
|
||||
const worker = setupServer();
|
||||
setupRequestMockHandlers(worker);
|
||||
|
||||
describe('read', () => {
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json({ id: 12345 })),
|
||||
),
|
||||
rest.get('*', (req, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
url: req.url.toString(),
|
||||
headers: req.headers.all(),
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
const createConfig = (token?: string) =>
|
||||
new ConfigReader(
|
||||
{
|
||||
integrations: { gitlab: [{ host: 'gitlab.com', token }] },
|
||||
},
|
||||
'test-config',
|
||||
);
|
||||
|
||||
it.each([
|
||||
// Scoped routes
|
||||
{
|
||||
url: 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
|
||||
config: createConfig(),
|
||||
response: expect.objectContaining({
|
||||
url: 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
|
||||
headers: expect.objectContaining({
|
||||
'private-token': '',
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
url: 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
|
||||
config: createConfig('0123456789'),
|
||||
response: expect.objectContaining({
|
||||
url: 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
|
||||
headers: expect.objectContaining({
|
||||
'private-token': '0123456789',
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
url: 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup
|
||||
config: createConfig(),
|
||||
response: expect.objectContaining({
|
||||
url: 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
|
||||
}),
|
||||
},
|
||||
|
||||
// Unscoped route
|
||||
{
|
||||
url: 'https://gitlab.example.com/a/b/blob/master/c.yaml',
|
||||
config: createConfig(),
|
||||
response: expect.objectContaining({
|
||||
url: 'https://gitlab.example.com/api/v4/projects/12345/repository/files/c.yaml/raw?ref=master',
|
||||
}),
|
||||
},
|
||||
])('should handle happy path %#', async ({ url, config, response }) => {
|
||||
const [{ reader }] = GitlabUrlReader.factory({
|
||||
config,
|
||||
logger,
|
||||
treeResponseFactory,
|
||||
});
|
||||
|
||||
const { buffer } = await reader.readUrl(url);
|
||||
const fromStream = await buffer();
|
||||
const res = await JSON.parse(fromStream.toString());
|
||||
expect(res).toEqual(response);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
url: '',
|
||||
config: createConfig(''),
|
||||
error:
|
||||
"Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string",
|
||||
},
|
||||
])('should handle error path %#', async ({ url, config, error }) => {
|
||||
await expect(async () => {
|
||||
const [{ reader }] = GitlabUrlReader.factory({
|
||||
config,
|
||||
logger,
|
||||
treeResponseFactory,
|
||||
});
|
||||
await reader.readUrl(url);
|
||||
}).rejects.toThrow(error);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readUrl', () => {
|
||||
const [{ reader }] = GitlabUrlReader.factory({
|
||||
config: new ConfigReader({}),
|
||||
logger,
|
||||
treeResponseFactory,
|
||||
});
|
||||
|
||||
it('should throw NotModified on HTTP 304 from etag', async () => {
|
||||
worker.use(
|
||||
rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json({ id: 12345 })),
|
||||
),
|
||||
rest.get('*', (req, res, ctx) => {
|
||||
expect(req.headers.get('If-None-Match')).toBe('999');
|
||||
return res(ctx.status(304));
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
reader.readUrl!(
|
||||
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
|
||||
{
|
||||
etag: '999',
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should throw NotModified on HTTP 304 from lastModifiedAt', async () => {
|
||||
worker.use(
|
||||
rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json({ id: 12345 })),
|
||||
),
|
||||
rest.get('*', (req, res, ctx) => {
|
||||
expect(req.headers.get('If-Modified-Since')).toBe(
|
||||
new Date('2019 12 31 23:59:59 GMT').toUTCString(),
|
||||
);
|
||||
return res(ctx.status(304));
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
reader.readUrl!(
|
||||
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
|
||||
{
|
||||
lastModifiedAfter: new Date('2019 12 31 23:59:59 GMT'),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should return etag and last-modified in response', async () => {
|
||||
worker.use(
|
||||
rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json({ id: 12345 })),
|
||||
),
|
||||
rest.get('*', (_req, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('ETag', '999'),
|
||||
ctx.set(
|
||||
'Last-Modified',
|
||||
new Date('2020 01 01 00:0:00 GMT').toUTCString(),
|
||||
),
|
||||
ctx.body('foo'),
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await reader.readUrl!(
|
||||
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
|
||||
);
|
||||
expect(result.etag).toBe('999');
|
||||
expect(result.lastModifiedAt).toEqual(new Date('2020 01 01 00:0:00 GMT'));
|
||||
const content = await result.buffer();
|
||||
expect(content.toString()).toBe('foo');
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTree', () => {
|
||||
const archiveBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/gitlab-archive.tar.gz'),
|
||||
);
|
||||
|
||||
let projectGitlabApiResponse: any;
|
||||
let commitsGitlabApiResponse: any;
|
||||
let specificPathCommitsGitlabApiResponse: any;
|
||||
|
||||
beforeEach(() => {
|
||||
projectGitlabApiResponse = {
|
||||
id: 11111111,
|
||||
default_branch: 'main',
|
||||
};
|
||||
|
||||
commitsGitlabApiResponse = [
|
||||
{
|
||||
id: 'sha123abc',
|
||||
},
|
||||
];
|
||||
|
||||
specificPathCommitsGitlabApiResponse = [
|
||||
{
|
||||
id: 'sha456def',
|
||||
},
|
||||
];
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename="mock-main-sha123abc.zip"',
|
||||
),
|
||||
ctx.body(archiveBuffer),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://gitlab.com/api/v4/projects/backstage%2Fmock',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json(projectGitlabApiResponse),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/commits',
|
||||
(req, res, ctx) => {
|
||||
const refName = req.url.searchParams.get('ref_name');
|
||||
if (refName === 'main') {
|
||||
const filepath = req.url.searchParams.get('path');
|
||||
if (filepath === 'testFilepath') {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json(specificPathCommitsGitlabApiResponse),
|
||||
);
|
||||
}
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json(commitsGitlabApiResponse),
|
||||
);
|
||||
}
|
||||
if (refName === 'branchDoesNotExist') {
|
||||
return res(ctx.status(404));
|
||||
}
|
||||
return res();
|
||||
},
|
||||
),
|
||||
rest.get(
|
||||
'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json(projectGitlabApiResponse),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/commits',
|
||||
(req, res, ctx) => {
|
||||
const refName = req.url.searchParams.get('ref_name');
|
||||
if (refName === 'main') {
|
||||
const filepath = req.url.searchParams.get('path');
|
||||
if (filepath === 'testFilepath') {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json(specificPathCommitsGitlabApiResponse),
|
||||
);
|
||||
}
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json(commitsGitlabApiResponse),
|
||||
);
|
||||
}
|
||||
return res();
|
||||
},
|
||||
),
|
||||
rest.get(
|
||||
'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/archive',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename="mock-main-sha123abc.zip"',
|
||||
),
|
||||
ctx.body(archiveBuffer),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns the wanted files from an archive', async () => {
|
||||
const response = await gitlabProcessor.readTree(
|
||||
'https://gitlab.com/backstage/mock/tree/main',
|
||||
);
|
||||
|
||||
const files = await response.files();
|
||||
expect(files.length).toBe(2);
|
||||
|
||||
const indexMarkdownFile = await files[0].content();
|
||||
const mkDocsFile = await files[1].content();
|
||||
|
||||
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('creates a directory with the wanted files', async () => {
|
||||
const response = await gitlabProcessor.readTree(
|
||||
'https://gitlab.com/backstage/mock',
|
||||
);
|
||||
|
||||
const dir = await response.dir({ targetDir: mockDir.path });
|
||||
|
||||
await expect(
|
||||
fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'),
|
||||
).resolves.toBe('site_name: Test\n');
|
||||
await expect(
|
||||
fs.readFile(path.join(dir, 'docs', 'index.md'), 'utf8'),
|
||||
).resolves.toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('returns the wanted files from hosted gitlab', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitlab.mycompany.com/backstage/mock/-/archive/main.tar.gz',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename="mock-main-sha123abc.zip"',
|
||||
),
|
||||
ctx.body(archiveBuffer),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const response = await hostedGitlabProcessor.readTree(
|
||||
'https://gitlab.mycompany.com/backstage/mock/tree/main/docs',
|
||||
);
|
||||
|
||||
const files = await response.files();
|
||||
|
||||
expect(files.length).toBe(1);
|
||||
const indexMarkdownFile = await files[0].content();
|
||||
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('returns the wanted files from an archive with a subpath', async () => {
|
||||
const response = await gitlabProcessor.readTree(
|
||||
'https://gitlab.com/backstage/mock/tree/main/docs',
|
||||
);
|
||||
|
||||
const files = await response.files();
|
||||
|
||||
expect(files.length).toBe(1);
|
||||
const indexMarkdownFile = await files[0].content();
|
||||
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('creates a directory with the wanted files with subpath', async () => {
|
||||
const response = await gitlabProcessor.readTree(
|
||||
'https://gitlab.com/backstage/mock/tree/main/docs',
|
||||
);
|
||||
|
||||
const dir = await response.dir({ targetDir: mockDir.path });
|
||||
|
||||
await expect(
|
||||
fs.readFile(path.join(dir, 'index.md'), 'utf8'),
|
||||
).resolves.toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('throws a NotModifiedError when given a etag in options matching last commit', async () => {
|
||||
const fnGitlab = async () => {
|
||||
await gitlabProcessor.readTree('https://gitlab.com/backstage/mock', {
|
||||
etag: 'sha123abc',
|
||||
});
|
||||
};
|
||||
|
||||
const fnHostedGitlab = async () => {
|
||||
await hostedGitlabProcessor.readTree(
|
||||
'https://gitlab.mycompany.com/backstage/mock',
|
||||
{
|
||||
etag: 'sha123abc',
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
await expect(fnGitlab).rejects.toThrow(NotModifiedError);
|
||||
await expect(fnHostedGitlab).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('throws a NotModifiedError when given a etag in options matching last commit affecting specified filepath', async () => {
|
||||
const fnGitlab = async () => {
|
||||
await gitlabProcessor.readTree(
|
||||
'https://gitlab.com/backstage/mock/blob/main/testFilepath',
|
||||
{
|
||||
etag: 'sha456def',
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
const fnHostedGitlab = async () => {
|
||||
await hostedGitlabProcessor.readTree(
|
||||
'https://gitlab.mycompany.com/backstage/mock/blob/main/testFilepath',
|
||||
{
|
||||
etag: 'sha456def',
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
await expect(fnGitlab).rejects.toThrow(NotModifiedError);
|
||||
await expect(fnHostedGitlab).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should not throw error when given an outdated etag in options', async () => {
|
||||
const response = await gitlabProcessor.readTree(
|
||||
'https://gitlab.com/backstage/mock/tree/main',
|
||||
{
|
||||
etag: 'outdatedsha123abc',
|
||||
},
|
||||
);
|
||||
expect((await response.files()).length).toBe(2);
|
||||
});
|
||||
|
||||
it('should detect the default branch', async () => {
|
||||
const response = await gitlabProcessor.readTree(
|
||||
'https://gitlab.com/backstage/mock',
|
||||
);
|
||||
expect((await response.files()).length).toBe(2);
|
||||
});
|
||||
|
||||
it('should throw error on missing branch', async () => {
|
||||
const fnGitlab = async () => {
|
||||
await gitlabProcessor.readTree(
|
||||
'https://gitlab.com/backstage/mock/tree/branchDoesNotExist',
|
||||
);
|
||||
};
|
||||
await expect(fnGitlab).rejects.toThrow(NotFoundError);
|
||||
});
|
||||
|
||||
it('should gracefully handle no matching commits', async () => {
|
||||
commitsGitlabApiResponse = [];
|
||||
|
||||
const response = await gitlabProcessor.readTree(
|
||||
'https://gitlab.com/backstage/mock/tree/main',
|
||||
);
|
||||
|
||||
const files = await response.files();
|
||||
expect(files.length).toBe(2);
|
||||
|
||||
const indexMarkdownFile = await files[0].content();
|
||||
const mkDocsFile = await files[1].content();
|
||||
|
||||
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
});
|
||||
|
||||
describe('search', () => {
|
||||
const archiveBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/gitlab-archive.tar.gz'),
|
||||
);
|
||||
|
||||
const archiveSubPathBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/gitlab-subpath-archive.tar.gz'),
|
||||
);
|
||||
|
||||
const projectGitlabApiResponse = {
|
||||
id: 11111111,
|
||||
default_branch: 'main',
|
||||
};
|
||||
|
||||
const commitsGitlabApiResponse = [
|
||||
{
|
||||
id: 'sha123abc',
|
||||
},
|
||||
];
|
||||
const commitsOfSubPathGitlabApiResponse = [
|
||||
{
|
||||
id: 'sha456abc',
|
||||
},
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive',
|
||||
(req, res, ctx) => {
|
||||
const filepath = req.url.searchParams.get('path');
|
||||
let filename = 'mock-main-sha123abc.zip';
|
||||
let body = archiveBuffer;
|
||||
if (filepath === 'docs') {
|
||||
filename = 'gitlab-subpath-archive.tar.gz';
|
||||
body = archiveSubPathBuffer;
|
||||
}
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/zip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
`attachment; filename="${filename}"`,
|
||||
),
|
||||
ctx.body(body),
|
||||
);
|
||||
},
|
||||
),
|
||||
rest.get(
|
||||
'https://gitlab.com/api/v4/projects/backstage%2Fmock',
|
||||
(_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json(projectGitlabApiResponse),
|
||||
),
|
||||
),
|
||||
rest.get(
|
||||
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/commits',
|
||||
(req, res, ctx) => {
|
||||
const refName = req.url.searchParams.get('ref_name');
|
||||
if (refName === 'main') {
|
||||
const filepath = req.url.searchParams.get('path');
|
||||
if (filepath === 'docs') {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json(commitsOfSubPathGitlabApiResponse),
|
||||
);
|
||||
}
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json(commitsGitlabApiResponse),
|
||||
);
|
||||
}
|
||||
return res();
|
||||
},
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('works for the naive case', async () => {
|
||||
const result = await gitlabProcessor.search(
|
||||
'https://gitlab.com/backstage/mock/tree/main/**/index.*',
|
||||
);
|
||||
expect(result.etag).toBe('sha123abc');
|
||||
expect(result.files.length).toBe(1);
|
||||
expect(result.files[0].url).toBe(
|
||||
'https://gitlab.com/backstage/mock/tree/main/docs/index.md',
|
||||
);
|
||||
await expect(result.files[0].content()).resolves.toEqual(
|
||||
Buffer.from('# Test\n'),
|
||||
);
|
||||
});
|
||||
|
||||
it('load only relevant path', async () => {
|
||||
const result = await gitlabProcessor.search(
|
||||
'https://gitlab.com/backstage/mock/tree/main/docs/**/index.*',
|
||||
);
|
||||
|
||||
expect(result.etag).toBe('sha456abc');
|
||||
expect(result.files.length).toBe(1);
|
||||
expect(result.files[0].url).toBe(
|
||||
'https://gitlab.com/backstage/mock/tree/main/docs/index.md',
|
||||
);
|
||||
await expect(result.files[0].content()).resolves.toEqual(
|
||||
Buffer.from('# Test Subpath\n'),
|
||||
);
|
||||
});
|
||||
|
||||
it('throws NotModifiedError when same etag', async () => {
|
||||
await expect(
|
||||
gitlabProcessor.search(
|
||||
'https://gitlab.com/backstage/mock/tree/main/**/index.*',
|
||||
{ etag: 'sha123abc' },
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getGitlabFetchUrl', () => {
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'*/api/v4/projects/group%2Fsubgroup%2Fproject',
|
||||
(_, res, ctx) => res(ctx.status(200), ctx.json({ id: 12345 })),
|
||||
),
|
||||
);
|
||||
});
|
||||
it('should fall back to getGitLabFileFetchUrl for blob urls', async () => {
|
||||
await expect(
|
||||
(gitlabProcessor as any).getGitlabFetchUrl(
|
||||
'https://gitlab.com/group/subgroup/project/-/blob/branch/my/path/to/file.yaml',
|
||||
),
|
||||
).resolves.toEqual(
|
||||
'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
|
||||
);
|
||||
});
|
||||
it('should work for job artifact urls', async () => {
|
||||
await expect(
|
||||
(gitlabProcessor as any).getGitlabFetchUrl(
|
||||
'https://gitlab.com/group/subgroup/project/-/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob',
|
||||
),
|
||||
).resolves.toEqual(
|
||||
'https://gitlab.com/api/v4/projects/12345/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob',
|
||||
);
|
||||
});
|
||||
it('should fail on unfamiliar or non-Gitlab urls', async () => {
|
||||
await expect(
|
||||
(gitlabProcessor as any).getGitlabFetchUrl(
|
||||
'https://gitlab.com/some/random/endpoint',
|
||||
),
|
||||
).rejects.toThrow('Please provide full path to yaml file from GitLab');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getGitlabArtifactFetchUrl', () => {
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'*/api/v4/projects/group%2Fsubgroup%2Fproject',
|
||||
(_, res, ctx) => res(ctx.status(200), ctx.json({ id: 12345 })),
|
||||
),
|
||||
);
|
||||
worker.use(
|
||||
rest.get(
|
||||
'*/api/v4/projects/groupA%2Fsubgroup%2Fproject',
|
||||
(_, res, ctx) => res(ctx.status(404)),
|
||||
),
|
||||
);
|
||||
});
|
||||
it('should reject urls that are not for the job artifacts API', async () => {
|
||||
await expect(
|
||||
(gitlabProcessor as any).getGitlabArtifactFetchUrl(
|
||||
new URL('https://gitlab.com/some/url'),
|
||||
),
|
||||
).rejects.toThrow('Unable to process url as an GitLab artifact');
|
||||
});
|
||||
it('should work for job artifact urls', async () => {
|
||||
await expect(
|
||||
(gitlabProcessor as any).getGitlabArtifactFetchUrl(
|
||||
new URL(
|
||||
'https://gitlab.com/group/subgroup/project/-/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob',
|
||||
),
|
||||
),
|
||||
).resolves.toEqual(
|
||||
new URL(
|
||||
'https://gitlab.com/api/v4/projects/12345/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob',
|
||||
),
|
||||
);
|
||||
});
|
||||
it('errors in mapping the project ID should be captured', async () => {
|
||||
await expect(
|
||||
(gitlabProcessor as any).getGitlabArtifactFetchUrl(
|
||||
new URL(
|
||||
'https://gitlab.com/groupA/subgroup/project/-/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob',
|
||||
),
|
||||
),
|
||||
).rejects.toThrow(/^Unable to translate GitLab artifact URL:/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveProjectToId', () => {
|
||||
it('should resolve the project path to a valid project id', async () => {
|
||||
worker.use(
|
||||
rest.get('*/api/v4/projects/some%2Fproject', (req, res, ctx) => {
|
||||
// the private-token header must be included on API calls
|
||||
expect(req.headers.get('private-token')).toBe('gl-dummy-token');
|
||||
return res(ctx.status(200), ctx.json({ id: 12345 }));
|
||||
}),
|
||||
);
|
||||
await expect(
|
||||
(gitlabProcessor as any).resolveProjectToId(
|
||||
new URL('https://gitlab.com/some/project'),
|
||||
),
|
||||
).resolves.toEqual(12345);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,356 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
getGitLabFileFetchUrl,
|
||||
getGitLabIntegrationRelativePath,
|
||||
getGitLabRequestOptions,
|
||||
GitLabIntegration,
|
||||
ScmIntegrations,
|
||||
} from '@backstage/integration';
|
||||
import fetch, { Response } from 'node-fetch';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { Minimatch } from 'minimatch';
|
||||
import { Readable } from 'stream';
|
||||
import { NotFoundError, NotModifiedError } from '@backstage/errors';
|
||||
import {
|
||||
ReadTreeResponseFactory,
|
||||
ReaderFactory,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
SearchOptions,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
ReadUrlResponse,
|
||||
ReadUrlOptions,
|
||||
} from './types';
|
||||
import { trimEnd, trimStart } from 'lodash';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import { parseLastModified } from './util';
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files on GitLab.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class GitlabUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
return integrations.gitlab.list().map(integration => {
|
||||
const reader = new GitlabUrlReader(integration, {
|
||||
treeResponseFactory,
|
||||
});
|
||||
const predicate = (url: URL) => url.host === integration.config.host;
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly integration: GitLabIntegration,
|
||||
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
|
||||
) {}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
const { etag, lastModifiedAfter, signal } = options ?? {};
|
||||
const builtUrl = await this.getGitlabFetchUrl(url);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(builtUrl, {
|
||||
headers: {
|
||||
...getGitLabRequestOptions(this.integration.config).headers,
|
||||
...(etag && { 'If-None-Match': etag }),
|
||||
...(lastModifiedAfter && {
|
||||
'If-Modified-Since': lastModifiedAfter.toUTCString(),
|
||||
}),
|
||||
},
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can be
|
||||
// removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
...(signal && { signal: signal as any }),
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${url}, ${e}`);
|
||||
}
|
||||
|
||||
if (response.status === 304) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
if (response.ok) {
|
||||
return ReadUrlResponseFactory.fromNodeJSReadable(response.body, {
|
||||
etag: response.headers.get('ETag') ?? undefined,
|
||||
lastModifiedAt: parseLastModified(
|
||||
response.headers.get('Last-Modified'),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`;
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
const { etag, signal } = options ?? {};
|
||||
const { ref, full_name, filepath } = parseGitUrl(url);
|
||||
|
||||
let repoFullName = full_name;
|
||||
|
||||
const relativePath = getGitLabIntegrationRelativePath(
|
||||
this.integration.config,
|
||||
);
|
||||
|
||||
// Considering self hosted gitlab with relative
|
||||
// assuming '/gitlab' is the relative path
|
||||
// from: /gitlab/repo/project
|
||||
// to: repo/project
|
||||
if (relativePath) {
|
||||
const rectifiedRelativePath = `${trimStart(relativePath, '/')}/`;
|
||||
repoFullName = full_name.replace(rectifiedRelativePath, '');
|
||||
}
|
||||
|
||||
// Use GitLab API to get the default branch
|
||||
// encodeURIComponent is required for GitLab API
|
||||
// https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding
|
||||
const projectGitlabResponse = await fetch(
|
||||
new URL(
|
||||
`${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent(
|
||||
repoFullName,
|
||||
)}`,
|
||||
).toString(),
|
||||
getGitLabRequestOptions(this.integration.config),
|
||||
);
|
||||
if (!projectGitlabResponse.ok) {
|
||||
const msg = `Failed to read tree from ${url}, ${projectGitlabResponse.status} ${projectGitlabResponse.statusText}`;
|
||||
if (projectGitlabResponse.status === 404) {
|
||||
throw new NotFoundError(msg);
|
||||
}
|
||||
throw new Error(msg);
|
||||
}
|
||||
const projectGitlabResponseJson = await projectGitlabResponse.json();
|
||||
|
||||
// ref is an empty string if no branch is set in provided url to readTree.
|
||||
const branch = ref || projectGitlabResponseJson.default_branch;
|
||||
|
||||
// Fetch the latest commit that modifies the filepath in the provided or default branch
|
||||
// to compare against the provided sha.
|
||||
const commitsReqParams = new URLSearchParams();
|
||||
commitsReqParams.set('ref_name', branch);
|
||||
if (!!filepath) {
|
||||
commitsReqParams.set('path', filepath);
|
||||
}
|
||||
const commitsGitlabResponse = await fetch(
|
||||
new URL(
|
||||
`${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent(
|
||||
repoFullName,
|
||||
)}/repository/commits?${commitsReqParams.toString()}`,
|
||||
).toString(),
|
||||
{
|
||||
...getGitLabRequestOptions(this.integration.config),
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can
|
||||
// be removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
...(signal && { signal: signal as any }),
|
||||
},
|
||||
);
|
||||
if (!commitsGitlabResponse.ok) {
|
||||
const message = `Failed to read tree (branch) from ${url}, ${commitsGitlabResponse.status} ${commitsGitlabResponse.statusText}`;
|
||||
if (commitsGitlabResponse.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
const commitSha = (await commitsGitlabResponse.json())[0]?.id ?? '';
|
||||
if (etag && etag === commitSha) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
const archiveReqParams = new URLSearchParams();
|
||||
archiveReqParams.set('sha', branch);
|
||||
if (!!filepath) {
|
||||
archiveReqParams.set('path', filepath);
|
||||
}
|
||||
// https://docs.gitlab.com/ee/api/repositories.html#get-file-archive
|
||||
const archiveGitLabResponse = await fetch(
|
||||
`${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent(
|
||||
repoFullName,
|
||||
)}/repository/archive?${archiveReqParams.toString()}`,
|
||||
{
|
||||
...getGitLabRequestOptions(this.integration.config),
|
||||
// TODO(freben): The signal cast is there because pre-3.x versions of
|
||||
// node-fetch have a very slightly deviating AbortSignal type signature.
|
||||
// The difference does not affect us in practice however. The cast can
|
||||
// be removed after we support ESM for CLI dependencies and migrate to
|
||||
// version 3 of node-fetch.
|
||||
// https://github.com/backstage/backstage/issues/8242
|
||||
...(signal && { signal: signal as any }),
|
||||
},
|
||||
);
|
||||
if (!archiveGitLabResponse.ok) {
|
||||
const message = `Failed to read tree (archive) from ${url}, ${archiveGitLabResponse.status} ${archiveGitLabResponse.statusText}`;
|
||||
if (archiveGitLabResponse.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
return await this.deps.treeResponseFactory.fromTarArchive({
|
||||
stream: Readable.from(archiveGitLabResponse.body),
|
||||
subpath: filepath,
|
||||
etag: commitSha,
|
||||
filter: options?.filter,
|
||||
});
|
||||
}
|
||||
|
||||
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
|
||||
const { filepath } = parseGitUrl(url);
|
||||
const staticPart = this.getStaticPart(filepath);
|
||||
const matcher = new Minimatch(filepath);
|
||||
const treeUrl = trimEnd(url.replace(filepath, staticPart), `/`);
|
||||
const pathPrefix = staticPart ? `${staticPart}/` : '';
|
||||
const tree = await this.readTree(treeUrl, {
|
||||
etag: options?.etag,
|
||||
signal: options?.signal,
|
||||
filter: path => matcher.match(`${pathPrefix}${path}`),
|
||||
});
|
||||
|
||||
const files = await tree.files();
|
||||
return {
|
||||
etag: tree.etag,
|
||||
files: files.map(file => ({
|
||||
url: this.integration.resolveUrl({
|
||||
url: `/${pathPrefix}${file.path}`,
|
||||
base: url,
|
||||
}),
|
||||
content: file.content,
|
||||
lastModifiedAt: file.lastModifiedAt,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* This function splits the input globPattern string into segments using the path separator /. It then iterates over
|
||||
* the segments from the end of the array towards the beginning, checking if the concatenated string up to that
|
||||
* segment matches the original globPattern using the minimatch function. If a match is found, it continues iterating.
|
||||
* If no match is found, it returns the concatenated string up to the current segment, which is the static part of the
|
||||
* glob pattern.
|
||||
*
|
||||
* E.g. `catalog/foo/*.yaml` will return `catalog/foo`.
|
||||
*
|
||||
* @param globPattern the glob pattern
|
||||
* @private
|
||||
*/
|
||||
private getStaticPart(globPattern: string) {
|
||||
const segments = globPattern.split('/');
|
||||
let i = segments.length;
|
||||
while (
|
||||
i > 0 &&
|
||||
new Minimatch(segments.slice(0, i).join('/')).match(globPattern)
|
||||
) {
|
||||
i--;
|
||||
}
|
||||
return segments.slice(0, i).join('/');
|
||||
}
|
||||
|
||||
toString() {
|
||||
const { host, token } = this.integration.config;
|
||||
return `gitlab{host=${host},authed=${Boolean(token)}}`;
|
||||
}
|
||||
|
||||
private async getGitlabFetchUrl(target: string): Promise<string> {
|
||||
// If the target is for a job artifact then go down that path
|
||||
const targetUrl = new URL(target);
|
||||
if (targetUrl.pathname.includes('/-/jobs/artifacts/')) {
|
||||
return this.getGitlabArtifactFetchUrl(targetUrl).then(value =>
|
||||
value.toString(),
|
||||
);
|
||||
}
|
||||
// Default to the old behavior of assuming the url is for a file
|
||||
return getGitLabFileFetchUrl(target, this.integration.config);
|
||||
}
|
||||
|
||||
// convert urls of the form:
|
||||
// https://example.com/<namespace>/<project>/-/jobs/artifacts/<ref>/raw/<path_to_file>?job=<job_name>
|
||||
// to urls of the form:
|
||||
// https://example.com/api/v4/projects/:id/jobs/artifacts/:ref_name/raw/*artifact_path?job=<job_name>
|
||||
private async getGitlabArtifactFetchUrl(target: URL): Promise<URL> {
|
||||
if (!target.pathname.includes('/-/jobs/artifacts/')) {
|
||||
throw new Error('Unable to process url as an GitLab artifact');
|
||||
}
|
||||
try {
|
||||
const [namespaceAndProject, ref] =
|
||||
target.pathname.split('/-/jobs/artifacts/');
|
||||
const projectPath = new URL(target);
|
||||
projectPath.pathname = namespaceAndProject;
|
||||
const projectId = await this.resolveProjectToId(projectPath);
|
||||
const relativePath = getGitLabIntegrationRelativePath(
|
||||
this.integration.config,
|
||||
);
|
||||
const newUrl = new URL(target);
|
||||
newUrl.pathname = `${relativePath}/api/v4/projects/${projectId}/jobs/artifacts/${ref}`;
|
||||
return newUrl;
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Unable to translate GitLab artifact URL: ${target}, ${e}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async resolveProjectToId(pathToProject: URL): Promise<number> {
|
||||
let project = pathToProject.pathname;
|
||||
// Check relative path exist and remove it if so
|
||||
const relativePath = getGitLabIntegrationRelativePath(
|
||||
this.integration.config,
|
||||
);
|
||||
if (relativePath) {
|
||||
project = project.replace(relativePath, '');
|
||||
}
|
||||
// Trim an initial / if it exists
|
||||
project = project.replace(/^\//, '');
|
||||
const result = await fetch(
|
||||
`${
|
||||
pathToProject.origin
|
||||
}${relativePath}/api/v4/projects/${encodeURIComponent(project)}`,
|
||||
getGitLabRequestOptions(this.integration.config),
|
||||
);
|
||||
const data = await result.json();
|
||||
if (!result.ok) {
|
||||
throw new Error(`Gitlab error: ${data.error}, ${data.error_description}`);
|
||||
}
|
||||
return Number(data.id);
|
||||
}
|
||||
}
|
||||
@@ -1,175 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { JsonObject } from '@backstage/types';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
|
||||
import { UrlReaderPredicateTuple } from './types';
|
||||
import packageinfo from '../../package.json';
|
||||
import { mockServices } from '@backstage/backend-test-utils';
|
||||
|
||||
const bucketGetFilesMock = jest.fn();
|
||||
jest.mock('@google-cloud/storage', () => {
|
||||
class Bucket {
|
||||
getFiles(query: any) {
|
||||
return bucketGetFilesMock(query);
|
||||
}
|
||||
}
|
||||
class Storage {
|
||||
bucket() {
|
||||
return new Bucket();
|
||||
}
|
||||
}
|
||||
return {
|
||||
__esModule: true,
|
||||
Storage,
|
||||
};
|
||||
});
|
||||
|
||||
describe('GcsUrlReader', () => {
|
||||
const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
|
||||
return GoogleGcsUrlReader.factory({
|
||||
config: new ConfigReader(config),
|
||||
logger: mockServices.logger.mock(),
|
||||
treeResponseFactory: DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
}),
|
||||
});
|
||||
};
|
||||
|
||||
it('does not create a reader without the googleGcs field', () => {
|
||||
const entries = createReader({
|
||||
integrations: {},
|
||||
});
|
||||
expect(entries).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('creates a reader with credentials correctly configured', () => {
|
||||
const entries = createReader({
|
||||
integrations: {
|
||||
googleGcs: {
|
||||
privateKey: '--- BEGIN KEY ---- fakekey --- END KEY ---',
|
||||
clientEmail: 'someone@example.com',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(entries).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('creates a reader with default credentials provider', () => {
|
||||
const entries = createReader({
|
||||
integrations: {
|
||||
googleGcs: {},
|
||||
},
|
||||
});
|
||||
expect(entries).toHaveLength(1);
|
||||
});
|
||||
it('check if userAgent has been called with this key value', async () => {
|
||||
const getStorage: any = {
|
||||
userAgent: `backstage/backend-common.GoogleGcsUrlReader/${packageinfo.version}`,
|
||||
};
|
||||
jest.mock('@google-cloud/storage', () => {
|
||||
return {
|
||||
Storage: jest.fn(() => getStorage),
|
||||
};
|
||||
});
|
||||
const getUserAgent = getStorage.userAgent.toString();
|
||||
expect(getUserAgent).toBe(
|
||||
`backstage/backend-common.GoogleGcsUrlReader/${packageinfo.version}`,
|
||||
);
|
||||
});
|
||||
|
||||
describe('predicates', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
googleGcs: {},
|
||||
},
|
||||
});
|
||||
const predicate = readers[0].predicate;
|
||||
|
||||
it('returns true for the correct google cloud storage host', () => {
|
||||
expect(predicate(new URL('https://storage.cloud.google.com'))).toBe(true);
|
||||
});
|
||||
it('returns true for a url with the full path and the correct host', () => {
|
||||
expect(
|
||||
predicate(
|
||||
new URL(
|
||||
'https://storage.cloud.google.com/team1/service1/catalog-info.yaml',
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
it('returns false for the wrong hostname under cloud.google.com', () => {
|
||||
expect(predicate(new URL('https://storage2.cloud.google.com'))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
it('returns false for a partially correct host', () => {
|
||||
expect(predicate(new URL('https://cloud.google.com'))).toBe(false);
|
||||
});
|
||||
it('returns false for a completely different host', () => {
|
||||
expect(predicate(new URL('https://a.example.com/test'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('search', () => {
|
||||
const { reader } = createReader({ integrations: { googleGcs: {} } })[0];
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('throws if search url does not end with *', async () => {
|
||||
const glob = 'https://storage.cloud.google.com/bucket/no-asterisk';
|
||||
await expect(() => reader.search(glob)).rejects.toThrow(
|
||||
'GcsUrlReader only supports prefix-based searches',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws if search url looks truly glob-y', async () => {
|
||||
const glob = 'https://storage.cloud.google.com/bucket/**/path*';
|
||||
await expect(() => reader.search(glob)).rejects.toThrow(
|
||||
'GcsUrlReader only supports prefix-based searches',
|
||||
);
|
||||
});
|
||||
|
||||
it('searches with expected prefix and pagination', () => {
|
||||
bucketGetFilesMock.mockResolvedValue([[]]);
|
||||
const glob = 'https://storage.cloud.google.com/bucket/path/some-prefix-*';
|
||||
|
||||
reader.search(glob);
|
||||
expect(bucketGetFilesMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
autoPaginate: true,
|
||||
prefix: 'path/some-prefix-',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns valid SearchResponse object', async () => {
|
||||
const expectedFile = { name: 'path/some-prefix-1.txt' };
|
||||
bucketGetFilesMock.mockResolvedValue([[expectedFile]]);
|
||||
const glob = 'https://storage.cloud.google.com/bucket/path/some-prefix-*';
|
||||
|
||||
const result = await reader.search(glob);
|
||||
expect(result.files).toHaveLength(1);
|
||||
expect(result.files[0].url).toEqual(
|
||||
'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt',
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,154 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Storage } from '@google-cloud/storage';
|
||||
import {
|
||||
ReaderFactory,
|
||||
ReadTreeResponse,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
} from './types';
|
||||
import getRawBody from 'raw-body';
|
||||
import {
|
||||
GoogleGcsIntegrationConfig,
|
||||
readGoogleGcsIntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
import { Readable } from 'stream';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import packageinfo from '../../package.json';
|
||||
|
||||
const GOOGLE_GCS_HOST = 'storage.cloud.google.com';
|
||||
|
||||
const parseURL = (
|
||||
url: string,
|
||||
): { host: string; bucket: string; key: string } => {
|
||||
const { host, pathname } = new URL(url);
|
||||
|
||||
if (host !== GOOGLE_GCS_HOST) {
|
||||
throw new Error(`not a valid GCS URL: ${url}`);
|
||||
}
|
||||
|
||||
const [, bucket, ...key] = pathname.split('/');
|
||||
return {
|
||||
host: host,
|
||||
bucket,
|
||||
key: key.join('/'),
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files on Google GCS.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class GoogleGcsUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, logger }) => {
|
||||
if (!config.has('integrations.googleGcs')) {
|
||||
return [];
|
||||
}
|
||||
const gcsConfig = readGoogleGcsIntegrationConfig(
|
||||
config.getConfig('integrations.googleGcs'),
|
||||
);
|
||||
let storage: Storage;
|
||||
if (!gcsConfig.clientEmail || !gcsConfig.privateKey) {
|
||||
logger.info(
|
||||
'googleGcs credentials not found in config. Using default credentials provider.',
|
||||
);
|
||||
storage = new Storage({
|
||||
userAgent: `backstage/backend-common.GoogleGcsUrlReader/${packageinfo.version}`,
|
||||
});
|
||||
} else {
|
||||
storage = new Storage({
|
||||
credentials: {
|
||||
client_email: gcsConfig.clientEmail || undefined,
|
||||
private_key: gcsConfig.privateKey || undefined,
|
||||
},
|
||||
userAgent: `backstage/backend-common.GoogleGcsUrlReader/${packageinfo.version}`,
|
||||
});
|
||||
}
|
||||
const reader = new GoogleGcsUrlReader(gcsConfig, storage);
|
||||
const predicate = (url: URL) => url.host === GOOGLE_GCS_HOST;
|
||||
return [{ reader, predicate }];
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly integration: GoogleGcsIntegrationConfig,
|
||||
private readonly storage: Storage,
|
||||
) {}
|
||||
|
||||
private readStreamFromUrl(url: string): Readable {
|
||||
const { bucket, key } = parseURL(url);
|
||||
return this.storage.bucket(bucket).file(key).createReadStream();
|
||||
}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
try {
|
||||
return await getRawBody(this.readStreamFromUrl(url));
|
||||
} catch (error) {
|
||||
throw new Error(`unable to read gcs file from ${url}, ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
_options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
// TODO etag is not implemented yet.
|
||||
const stream = this.readStreamFromUrl(url);
|
||||
return ReadUrlResponseFactory.fromReadable(stream);
|
||||
}
|
||||
|
||||
async readTree(): Promise<ReadTreeResponse> {
|
||||
throw new Error('GcsUrlReader does not implement readTree');
|
||||
}
|
||||
|
||||
async search(url: string): Promise<SearchResponse> {
|
||||
const { bucket, key: pattern } = parseURL(url);
|
||||
|
||||
if (!pattern.endsWith('*') || pattern.indexOf('*') !== pattern.length - 1) {
|
||||
throw new Error('GcsUrlReader only supports prefix-based searches');
|
||||
}
|
||||
|
||||
const [files] = await this.storage.bucket(bucket).getFiles({
|
||||
autoPaginate: true,
|
||||
prefix: pattern.split('*').join(''),
|
||||
});
|
||||
|
||||
return {
|
||||
files: files.map(file => {
|
||||
const fullUrl = ['https:/', GOOGLE_GCS_HOST, bucket, file.name].join(
|
||||
'/',
|
||||
);
|
||||
return {
|
||||
url: fullUrl,
|
||||
content: async () => {
|
||||
const readResponse = await this.readUrl(fullUrl);
|
||||
return readResponse.buffer();
|
||||
},
|
||||
};
|
||||
}),
|
||||
// TODO etag is not implemented yet.
|
||||
etag: 'NOT/IMPLEMENTED',
|
||||
};
|
||||
}
|
||||
|
||||
toString() {
|
||||
const key = this.integration.privateKey;
|
||||
return `googleGcs{host=${GOOGLE_GCS_HOST},authed=${Boolean(key)}}`;
|
||||
}
|
||||
}
|
||||
@@ -1,260 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
mockServices,
|
||||
setupRequestMockHandlers,
|
||||
} from '@backstage/backend-test-utils';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { HarnessIntegration, readHarnessConfig } from '@backstage/integration';
|
||||
import { JsonObject } from '@backstage/types';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { UrlReaderPredicateTuple } from './types';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
import getRawBody from 'raw-body';
|
||||
import { HarnessUrlReader } from './HarnessUrlReader';
|
||||
import { NotFoundError, NotModifiedError } from '@backstage/errors';
|
||||
import fs from 'fs-extra';
|
||||
import path from 'path';
|
||||
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config: new ConfigReader({}),
|
||||
});
|
||||
|
||||
jest.mock('./git', () => ({
|
||||
Git: {
|
||||
fromAuth: () => ({
|
||||
clone: jest.fn(() => Promise.resolve({})),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
const harnessProcessor = new HarnessUrlReader(
|
||||
new HarnessIntegration(
|
||||
readHarnessConfig(
|
||||
new ConfigReader({
|
||||
host: 'app.harness.io',
|
||||
token: 'p',
|
||||
}),
|
||||
),
|
||||
),
|
||||
{ treeResponseFactory },
|
||||
);
|
||||
|
||||
const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
|
||||
return HarnessUrlReader.factory({
|
||||
config: new ConfigReader(config),
|
||||
logger: mockServices.logger.mock(),
|
||||
treeResponseFactory,
|
||||
});
|
||||
};
|
||||
const responseBuffer = Buffer.from('Apache License');
|
||||
const harnessApiResponse = (content: any) => {
|
||||
return content;
|
||||
};
|
||||
const commitHash = '3bdd5457286abdf920db4b77bf2fef79a06190c2';
|
||||
|
||||
const handlers = [
|
||||
rest.get(
|
||||
'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/:path+/raw/all-apis.yaml',
|
||||
(_req, res, ctx) => {
|
||||
return res(ctx.status(500), ctx.json({ message: 'Error!!!' }));
|
||||
},
|
||||
),
|
||||
rest.get(
|
||||
'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/:path+/raw/404error.yaml',
|
||||
(_req, res, ctx) => {
|
||||
return res(ctx.status(404), ctx.json({ message: 'File not found.' }));
|
||||
},
|
||||
),
|
||||
rest.get(
|
||||
'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/:path+/raw/stream.TXT',
|
||||
(_req, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body(harnessApiResponse(responseBuffer.toString())),
|
||||
);
|
||||
},
|
||||
),
|
||||
|
||||
rest.get(
|
||||
'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/:path+/raw/buffer.TXT',
|
||||
(_req, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.body(harnessApiResponse(responseBuffer.toString())),
|
||||
);
|
||||
},
|
||||
),
|
||||
rest.get(
|
||||
'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName2/projectName/repoName/:path+/content?routingId=accountId&include_commit=true&git_ref=refs/heads/branchName',
|
||||
(_req, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json({ latest_commit: { sha: commitHash } }),
|
||||
);
|
||||
},
|
||||
),
|
||||
rest.get(
|
||||
'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName3/projectName/repoName/:path+/content?routingId=accountId&include_commit=true&git_ref=refs/heads/branchName',
|
||||
(_, res, ctx) => {
|
||||
return res(ctx.status(404));
|
||||
},
|
||||
),
|
||||
];
|
||||
|
||||
describe('HarnessUrlReader', () => {
|
||||
const worker = setupServer(...handlers);
|
||||
setupRequestMockHandlers(worker);
|
||||
beforeAll(() => worker.listen({ onUnhandledRequest: 'bypass' }));
|
||||
afterAll(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('reader factory', () => {
|
||||
it('creates a reader.', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
harness: [{ host: 'app.harness.io' }],
|
||||
},
|
||||
});
|
||||
expect(readers).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should not create a default entry.', () => {
|
||||
const readers = createReader({
|
||||
integrations: {},
|
||||
});
|
||||
expect(readers).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('predicates', () => {
|
||||
it('returns true for the configured host', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
harness: [{ host: 'app.harness.io' }],
|
||||
},
|
||||
});
|
||||
const predicate = readers[0].predicate;
|
||||
|
||||
expect(predicate(new URL('https://app.harness.io/path'))).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for a different host.', () => {
|
||||
const readers = createReader({
|
||||
integrations: {
|
||||
harness: [{ host: 'app.harness.io' }],
|
||||
},
|
||||
});
|
||||
const predicate = readers[0].predicate;
|
||||
|
||||
expect(predicate(new URL('https://github.com/path'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readUrl part 1', () => {
|
||||
it('should be able to read file contents as buffer', async () => {
|
||||
const result = await harnessProcessor.readUrl(
|
||||
'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/buffer.TXT',
|
||||
);
|
||||
const buffer = await result.buffer();
|
||||
expect(buffer.toString()).toBe(responseBuffer.toString());
|
||||
});
|
||||
|
||||
it('should be able to read file contents as stream', async () => {
|
||||
const result = await harnessProcessor.readUrl(
|
||||
'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/stream.TXT',
|
||||
);
|
||||
const fromStream = await getRawBody(result.stream!());
|
||||
expect(fromStream.toString()).toBe(responseBuffer.toString());
|
||||
});
|
||||
|
||||
it('should raise NotFoundError on 404.', async () => {
|
||||
await expect(
|
||||
harnessProcessor.readUrl(
|
||||
'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml',
|
||||
),
|
||||
).rejects.toThrow(
|
||||
'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml x https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/404error.yaml?routingId=accountId&git_ref=refs/heads/refMain, 404 Not Found',
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw an error on non 404 errors.', async () => {
|
||||
await expect(
|
||||
harnessProcessor.readUrl(
|
||||
'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/all-apis.yaml',
|
||||
),
|
||||
).rejects.toThrow(
|
||||
'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/all-apis.yaml x https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/all-apis.yaml?routingId=accountId&git_ref=refs/heads/refMain, 500 Internal Server Error',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTree', () => {
|
||||
const repoBuffer = fs.readFileSync(
|
||||
path.resolve(__dirname, '__fixtures__/mock-main.zip'),
|
||||
);
|
||||
|
||||
it('should be able to get archive', async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName2/projectName/repoName/:path+/archive/branchName.zip',
|
||||
(_, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/gzip'),
|
||||
ctx.set(
|
||||
'content-disposition',
|
||||
'attachment; filename=backstage-mock.zip',
|
||||
),
|
||||
ctx.body(repoBuffer),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const response = await harnessProcessor.readTree(
|
||||
'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName2/projects/projectName/repos/repoName/files/branchName',
|
||||
);
|
||||
expect(response.etag).toBe(commitHash);
|
||||
|
||||
const files = await response.files();
|
||||
expect(files.length).toBe(2);
|
||||
});
|
||||
|
||||
it('should return not modified', async () => {
|
||||
await expect(
|
||||
harnessProcessor.readTree(
|
||||
'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName2/projects/projectName/repos/repoName/files/branchName2',
|
||||
{
|
||||
etag: commitHash,
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(NotModifiedError);
|
||||
});
|
||||
|
||||
it('should return not found', async () => {
|
||||
await expect(
|
||||
harnessProcessor.readTree(
|
||||
'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName3/projects/projectName/repos/repoName/files/branchName3',
|
||||
),
|
||||
).rejects.toThrow(NotFoundError);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,183 +0,0 @@
|
||||
/*
|
||||
* Copyright 2025 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import {
|
||||
getHarnessRequestOptions,
|
||||
getHarnessFileContentsUrl,
|
||||
HarnessIntegration,
|
||||
ScmIntegrations,
|
||||
getHarnessLatestCommitUrl,
|
||||
getHarnessArchiveUrl,
|
||||
parseHarnessUrl,
|
||||
} from '@backstage/integration';
|
||||
import {
|
||||
ReaderFactory,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
SearchResponse,
|
||||
ReadTreeResponseFactory,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
UrlReader,
|
||||
} from './types';
|
||||
import fetch, { Response } from 'node-fetch';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
import {
|
||||
AuthenticationError,
|
||||
NotFoundError,
|
||||
NotModifiedError,
|
||||
} from '@backstage/errors';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
/**
|
||||
* Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for the Harness code v1 api.
|
||||
*
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class HarnessUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
return ScmIntegrations.fromConfig(config)
|
||||
.harness.list()
|
||||
.map(integration => {
|
||||
const reader = new HarnessUrlReader(integration, {
|
||||
treeResponseFactory,
|
||||
});
|
||||
const predicate = (url: URL) => {
|
||||
return url.host === integration.config.host;
|
||||
};
|
||||
return { reader, predicate };
|
||||
});
|
||||
};
|
||||
|
||||
constructor(
|
||||
private readonly integration: HarnessIntegration,
|
||||
private readonly deps: {
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
},
|
||||
) {}
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const response = await this.readUrl(url);
|
||||
return response.buffer();
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
let response: Response;
|
||||
const blobUrl = getHarnessFileContentsUrl(this.integration.config, url);
|
||||
|
||||
try {
|
||||
response = await fetch(blobUrl, {
|
||||
method: 'GET',
|
||||
...getHarnessRequestOptions(this.integration.config),
|
||||
signal: options?.signal as any,
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${blobUrl}, ${e}`);
|
||||
}
|
||||
|
||||
if (response.ok) {
|
||||
// Harness Code returns the raw content object
|
||||
const jsonResponse = { data: response.body };
|
||||
if (jsonResponse) {
|
||||
return ReadUrlResponseFactory.fromReadable(
|
||||
Readable.from(jsonResponse.data),
|
||||
{
|
||||
etag: response.headers.get('ETag') ?? undefined,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(`Unknown json: ${jsonResponse}`);
|
||||
}
|
||||
|
||||
const message = `${url} x ${blobUrl}, ${response.status} ${response.statusText}`;
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
|
||||
if (response.status === 304) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
if (response.status === 403) {
|
||||
throw new AuthenticationError();
|
||||
}
|
||||
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
const lastCommitHash = await this.getLastCommitHash(url);
|
||||
|
||||
if (options?.etag && options.etag === lastCommitHash) {
|
||||
throw new NotModifiedError();
|
||||
}
|
||||
|
||||
const archiveUri = getHarnessArchiveUrl(this.integration.config, url);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(archiveUri, {
|
||||
method: 'GET',
|
||||
...getHarnessRequestOptions(this.integration.config),
|
||||
signal: options?.signal as any,
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${archiveUri}, ${e}`);
|
||||
}
|
||||
|
||||
const parsedUri = parseHarnessUrl(this.integration.config, url);
|
||||
|
||||
return this.deps.treeResponseFactory.fromZipArchive({
|
||||
stream: Readable.from(response.body),
|
||||
subpath: parsedUri.path,
|
||||
etag: lastCommitHash,
|
||||
filter: options?.filter,
|
||||
});
|
||||
}
|
||||
search(): Promise<SearchResponse> {
|
||||
throw new Error('HarnessUrlReader search not implemented.');
|
||||
}
|
||||
|
||||
toString() {
|
||||
const { host } = this.integration.config;
|
||||
return `harness{host=${host},authed=${Boolean(
|
||||
this.integration.config.token || this.integration.config.apiKey,
|
||||
)}}`;
|
||||
}
|
||||
private async getLastCommitHash(url: string): Promise<string> {
|
||||
const commitUri = getHarnessLatestCommitUrl(this.integration.config, url);
|
||||
|
||||
const response = await fetch(
|
||||
commitUri,
|
||||
getHarnessRequestOptions(this.integration.config),
|
||||
);
|
||||
if (!response.ok) {
|
||||
const message = `Failed to retrieve latest commit information from ${commitUri}, ${response.status} ${response.statusText}`;
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
return (await response.json()).latest_commit.sha;
|
||||
}
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConflictError } from '@backstage/errors';
|
||||
import getRawBody from 'raw-body';
|
||||
import { Readable } from 'stream';
|
||||
import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
|
||||
describe('ReadUrlResponseFactory', () => {
|
||||
describe("fromReadable's", () => {
|
||||
const expectedText = 'expected text';
|
||||
let readable: Readable;
|
||||
|
||||
beforeEach(() => {
|
||||
readable = Readable.from(expectedText, {
|
||||
objectMode: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('etag is passed through', async () => {
|
||||
const expectedEtag = 'xyz';
|
||||
const response = await ReadUrlResponseFactory.fromReadable(readable, {
|
||||
etag: expectedEtag,
|
||||
});
|
||||
expect(response.etag).toEqual(expectedEtag);
|
||||
});
|
||||
|
||||
it('buffer returns expected data', async () => {
|
||||
const response = await ReadUrlResponseFactory.fromReadable(readable);
|
||||
const buffer = await response.buffer();
|
||||
expect(buffer.toString()).toEqual(expectedText);
|
||||
});
|
||||
|
||||
it('buffer can be called multiple times', async () => {
|
||||
const response = await ReadUrlResponseFactory.fromReadable(readable);
|
||||
const buffer1 = await response.buffer();
|
||||
const buffer2 = await response.buffer();
|
||||
expect(buffer1.toString()).toEqual(expectedText);
|
||||
expect(buffer2.toString()).toEqual(expectedText);
|
||||
});
|
||||
|
||||
it('buffer cannot be called after stream is called', async () => {
|
||||
const response = await ReadUrlResponseFactory.fromReadable(readable);
|
||||
response.stream!();
|
||||
expect(() => response.buffer()).toThrow(ConflictError);
|
||||
});
|
||||
|
||||
it('stream returns expected data', async () => {
|
||||
const response = await ReadUrlResponseFactory.fromReadable(readable);
|
||||
const stream = response.stream!();
|
||||
const bufferFromStream = await getRawBody(stream);
|
||||
expect(bufferFromStream.toString()).toEqual(expectedText);
|
||||
});
|
||||
|
||||
it('stream cannot be called after buffer is called', async () => {
|
||||
const response = await ReadUrlResponseFactory.fromReadable(readable);
|
||||
response.buffer();
|
||||
expect(() => response.stream!()).toThrow(ConflictError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fromNodeJSReadable's", () => {
|
||||
const expectedText = 'expected text';
|
||||
let readable: NodeJS.ReadableStream;
|
||||
|
||||
beforeEach(() => {
|
||||
readable = Readable.from(Buffer.from(expectedText));
|
||||
});
|
||||
|
||||
it('etag is passed through', async () => {
|
||||
const expectedEtag = 'xyz';
|
||||
const response = await ReadUrlResponseFactory.fromNodeJSReadable(
|
||||
readable,
|
||||
{
|
||||
etag: expectedEtag,
|
||||
},
|
||||
);
|
||||
expect(response.etag).toEqual(expectedEtag);
|
||||
});
|
||||
|
||||
it('buffer returns expected data', async () => {
|
||||
const response = await ReadUrlResponseFactory.fromNodeJSReadable(
|
||||
readable,
|
||||
);
|
||||
const buffer = await response.buffer();
|
||||
expect(buffer.toString()).toEqual(expectedText);
|
||||
});
|
||||
|
||||
it('stream returns expected data', async () => {
|
||||
const response = await ReadUrlResponseFactory.fromNodeJSReadable(
|
||||
readable,
|
||||
);
|
||||
const stream = response.stream!();
|
||||
const bufferFromStream = await getRawBody(stream);
|
||||
expect(bufferFromStream.toString()).toEqual(expectedText);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,78 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConflictError } from '@backstage/errors';
|
||||
import getRawBody from 'raw-body';
|
||||
import { Readable } from 'stream';
|
||||
import {
|
||||
ReadUrlResponse,
|
||||
ReadUrlResponseFactoryFromStreamOptions,
|
||||
} from './types';
|
||||
|
||||
/**
|
||||
* Utility class for UrlReader implementations to create valid ReadUrlResponse
|
||||
* instances from common response primitives.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class ReadUrlResponseFactory {
|
||||
/**
|
||||
* Resolves a ReadUrlResponse from a Readable stream.
|
||||
*/
|
||||
static async fromReadable(
|
||||
stream: Readable,
|
||||
options?: ReadUrlResponseFactoryFromStreamOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
// Reference to eventual buffer enables callers to call buffer() multiple
|
||||
// times without consequence.
|
||||
let buffer: Promise<Buffer>;
|
||||
|
||||
// Prevent "stream is not readable" errors from bubbling up.
|
||||
const conflictError = new ConflictError(
|
||||
'Cannot use buffer() and stream() from the same ReadUrlResponse',
|
||||
);
|
||||
let hasCalledStream = false;
|
||||
let hasCalledBuffer = false;
|
||||
|
||||
return {
|
||||
buffer: () => {
|
||||
hasCalledBuffer = true;
|
||||
if (hasCalledStream) throw conflictError;
|
||||
if (buffer) return buffer;
|
||||
buffer = getRawBody(stream);
|
||||
return buffer;
|
||||
},
|
||||
stream: () => {
|
||||
hasCalledStream = true;
|
||||
if (hasCalledBuffer) throw conflictError;
|
||||
return stream;
|
||||
},
|
||||
etag: options?.etag,
|
||||
lastModifiedAt: options?.lastModifiedAt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a ReadUrlResponse from an old-style NodeJS.ReadableStream.
|
||||
*/
|
||||
static async fromNodeJSReadable(
|
||||
oldStyleStream: NodeJS.ReadableStream,
|
||||
options?: ReadUrlResponseFactoryFromStreamOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
const readable = Readable.from(oldStyleStream);
|
||||
return ReadUrlResponseFactory.fromReadable(readable, options);
|
||||
}
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { UrlReaderPredicateMux } from './UrlReaderPredicateMux';
|
||||
|
||||
describe('UrlReaderPredicateMux', () => {
|
||||
it('forwards methods based on predicate', async () => {
|
||||
const fooReader = {
|
||||
read: jest.fn(),
|
||||
readUrl: jest.fn(),
|
||||
readTree: jest.fn(),
|
||||
search: jest.fn(),
|
||||
};
|
||||
const barReader = {
|
||||
read: jest.fn(),
|
||||
readUrl: jest.fn(),
|
||||
readTree: jest.fn(),
|
||||
search: jest.fn(),
|
||||
};
|
||||
|
||||
const mux = new UrlReaderPredicateMux();
|
||||
mux.register({
|
||||
predicate: url => url.hostname === 'foo',
|
||||
reader: fooReader,
|
||||
});
|
||||
mux.register({
|
||||
predicate: url => url.hostname === 'bar',
|
||||
reader: barReader,
|
||||
});
|
||||
|
||||
await mux.readUrl('http://foo/1');
|
||||
expect(fooReader.readUrl).toHaveBeenCalledWith('http://foo/1', undefined);
|
||||
await mux.readUrl('http://foo/2');
|
||||
expect(fooReader.readUrl).toHaveBeenCalledWith('http://foo/2', undefined);
|
||||
await mux.readTree('http://foo/3');
|
||||
expect(fooReader.readTree).toHaveBeenCalledWith('http://foo/3', undefined);
|
||||
await mux.search('http://foo/4');
|
||||
expect(fooReader.search).toHaveBeenCalledWith('http://foo/4', undefined);
|
||||
|
||||
await mux.readUrl('http://bar/1');
|
||||
expect(barReader.readUrl).toHaveBeenCalledWith('http://bar/1', undefined);
|
||||
await mux.readUrl('http://bar/2');
|
||||
expect(barReader.readUrl).toHaveBeenCalledWith('http://bar/2', undefined);
|
||||
await mux.readTree('http://bar/3');
|
||||
expect(barReader.readTree).toHaveBeenCalledWith('http://bar/3', undefined);
|
||||
await mux.search('http://bar/4');
|
||||
expect(barReader.search).toHaveBeenCalledWith('http://bar/4', undefined);
|
||||
});
|
||||
|
||||
it('throws an error if no predicate matches', async () => {
|
||||
const mux = new UrlReaderPredicateMux();
|
||||
|
||||
await expect(mux.readUrl('http://foo/1')).rejects.toThrow(
|
||||
/^Reading from 'http:\/\/foo\/1' is not allowed. You may/,
|
||||
);
|
||||
|
||||
mux.register({
|
||||
predicate: url => url.hostname === 'foo',
|
||||
reader: {
|
||||
readUrl: jest.fn(),
|
||||
readTree: jest.fn(),
|
||||
search: jest.fn(),
|
||||
},
|
||||
});
|
||||
|
||||
await expect(mux.readUrl('http://foo/1')).resolves.toBeUndefined();
|
||||
|
||||
await expect(mux.readUrl('http://bar/1')).rejects.toThrow(
|
||||
/^Reading from 'http:\/\/bar\/1' is not allowed. You may/,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,93 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { NotAllowedError } from '@backstage/errors';
|
||||
import {
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
SearchOptions,
|
||||
SearchResponse,
|
||||
UrlReader,
|
||||
UrlReaderPredicateTuple,
|
||||
} from './types';
|
||||
|
||||
function notAllowedMessage(url: string) {
|
||||
return (
|
||||
`Reading from '${url}' is not allowed. ` +
|
||||
`You may need to configure an integration for the target host, or add it ` +
|
||||
`to the configured list of allowed hosts at 'backend.reading.allow'`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A UrlReader implementation that selects from a set of UrlReaders
|
||||
* based on a predicate tied to each reader.
|
||||
*/
|
||||
export class UrlReaderPredicateMux implements UrlReader {
|
||||
private readonly readers: UrlReaderPredicateTuple[] = [];
|
||||
|
||||
register(tuple: UrlReaderPredicateTuple): void {
|
||||
this.readers.push(tuple);
|
||||
}
|
||||
|
||||
async readUrl(
|
||||
url: string,
|
||||
options?: ReadUrlOptions,
|
||||
): Promise<ReadUrlResponse> {
|
||||
const parsed = new URL(url);
|
||||
|
||||
for (const { predicate, reader } of this.readers) {
|
||||
if (predicate(parsed)) {
|
||||
return reader.readUrl(url, options);
|
||||
}
|
||||
}
|
||||
|
||||
throw new NotAllowedError(notAllowedMessage(url));
|
||||
}
|
||||
|
||||
async readTree(
|
||||
url: string,
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
const parsed = new URL(url);
|
||||
|
||||
for (const { predicate, reader } of this.readers) {
|
||||
if (predicate(parsed)) {
|
||||
return await reader.readTree(url, options);
|
||||
}
|
||||
}
|
||||
|
||||
throw new NotAllowedError(notAllowedMessage(url));
|
||||
}
|
||||
|
||||
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
|
||||
const parsed = new URL(url);
|
||||
|
||||
for (const { predicate, reader } of this.readers) {
|
||||
if (predicate(parsed)) {
|
||||
return await reader.search(url, options);
|
||||
}
|
||||
}
|
||||
|
||||
throw new NotAllowedError(notAllowedMessage(url));
|
||||
}
|
||||
|
||||
toString() {
|
||||
return `predicateMux{readers=${this.readers.map(t => t.reader).join(',')}`;
|
||||
}
|
||||
}
|
||||
@@ -1,104 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { LoggerService } from '@backstage/backend-plugin-api';
|
||||
import { Config } from '@backstage/config';
|
||||
import { ReaderFactory, UrlReader } from './types';
|
||||
import { UrlReaderPredicateMux } from './UrlReaderPredicateMux';
|
||||
import { AzureUrlReader } from './AzureUrlReader';
|
||||
import { BitbucketCloudUrlReader } from './BitbucketCloudUrlReader';
|
||||
import { BitbucketServerUrlReader } from './BitbucketServerUrlReader';
|
||||
import { BitbucketUrlReader } from './BitbucketUrlReader';
|
||||
import { GerritUrlReader } from './GerritUrlReader';
|
||||
import { GithubUrlReader } from './GithubUrlReader';
|
||||
import { GitlabUrlReader } from './GitlabUrlReader';
|
||||
import { DefaultReadTreeResponseFactory } from './tree';
|
||||
import { FetchUrlReader } from './FetchUrlReader';
|
||||
import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
|
||||
import { AwsS3UrlReader } from './AwsS3UrlReader';
|
||||
import { GiteaUrlReader } from './GiteaUrlReader';
|
||||
import { AwsCodeCommitUrlReader } from './AwsCodeCommitUrlReader';
|
||||
import { HarnessUrlReader } from './HarnessUrlReader';
|
||||
|
||||
/**
|
||||
* Creation options for {@link @backstage/backend-plugin-api#UrlReaderService}.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type UrlReadersOptions = {
|
||||
/** Root config object */
|
||||
config: Config;
|
||||
/** Logger used by all the readers */
|
||||
logger: LoggerService;
|
||||
/** A list of factories used to construct individual readers that match on URLs */
|
||||
factories?: ReaderFactory[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Helps construct {@link @backstage/backend-plugin-api#UrlReaderService}s.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class UrlReaders {
|
||||
/**
|
||||
* Creates a custom {@link @backstage/backend-plugin-api#UrlReaderService} wrapper for your own set of factories.
|
||||
*/
|
||||
static create(options: UrlReadersOptions): UrlReader {
|
||||
const { logger, config, factories } = options;
|
||||
const mux = new UrlReaderPredicateMux();
|
||||
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
|
||||
config,
|
||||
});
|
||||
for (const factory of factories ?? []) {
|
||||
const tuples = factory({ config, logger: logger, treeResponseFactory });
|
||||
|
||||
for (const tuple of tuples) {
|
||||
mux.register(tuple);
|
||||
}
|
||||
}
|
||||
|
||||
return mux;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@link @backstage/backend-plugin-api#UrlReaderService} wrapper that includes all the default factories
|
||||
* from this package.
|
||||
*
|
||||
* Any additional factories passed will be loaded before the default ones.
|
||||
*/
|
||||
static default(options: UrlReadersOptions) {
|
||||
const { logger, config, factories = [] } = options;
|
||||
return UrlReaders.create({
|
||||
logger,
|
||||
config,
|
||||
factories: factories.concat([
|
||||
AzureUrlReader.factory,
|
||||
BitbucketCloudUrlReader.factory,
|
||||
BitbucketServerUrlReader.factory,
|
||||
BitbucketUrlReader.factory,
|
||||
GerritUrlReader.factory,
|
||||
GithubUrlReader.factory,
|
||||
GiteaUrlReader.factory,
|
||||
GitlabUrlReader.factory,
|
||||
GoogleGcsUrlReader.factory,
|
||||
HarnessUrlReader.factory,
|
||||
AwsS3UrlReader.factory,
|
||||
AwsCodeCommitUrlReader.factory,
|
||||
FetchUrlReader.factory,
|
||||
]),
|
||||
});
|
||||
}
|
||||
}
|
||||
-1
@@ -1 +0,0 @@
|
||||
site_name: Test
|
||||
-1
@@ -1 +0,0 @@
|
||||
site_name: Test2
|
||||
@@ -1 +0,0 @@
|
||||
site_name: Test
|
||||
@@ -1 +0,0 @@
|
||||
site_name: Test2
|
||||
@@ -1 +0,0 @@
|
||||
site_name: Test3
|
||||
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -1,2 +0,0 @@
|
||||
)]}'
|
||||
{"web_links":[{"name":"browse","url":"https://gerrit.googlesource.com/app/web/+/refs/heads/master","target":"_blank"}],"ref":"refs/heads/master","revision":"52432507a70b677b5674b019c9a46b2e9f29d0a1"}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,148 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import fs from 'fs-extra';
|
||||
import isomorphicGit, { ProgressCallback, AuthCallback } from 'isomorphic-git';
|
||||
import http from 'isomorphic-git/http/node';
|
||||
import { LoggerService } from '@backstage/backend-plugin-api';
|
||||
|
||||
/**
|
||||
* Configure static credential for authentication
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type StaticAuthOptions = {
|
||||
username?: string;
|
||||
password?: string;
|
||||
token?: string;
|
||||
logger?: LoggerService;
|
||||
};
|
||||
|
||||
/**
|
||||
* Configure an authentication callback that can provide credentials on demand
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type AuthCallbackOptions = {
|
||||
onAuth: AuthCallback;
|
||||
logger?: LoggerService;
|
||||
};
|
||||
|
||||
function isAuthCallbackOptions(
|
||||
options: StaticAuthOptions | AuthCallbackOptions,
|
||||
): options is AuthCallbackOptions {
|
||||
return 'onAuth' in options;
|
||||
}
|
||||
|
||||
/*
|
||||
provider username password
|
||||
Azure 'notempty' token
|
||||
Bitbucket Cloud 'x-token-auth' token
|
||||
Bitbucket Server username password or token
|
||||
GitHub 'x-access-token' token
|
||||
GitLab 'oauth2' token
|
||||
|
||||
From : https://isomorphic-git.org/docs/en/onAuth with fix for GitHub
|
||||
|
||||
Or token provided as `token` for Bearer auth header
|
||||
instead of Basic Auth (e.g., Bitbucket Server).
|
||||
*/
|
||||
/**
|
||||
* A convenience wrapper around the `isomorphic-git` library.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class Git {
|
||||
private readonly headers: {
|
||||
[x: string]: string;
|
||||
};
|
||||
|
||||
private constructor(
|
||||
private readonly config: {
|
||||
onAuth: AuthCallback;
|
||||
token?: string;
|
||||
logger?: LoggerService;
|
||||
},
|
||||
) {
|
||||
this.onAuth = config.onAuth;
|
||||
|
||||
this.headers = {
|
||||
'user-agent': 'git/@isomorphic-git',
|
||||
...(config.token ? { Authorization: `Bearer ${config.token}` } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** https://isomorphic-git.org/docs/en/clone */
|
||||
async clone(options: {
|
||||
url: string;
|
||||
dir: string;
|
||||
ref?: string;
|
||||
depth?: number;
|
||||
noCheckout?: boolean;
|
||||
}): Promise<void> {
|
||||
const { url, dir, ref, depth, noCheckout } = options;
|
||||
this.config.logger?.info(`Cloning repo {dir=${dir},url=${url}}`);
|
||||
|
||||
try {
|
||||
return await isomorphicGit.clone({
|
||||
fs,
|
||||
http,
|
||||
url,
|
||||
dir,
|
||||
ref,
|
||||
singleBranch: true,
|
||||
depth: depth ?? 1,
|
||||
noCheckout,
|
||||
onProgress: this.onProgressHandler(),
|
||||
headers: this.headers,
|
||||
onAuth: this.onAuth,
|
||||
});
|
||||
} catch (ex) {
|
||||
this.config.logger?.error(`Failed to clone repo {dir=${dir},url=${url}}`);
|
||||
if (ex.data) {
|
||||
throw new Error(`${ex.message} {data=${JSON.stringify(ex.data)}}`);
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
private onAuth: AuthCallback;
|
||||
|
||||
private onProgressHandler = (): ProgressCallback => {
|
||||
let currentPhase = '';
|
||||
|
||||
return event => {
|
||||
if (currentPhase !== event.phase) {
|
||||
currentPhase = event.phase;
|
||||
this.config.logger?.info(event.phase);
|
||||
}
|
||||
const total = event.total
|
||||
? `${Math.round((event.loaded / event.total) * 100)}%`
|
||||
: event.loaded;
|
||||
this.config.logger?.debug(`status={${event.phase},total={${total}}}`);
|
||||
};
|
||||
};
|
||||
|
||||
static fromAuth = (options: StaticAuthOptions | AuthCallbackOptions) => {
|
||||
if (isAuthCallbackOptions(options)) {
|
||||
const { onAuth, logger } = options;
|
||||
return new Git({ onAuth, logger });
|
||||
}
|
||||
|
||||
const { username, password, token, logger } = options;
|
||||
return new Git({ onAuth: () => ({ username, password }), token, logger });
|
||||
};
|
||||
}
|
||||
@@ -14,35 +14,207 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { AzureUrlReader } from './AzureUrlReader';
|
||||
export { BitbucketCloudUrlReader } from './BitbucketCloudUrlReader';
|
||||
export { BitbucketUrlReader } from './BitbucketUrlReader';
|
||||
export { BitbucketServerUrlReader } from './BitbucketServerUrlReader';
|
||||
export { GerritUrlReader } from './GerritUrlReader';
|
||||
export { GithubUrlReader } from './GithubUrlReader';
|
||||
export { GitlabUrlReader } from './GitlabUrlReader';
|
||||
export { GiteaUrlReader } from './GiteaUrlReader';
|
||||
export { HarnessUrlReader } from './HarnessUrlReader';
|
||||
export { AwsS3UrlReader } from './AwsS3UrlReader';
|
||||
export { FetchUrlReader } from './FetchUrlReader';
|
||||
export { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
|
||||
export type {
|
||||
FromReadableArrayOptions,
|
||||
ReaderFactory,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseFactory,
|
||||
ReadTreeResponseFile,
|
||||
ReadTreeResponseDirOptions,
|
||||
ReadTreeResponseFactoryOptions,
|
||||
ReadUrlOptions,
|
||||
ReadUrlResponse,
|
||||
ReadUrlResponseFactoryFromStreamOptions,
|
||||
SearchOptions,
|
||||
SearchResponse,
|
||||
SearchResponseFile,
|
||||
UrlReader,
|
||||
UrlReaderPredicateTuple,
|
||||
} from './types';
|
||||
export { UrlReaders } from './UrlReaders';
|
||||
export type { UrlReadersOptions } from './UrlReaders';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { AzureUrlReader as _AzureUrlReader } from '../../../backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { BitbucketCloudUrlReader as _BitbucketCloudUrlReader } from '../../../backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { BitbucketUrlReader as _BitbucketUrlReader } from '../../../backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { BitbucketServerUrlReader as _BitbucketServerUrlReader } from '../../../backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { GerritUrlReader as _GerritUrlReader } from '../../../backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { GithubUrlReader as _GithubUrlReader } from '../../../backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { GitlabUrlReader as _GitlabUrlReader } from '../../../backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { GiteaUrlReader as _GiteaUrlReader } from '../../../backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { HarnessUrlReader as _HarnessUrlReader } from '../../../backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { AwsS3UrlReader as _AwsS3UrlReader } from '../../../backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { FetchUrlReader as _FetchUrlReader } from '../../../backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { UrlReaders as _UrlReaders } from '../../../backend-defaults/src/entrypoints/urlReader/lib/UrlReaders';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import { ReadUrlResponseFactory as _ReadUrlResponseFactory } from '../../../backend-defaults/src/entrypoints/urlReader/lib/ReadUrlResponseFactory';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import type { UrlReadersOptions as _UrlReadersOptions } from '../../../backend-defaults/src/entrypoints/urlReader/lib/UrlReaders';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import type { FromReadableArrayOptions as _FromReadableArrayOptions } from '../../../backend-defaults/src/entrypoints/urlReader/lib/types';
|
||||
// eslint-disable-next-line @backstage/no-relative-monorepo-imports
|
||||
import type {
|
||||
ReaderFactory as _ReaderFactory,
|
||||
ReadTreeResponseFactory as _ReadTreeResponseFactory,
|
||||
ReadTreeResponseFactoryOptions as _ReadTreeResponseFactoryOptions,
|
||||
ReadUrlResponseFactoryFromStreamOptions as _ReadUrlResponseFactoryFromStreamOptions,
|
||||
UrlReaderPredicateTuple as _UrlReaderPredicateTuple,
|
||||
} from '../../../backend-defaults/src/entrypoints/urlReader/lib/types';
|
||||
|
||||
import type {
|
||||
ReadTreeOptions as _ReadTreeOptions,
|
||||
ReadTreeResponse as _ReadTreeResponse,
|
||||
ReadTreeResponseFile as _ReadTreeResponseFile,
|
||||
ReadTreeResponseDirOptions as _ReadTreeResponseDirOptions,
|
||||
ReadUrlOptions as _ReadUrlOptions,
|
||||
ReadUrlResponse as _ReadUrlResponse,
|
||||
SearchOptions as _SearchOptions,
|
||||
SearchResponse as _SearchResponse,
|
||||
SearchResponseFile as _SearchResponseFile,
|
||||
UrlReaderService as _UrlReaderService,
|
||||
} from '@backstage/backend-plugin-api';
|
||||
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const AzureUrlReader = _AzureUrlReader;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const BitbucketCloudUrlReader = _BitbucketCloudUrlReader;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const BitbucketUrlReader = _BitbucketUrlReader;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const BitbucketServerUrlReader = _BitbucketServerUrlReader;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const GerritUrlReader = _GerritUrlReader;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const GithubUrlReader = _GithubUrlReader;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const GitlabUrlReader = _GitlabUrlReader;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const GiteaUrlReader = _GiteaUrlReader;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const HarnessUrlReader = _HarnessUrlReader;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const AwsS3UrlReader = _AwsS3UrlReader;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const FetchUrlReader = _FetchUrlReader;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const UrlReaders = _UrlReaders;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export const ReadUrlResponseFactory = _ReadUrlResponseFactory;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export type UrlReadersOptions = _UrlReadersOptions;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export type FromReadableArrayOptions = _FromReadableArrayOptions;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export type ReaderFactory = _ReaderFactory;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export type ReadTreeResponseFactory = _ReadTreeResponseFactory;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export type ReadTreeResponseFactoryOptions = _ReadTreeResponseFactoryOptions;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export type ReadUrlResponseFactoryFromStreamOptions =
|
||||
_ReadUrlResponseFactoryFromStreamOptions;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Import from `@backstage/backend-defaults/urlReader` instead
|
||||
*/
|
||||
export type UrlReaderPredicateTuple = _UrlReaderPredicateTuple;
|
||||
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Use `ÙrlReaderReadTreeOptions` from `@backstage/backend-plugin-api` instead
|
||||
*/
|
||||
export type ReadTreeOptions = _ReadTreeOptions;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Use `ÙrlReaderReadTreeResponse` from `@backstage/backend-plugin-api` instead
|
||||
*/
|
||||
export type ReadTreeResponse = _ReadTreeResponse;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Use `ÙrlReaderReadTreeResponseFile` from `@backstage/backend-plugin-api` instead
|
||||
*/
|
||||
export type ReadTreeResponseFile = _ReadTreeResponseFile;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Use `ÙrlReaderReadTreeResponseDirOptions` from `@backstage/backend-plugin-api` instead
|
||||
*/
|
||||
export type ReadTreeResponseDirOptions = _ReadTreeResponseDirOptions;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Use `ÙrlReaderReadUrlOptions` from `@backstage/backend-plugin-api` instead
|
||||
*/
|
||||
export type ReadUrlOptions = _ReadUrlOptions;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Use `ÙrlReaderReadUrlResponse` from `@backstage/backend-plugin-api` instead
|
||||
*/
|
||||
export type ReadUrlResponse = _ReadUrlResponse;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Use `ÙrlReaderSearchOptions` from `@backstage/backend-plugin-api` instead
|
||||
*/
|
||||
export type SearchOptions = _SearchOptions;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Use `ÙrlReaderSearchResponse` from `@backstage/backend-plugin-api` instead
|
||||
*/
|
||||
export type SearchResponse = _SearchResponse;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Use `ÙrlReaderSearchResponseFile` from `@backstage/backend-plugin-api` instead
|
||||
*/
|
||||
export type SearchResponseFile = _SearchResponseFile;
|
||||
/**
|
||||
* @public
|
||||
* @deprecated Use `ÙrlReaderService` from `@backstage/backend-plugin-api` instead
|
||||
*/
|
||||
export type UrlReader = _UrlReaderService;
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import os from 'os';
|
||||
import { Config } from '@backstage/config';
|
||||
import {
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseFactoryOptions,
|
||||
ReadTreeResponseFactory,
|
||||
FromReadableArrayOptions,
|
||||
} from '../types';
|
||||
import { TarArchiveResponse } from './TarArchiveResponse';
|
||||
import { ZipArchiveResponse } from './ZipArchiveResponse';
|
||||
import { ReadableArrayResponse } from './ReadableArrayResponse';
|
||||
|
||||
export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory {
|
||||
static create(options: { config: Config }): DefaultReadTreeResponseFactory {
|
||||
return new DefaultReadTreeResponseFactory(
|
||||
options.config.getOptionalString('backend.workingDirectory') ??
|
||||
os.tmpdir(),
|
||||
);
|
||||
}
|
||||
|
||||
constructor(private readonly workDir: string) {}
|
||||
|
||||
async fromTarArchive(
|
||||
options: ReadTreeResponseFactoryOptions & {
|
||||
stripFirstDirectory?: boolean;
|
||||
},
|
||||
): Promise<ReadTreeResponse> {
|
||||
return new TarArchiveResponse(
|
||||
options.stream,
|
||||
options.subpath ?? '',
|
||||
this.workDir,
|
||||
options.etag,
|
||||
options.filter,
|
||||
options.stripFirstDirectory ?? true,
|
||||
);
|
||||
}
|
||||
|
||||
async fromZipArchive(
|
||||
options: ReadTreeResponseFactoryOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
return new ZipArchiveResponse(
|
||||
options.stream,
|
||||
options.subpath ?? '',
|
||||
this.workDir,
|
||||
options.etag,
|
||||
options.filter,
|
||||
);
|
||||
}
|
||||
|
||||
async fromReadableArray(
|
||||
options: FromReadableArrayOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
return new ReadableArrayResponse(options, this.workDir, '');
|
||||
}
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import fs from 'fs-extra';
|
||||
import path from 'path';
|
||||
import { FromReadableArrayOptions } from '../types';
|
||||
import { ReadableArrayResponse } from './ReadableArrayResponse';
|
||||
import { createMockDirectory } from '@backstage/backend-test-utils';
|
||||
import { relative } from 'path/posix';
|
||||
|
||||
const name1 = 'file1.yaml';
|
||||
const file1 = fs.readFileSync(
|
||||
path.resolve(__filename, '../../__fixtures__/awsS3/awsS3-mock-object.yaml'),
|
||||
);
|
||||
|
||||
const name2 = 'file2.yaml';
|
||||
const file2 = fs.readFileSync(
|
||||
path.resolve(__filename, '../../__fixtures__/awsS3/awsS3-mock-object2.yaml'),
|
||||
);
|
||||
|
||||
const dir1 = 'dir1';
|
||||
const name3 = `file3.yaml`;
|
||||
const file3 = fs.readFileSync(
|
||||
path.resolve(__filename, '../../__fixtures__/awsS3/awsS3-mock-object3.yaml'),
|
||||
);
|
||||
|
||||
describe('ReadableArrayResponse', () => {
|
||||
const sourceDir = createMockDirectory();
|
||||
const targetDir = createMockDirectory();
|
||||
|
||||
beforeEach(() => {
|
||||
sourceDir.setContent({
|
||||
[name1]: file1,
|
||||
[name2]: file2,
|
||||
[dir1]: {
|
||||
[name3]: file3,
|
||||
},
|
||||
});
|
||||
targetDir.clear();
|
||||
});
|
||||
|
||||
const openStreams = new Array<fs.ReadStream>();
|
||||
function createReadStream(filePath: string) {
|
||||
const stream = fs.createReadStream(filePath);
|
||||
openStreams.push(stream);
|
||||
return stream;
|
||||
}
|
||||
afterEach(() => {
|
||||
openStreams.forEach(stream => stream.destroy());
|
||||
openStreams.length = 0;
|
||||
});
|
||||
|
||||
const path1 = sourceDir.resolve(name1);
|
||||
const path2 = sourceDir.resolve(name2);
|
||||
const path3 = sourceDir.resolve(`${dir1}/${name3}`);
|
||||
|
||||
it('should read files', async () => {
|
||||
const arr: FromReadableArrayOptions = [
|
||||
{ data: createReadStream(path1), path: path1 },
|
||||
{ data: createReadStream(path2), path: path2 },
|
||||
{ data: createReadStream(path3), path: path3 },
|
||||
];
|
||||
|
||||
const res = new ReadableArrayResponse(arr, targetDir.path, 'etag');
|
||||
const files = await res.files();
|
||||
|
||||
expect(files).toEqual([
|
||||
{ path: path1, content: expect.any(Function) },
|
||||
{ path: path2, content: expect.any(Function) },
|
||||
{ path: path3, content: expect.any(Function) },
|
||||
]);
|
||||
const contents = await Promise.all(files.map(f => f.content()));
|
||||
expect(contents).toEqual([file1, file2, file3]);
|
||||
});
|
||||
|
||||
it('should extract entire archive into directory', async () => {
|
||||
const relativePath1 = relative(sourceDir.path, path1);
|
||||
const relativePath2 = relative(sourceDir.path, path2);
|
||||
const relativePath3 = relative(sourceDir.path, path3);
|
||||
|
||||
const arr: FromReadableArrayOptions = [
|
||||
{ data: createReadStream(path1), path: relativePath1 },
|
||||
{ data: createReadStream(path2), path: relativePath2 },
|
||||
{ data: createReadStream(path3), path: relativePath3 },
|
||||
];
|
||||
|
||||
const res = new ReadableArrayResponse(arr, targetDir.path, 'etag');
|
||||
const dir = await res.dir();
|
||||
|
||||
expect(targetDir.content({ path: dir })).toEqual({
|
||||
[name1]: file1.toString('utf8'),
|
||||
[name2]: file2.toString('utf8'),
|
||||
[dir1]: {
|
||||
[name3]: file3.toString('utf8'),
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,107 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import concatStream from 'concat-stream';
|
||||
import platformPath, { dirname } from 'path';
|
||||
|
||||
import getRawBody from 'raw-body';
|
||||
import fs from 'fs-extra';
|
||||
import { promisify } from 'util';
|
||||
import tar from 'tar';
|
||||
import { pipeline as pipelineCb, Readable } from 'stream';
|
||||
import {
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseFile,
|
||||
ReadTreeResponseDirOptions,
|
||||
FromReadableArrayOptions,
|
||||
} from '../types';
|
||||
|
||||
const pipeline = promisify(pipelineCb);
|
||||
|
||||
/**
|
||||
* Wraps a array of Readable objects into a tree response reader.
|
||||
*/
|
||||
export class ReadableArrayResponse implements ReadTreeResponse {
|
||||
private read = false;
|
||||
|
||||
constructor(
|
||||
private readonly stream: FromReadableArrayOptions,
|
||||
private readonly workDir: string,
|
||||
public readonly etag: string,
|
||||
) {
|
||||
this.etag = etag;
|
||||
}
|
||||
|
||||
// Make sure the input stream is only read once
|
||||
private onlyOnce() {
|
||||
if (this.read) {
|
||||
throw new Error('Response has already been read');
|
||||
}
|
||||
this.read = true;
|
||||
}
|
||||
|
||||
async files(): Promise<ReadTreeResponseFile[]> {
|
||||
this.onlyOnce();
|
||||
|
||||
const files = Array<ReadTreeResponseFile>();
|
||||
|
||||
for (let i = 0; i < this.stream.length; i++) {
|
||||
if (!this.stream[i].path.endsWith('/')) {
|
||||
files.push({
|
||||
path: this.stream[i].path,
|
||||
content: () => getRawBody(this.stream[i].data),
|
||||
lastModifiedAt: this.stream[i]?.lastModifiedAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
async archive(): Promise<NodeJS.ReadableStream> {
|
||||
const tmpDir = await this.dir();
|
||||
|
||||
try {
|
||||
const data = await new Promise<Buffer>(async resolve => {
|
||||
await pipeline(
|
||||
tar.create({ cwd: tmpDir }, ['']),
|
||||
concatStream(resolve),
|
||||
);
|
||||
});
|
||||
return Readable.from(data);
|
||||
} finally {
|
||||
await fs.remove(tmpDir);
|
||||
}
|
||||
}
|
||||
|
||||
async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
|
||||
this.onlyOnce();
|
||||
|
||||
const dir =
|
||||
options?.targetDir ??
|
||||
(await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-')));
|
||||
|
||||
for (let i = 0; i < this.stream.length; i++) {
|
||||
if (!this.stream[i].path.endsWith('/')) {
|
||||
const filePath = platformPath.join(dir, this.stream[i].path);
|
||||
await fs.mkdir(dirname(filePath), { recursive: true });
|
||||
await pipeline(this.stream[i].data, fs.createWriteStream(filePath));
|
||||
}
|
||||
}
|
||||
|
||||
return dir;
|
||||
}
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import fs from 'fs-extra';
|
||||
import { resolve as resolvePath } from 'path';
|
||||
import { TarArchiveResponse } from './TarArchiveResponse';
|
||||
import { createMockDirectory } from '@backstage/backend-test-utils';
|
||||
|
||||
const archiveData = fs.readFileSync(
|
||||
resolvePath(__filename, '../../__fixtures__/mock-main.tar.gz'),
|
||||
);
|
||||
|
||||
describe('TarArchiveResponse', () => {
|
||||
const sourceDir = createMockDirectory();
|
||||
const targetDir = createMockDirectory();
|
||||
|
||||
beforeAll(() => {
|
||||
sourceDir.setContent({ 'test-archive.tar.gz': archiveData });
|
||||
});
|
||||
beforeEach(() => {
|
||||
targetDir.clear();
|
||||
});
|
||||
|
||||
it('should read files', async () => {
|
||||
const stream = fs.createReadStream(
|
||||
sourceDir.resolve('test-archive.tar.gz'),
|
||||
);
|
||||
|
||||
const res = new TarArchiveResponse(stream, '', targetDir.path, 'etag');
|
||||
const files = await res.files();
|
||||
|
||||
expect(files).toEqual([
|
||||
{
|
||||
path: 'mkdocs.yml',
|
||||
content: expect.any(Function),
|
||||
lastModifiedAt: undefined,
|
||||
},
|
||||
{
|
||||
path: 'docs/index.md',
|
||||
content: expect.any(Function),
|
||||
lastModifiedAt: undefined,
|
||||
},
|
||||
]);
|
||||
const contents = await Promise.all(files.map(f => f.content()));
|
||||
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
|
||||
'site_name: Test',
|
||||
'# Test',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should read files with filter', async () => {
|
||||
const stream = fs.createReadStream(
|
||||
sourceDir.resolve('test-archive.tar.gz'),
|
||||
);
|
||||
|
||||
const res = new TarArchiveResponse(
|
||||
stream,
|
||||
'',
|
||||
targetDir.path,
|
||||
'etag',
|
||||
path => path.endsWith('.yml'),
|
||||
);
|
||||
const files = await res.files();
|
||||
|
||||
expect(files).toEqual([
|
||||
{
|
||||
path: 'mkdocs.yml',
|
||||
content: expect.any(Function),
|
||||
lastModifiedAt: undefined,
|
||||
},
|
||||
]);
|
||||
const content = await files[0].content();
|
||||
expect(content.toString('utf8').trim()).toEqual('site_name: Test');
|
||||
});
|
||||
|
||||
it('should read as archive and files', async () => {
|
||||
const stream = fs.createReadStream(
|
||||
sourceDir.resolve('test-archive.tar.gz'),
|
||||
);
|
||||
|
||||
const res = new TarArchiveResponse(stream, '', targetDir.path, 'etag');
|
||||
const buffer = await res.archive();
|
||||
|
||||
await expect(res.archive()).rejects.toThrow(
|
||||
'Response has already been read',
|
||||
);
|
||||
|
||||
const res2 = new TarArchiveResponse(buffer, '', targetDir.path, 'etag');
|
||||
const files = await res2.files();
|
||||
|
||||
expect(files).toEqual([
|
||||
{
|
||||
path: 'mkdocs.yml',
|
||||
content: expect.any(Function),
|
||||
lastModifiedAt: undefined,
|
||||
},
|
||||
{
|
||||
path: 'docs/index.md',
|
||||
content: expect.any(Function),
|
||||
lastModifiedAt: undefined,
|
||||
},
|
||||
]);
|
||||
const contents = await Promise.all(files.map(f => f.content()));
|
||||
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
|
||||
'site_name: Test',
|
||||
'# Test',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should extract entire archive into directory', async () => {
|
||||
const stream = fs.createReadStream(
|
||||
sourceDir.resolve('test-archive.tar.gz'),
|
||||
);
|
||||
|
||||
const res = new TarArchiveResponse(stream, '', targetDir.path, 'etag');
|
||||
const dir = await res.dir();
|
||||
await expect(
|
||||
fs.readFile(resolvePath(dir, 'mkdocs.yml'), 'utf8'),
|
||||
).resolves.toBe('site_name: Test\n');
|
||||
await expect(
|
||||
fs.readFile(resolvePath(dir, 'docs/index.md'), 'utf8'),
|
||||
).resolves.toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('should extract archive into directory with a subpath', async () => {
|
||||
const stream = fs.createReadStream(
|
||||
sourceDir.resolve('test-archive.tar.gz'),
|
||||
);
|
||||
|
||||
const res = new TarArchiveResponse(stream, 'docs', targetDir.path, 'etag');
|
||||
const dir = await res.dir();
|
||||
|
||||
expect(targetDir.content({ path: dir })).toEqual({
|
||||
'index.md': '# Test\n',
|
||||
});
|
||||
});
|
||||
|
||||
it('should extract archive into directory with a subpath and filter', async () => {
|
||||
const stream = fs.createReadStream(
|
||||
sourceDir.resolve('test-archive.tar.gz'),
|
||||
);
|
||||
|
||||
const res = new TarArchiveResponse(
|
||||
stream,
|
||||
'',
|
||||
targetDir.path,
|
||||
'etag',
|
||||
path => path.endsWith('.yml'),
|
||||
);
|
||||
|
||||
targetDir.addContent({ sub: {} });
|
||||
const dir = await res.dir({ targetDir: targetDir.resolve('sub') });
|
||||
|
||||
expect(dir).toBe(targetDir.resolve('sub'));
|
||||
expect(targetDir.content()).toEqual({
|
||||
sub: {
|
||||
'mkdocs.yml': 'site_name: Test\n',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should clean up temporary directories in place in the case of an error', async () => {
|
||||
const stream = fs.createReadStream(
|
||||
sourceDir.resolve('test-archive.tar.gz'),
|
||||
);
|
||||
|
||||
const res = new TarArchiveResponse(
|
||||
stream,
|
||||
'',
|
||||
targetDir.path,
|
||||
'etag',
|
||||
() => {
|
||||
throw new Error('NOPE');
|
||||
},
|
||||
);
|
||||
|
||||
targetDir.addContent({ sub: {} });
|
||||
const sub = targetDir.resolve('sub');
|
||||
|
||||
const mkdtemp = jest
|
||||
.spyOn(fs, 'mkdtemp')
|
||||
.mockImplementation(async () => sub);
|
||||
|
||||
await expect(fs.pathExists(sub)).resolves.toBe(true);
|
||||
await expect(res.dir()).rejects.toThrow('NOPE');
|
||||
await expect(fs.pathExists(sub)).resolves.toBe(false);
|
||||
|
||||
mkdtemp.mockRestore();
|
||||
});
|
||||
|
||||
it('should leave directory in place if provided in the case of an error', async () => {
|
||||
const stream = fs.createReadStream(
|
||||
sourceDir.resolve('test-archive.tar.gz'),
|
||||
);
|
||||
|
||||
const res = new TarArchiveResponse(
|
||||
stream,
|
||||
'',
|
||||
targetDir.path,
|
||||
'etag',
|
||||
() => {
|
||||
throw new Error('NOPE');
|
||||
},
|
||||
);
|
||||
|
||||
targetDir.addContent({ sub: {} });
|
||||
const sub = targetDir.resolve('sub');
|
||||
|
||||
await expect(fs.pathExists(sub)).resolves.toBe(true);
|
||||
await expect(res.dir({ targetDir: sub })).rejects.toThrow('NOPE');
|
||||
await expect(fs.pathExists(sub)).resolves.toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,204 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import concatStream from 'concat-stream';
|
||||
import fs from 'fs-extra';
|
||||
import platformPath from 'path';
|
||||
import { pipeline as pipelineCb, Readable } from 'stream';
|
||||
import tar, { Parse, ParseStream, ReadEntry } from 'tar';
|
||||
import { promisify } from 'util';
|
||||
import {
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseDirOptions,
|
||||
ReadTreeResponseFile,
|
||||
} from '../types';
|
||||
import { stripFirstDirectoryFromPath } from './util';
|
||||
|
||||
// Tar types for `Parse` is not a proper constructor, but it should be
|
||||
const TarParseStream = Parse as unknown as { new (): ParseStream };
|
||||
|
||||
const pipeline = promisify(pipelineCb);
|
||||
|
||||
/**
|
||||
* Wraps a tar archive stream into a tree response reader.
|
||||
*/
|
||||
export class TarArchiveResponse implements ReadTreeResponse {
|
||||
private read = false;
|
||||
|
||||
constructor(
|
||||
private readonly stream: Readable,
|
||||
private readonly subPath: string,
|
||||
private readonly workDir: string,
|
||||
public readonly etag: string,
|
||||
private readonly filter?: (path: string, info: { size: number }) => boolean,
|
||||
private readonly stripFirstDirectory: boolean = true,
|
||||
) {
|
||||
if (subPath) {
|
||||
if (!subPath.endsWith('/')) {
|
||||
this.subPath += '/';
|
||||
}
|
||||
if (subPath.startsWith('/')) {
|
||||
throw new TypeError(
|
||||
`TarArchiveResponse subPath must not start with a /, got '${subPath}'`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
this.etag = etag;
|
||||
}
|
||||
|
||||
// Make sure the input stream is only read once
|
||||
private onlyOnce() {
|
||||
if (this.read) {
|
||||
throw new Error('Response has already been read');
|
||||
}
|
||||
this.read = true;
|
||||
}
|
||||
|
||||
async files(): Promise<ReadTreeResponseFile[]> {
|
||||
this.onlyOnce();
|
||||
|
||||
const files = Array<ReadTreeResponseFile>();
|
||||
const parser = new TarParseStream();
|
||||
|
||||
parser.on('entry', (entry: ReadEntry & Readable) => {
|
||||
if (entry.type === 'Directory') {
|
||||
entry.resume();
|
||||
return;
|
||||
}
|
||||
|
||||
// File path relative to the root extracted directory. Will remove the
|
||||
// top level dir name from the path since its name is hard to predetermine.
|
||||
const relativePath = this.stripFirstDirectory
|
||||
? stripFirstDirectoryFromPath(entry.path)
|
||||
: entry.path;
|
||||
|
||||
if (this.subPath) {
|
||||
if (!relativePath.startsWith(this.subPath)) {
|
||||
entry.resume();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const path = relativePath.slice(this.subPath.length);
|
||||
if (this.filter) {
|
||||
if (!this.filter(path, { size: entry.remain })) {
|
||||
entry.resume();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const content = new Promise<Buffer>(async resolve => {
|
||||
await pipeline(entry, concatStream(resolve));
|
||||
});
|
||||
|
||||
files.push({
|
||||
path,
|
||||
content: () => content,
|
||||
});
|
||||
|
||||
entry.resume();
|
||||
});
|
||||
|
||||
await pipeline(this.stream, parser);
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
async archive(): Promise<Readable> {
|
||||
if (!this.subPath) {
|
||||
this.onlyOnce();
|
||||
|
||||
return this.stream;
|
||||
}
|
||||
|
||||
// TODO(Rugvip): method for repacking a tar with a subpath is to simply extract into a
|
||||
// tmp dir and recreate the archive. Would be nicer to stream things instead.
|
||||
const tmpDir = await this.dir();
|
||||
|
||||
try {
|
||||
const data = await new Promise<Buffer>(async resolve => {
|
||||
await pipeline(
|
||||
tar.create({ cwd: tmpDir }, ['']),
|
||||
concatStream(resolve),
|
||||
);
|
||||
});
|
||||
return Readable.from(data);
|
||||
} finally {
|
||||
await fs.remove(tmpDir);
|
||||
}
|
||||
}
|
||||
|
||||
async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
|
||||
this.onlyOnce();
|
||||
|
||||
const dir =
|
||||
options?.targetDir ??
|
||||
(await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-')));
|
||||
|
||||
// Equivalent of tar --strip-components=N
|
||||
// When no subPath is given, remove just 1 top level directory
|
||||
let strip = this.subPath ? this.subPath.split('/').length : 1;
|
||||
if (!this.stripFirstDirectory) {
|
||||
strip--;
|
||||
}
|
||||
|
||||
let filterError: Error | undefined = undefined;
|
||||
await pipeline(
|
||||
this.stream,
|
||||
tar.extract({
|
||||
strip,
|
||||
cwd: dir,
|
||||
filter: (path, stat) => {
|
||||
// Filter errors will short-circuit the rest of the filtering and then throw
|
||||
if (filterError) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// File path relative to the root extracted directory. Will remove the
|
||||
// top level dir name from the path since its name is hard to predetermine.
|
||||
const relativePath = this.stripFirstDirectory
|
||||
? stripFirstDirectoryFromPath(path)
|
||||
: path;
|
||||
if (this.subPath && !relativePath.startsWith(this.subPath)) {
|
||||
return false;
|
||||
}
|
||||
if (this.filter) {
|
||||
const innerPath = path.split('/').slice(strip).join('/');
|
||||
try {
|
||||
return this.filter(innerPath, { size: stat.size });
|
||||
} catch (error) {
|
||||
filterError = error;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
if (filterError) {
|
||||
// If the dir was provided we don't want to remove it, but if it wasn't it means
|
||||
// we created a temporary directory and we should remove it.
|
||||
if (!options?.targetDir) {
|
||||
await fs.remove(dir).catch(() => {});
|
||||
}
|
||||
throw filterError;
|
||||
}
|
||||
|
||||
return dir;
|
||||
}
|
||||
}
|
||||
@@ -1,274 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import fs from 'fs-extra';
|
||||
import { Readable } from 'stream';
|
||||
import { create as createArchive } from 'archiver';
|
||||
import { resolve as resolvePath } from 'path';
|
||||
import { ZipArchiveResponse } from './ZipArchiveResponse';
|
||||
import { createMockDirectory } from '@backstage/backend-test-utils';
|
||||
|
||||
const archiveData = fs.readFileSync(
|
||||
resolvePath(__filename, '../../__fixtures__/mock-main.zip'),
|
||||
);
|
||||
const archiveDataCorrupted = fs.readFileSync(
|
||||
resolvePath(__filename, '../../__fixtures__/mock-corrupted.zip'),
|
||||
);
|
||||
const archiveDataWithExtraDir = fs.readFileSync(
|
||||
resolvePath(__filename, '../../__fixtures__/mock-with-extra-root-dir.zip'),
|
||||
);
|
||||
const archiveWithMaliciousEntry = fs.readFileSync(
|
||||
resolvePath(__filename, '../../__fixtures__/mallory.zip'),
|
||||
);
|
||||
|
||||
describe('ZipArchiveResponse', () => {
|
||||
const sourceDir = createMockDirectory();
|
||||
const targetDir = createMockDirectory();
|
||||
|
||||
beforeAll(() => {
|
||||
sourceDir.setContent({
|
||||
'test-archive.zip': archiveData,
|
||||
'test-archive-with-extra-root-dir.zip': archiveDataWithExtraDir,
|
||||
'test-archive-corrupted.zip': archiveDataCorrupted,
|
||||
'test-archive-malicious.zip': archiveWithMaliciousEntry,
|
||||
});
|
||||
});
|
||||
beforeEach(() => {
|
||||
targetDir.clear();
|
||||
});
|
||||
|
||||
const openStreams = new Array<fs.ReadStream>();
|
||||
function createReadStream(filePath: string) {
|
||||
const stream = fs.createReadStream(filePath);
|
||||
openStreams.push(stream);
|
||||
return stream;
|
||||
}
|
||||
afterEach(() => {
|
||||
openStreams.forEach(stream => stream.destroy());
|
||||
openStreams.length = 0;
|
||||
});
|
||||
|
||||
it('should read files', async () => {
|
||||
const stream = createReadStream(sourceDir.resolve('test-archive.zip'));
|
||||
|
||||
const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag');
|
||||
const files = await res.files();
|
||||
|
||||
expect(files).toEqual([
|
||||
{
|
||||
path: 'mkdocs.yml',
|
||||
content: expect.any(Function),
|
||||
lastModifiedAt: expect.any(Date),
|
||||
},
|
||||
{
|
||||
path: 'docs/index.md',
|
||||
content: expect.any(Function),
|
||||
lastModifiedAt: expect.any(Date),
|
||||
},
|
||||
]);
|
||||
|
||||
const contents = await Promise.all(files.map(f => f.content()));
|
||||
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
|
||||
'site_name: Test',
|
||||
'# Test',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should read files with filter', async () => {
|
||||
const stream = createReadStream(sourceDir.resolve('test-archive.zip'));
|
||||
|
||||
const res = new ZipArchiveResponse(
|
||||
stream,
|
||||
'',
|
||||
targetDir.path,
|
||||
'etag',
|
||||
path => path.endsWith('.yml'),
|
||||
);
|
||||
const files = await res.files();
|
||||
|
||||
expect(files).toEqual([
|
||||
{
|
||||
path: 'mkdocs.yml',
|
||||
content: expect.any(Function),
|
||||
lastModifiedAt: expect.any(Date),
|
||||
},
|
||||
]);
|
||||
const content = await files[0].content();
|
||||
expect(content.toString('utf8').trim()).toEqual('site_name: Test');
|
||||
});
|
||||
|
||||
it('should read as archive and files', async () => {
|
||||
const stream = createReadStream(sourceDir.resolve('test-archive.zip'));
|
||||
|
||||
const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag');
|
||||
const buffer = await res.archive();
|
||||
|
||||
await expect(res.archive()).rejects.toThrow(
|
||||
'Response has already been read',
|
||||
);
|
||||
|
||||
const res2 = new ZipArchiveResponse(buffer, '', targetDir.path, 'etag');
|
||||
const files = await res2.files();
|
||||
|
||||
expect(files).toEqual([
|
||||
{
|
||||
path: 'mkdocs.yml',
|
||||
content: expect.any(Function),
|
||||
lastModifiedAt: expect.any(Date),
|
||||
},
|
||||
{
|
||||
path: 'docs/index.md',
|
||||
content: expect.any(Function),
|
||||
lastModifiedAt: expect.any(Date),
|
||||
},
|
||||
]);
|
||||
const contents = await Promise.all(files.map(f => f.content()));
|
||||
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
|
||||
'site_name: Test',
|
||||
'# Test',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should extract entire archive into directory', async () => {
|
||||
const stream = createReadStream(sourceDir.resolve('test-archive.zip'));
|
||||
|
||||
const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag');
|
||||
const dir = await res.dir();
|
||||
|
||||
await expect(
|
||||
fs.readFile(resolvePath(dir, 'mkdocs.yml'), 'utf8'),
|
||||
).resolves.toBe('site_name: Test\n');
|
||||
await expect(
|
||||
fs.readFile(resolvePath(dir, 'docs/index.md'), 'utf8'),
|
||||
).resolves.toBe('# Test\n');
|
||||
});
|
||||
|
||||
it('should extract archive into directory with a subpath', async () => {
|
||||
const stream = createReadStream(sourceDir.resolve('test-archive.zip'));
|
||||
|
||||
const res = new ZipArchiveResponse(stream, 'docs/', targetDir.path, 'etag');
|
||||
|
||||
const dir = await res.dir();
|
||||
expect(targetDir.content({ path: dir })).toEqual({
|
||||
'index.md': '# Test\n',
|
||||
});
|
||||
});
|
||||
|
||||
it('should extract archive into directory with a subpath and filter', async () => {
|
||||
const stream = createReadStream(sourceDir.resolve('test-archive.zip'));
|
||||
|
||||
const res = new ZipArchiveResponse(
|
||||
stream,
|
||||
'',
|
||||
targetDir.path,
|
||||
'etag',
|
||||
path => path.endsWith('.yml'),
|
||||
);
|
||||
|
||||
targetDir.addContent({ sub: {} });
|
||||
const sub = targetDir.resolve('sub');
|
||||
const dir = await res.dir({ targetDir: sub });
|
||||
|
||||
expect(dir).toBe(sub);
|
||||
|
||||
expect(targetDir.content()).toEqual({
|
||||
sub: {
|
||||
'mkdocs.yml': 'site_name: Test\n',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should extract a large archive', async () => {
|
||||
const fileCount = 10;
|
||||
const fileSize = 1000 * 1000;
|
||||
const filePath = await new Promise<string>((resolve, reject) => {
|
||||
const outFile = targetDir.resolve('large-archive.zip');
|
||||
|
||||
const outStream = fs.createWriteStream(outFile);
|
||||
outStream.on('close', () => resolve(outFile));
|
||||
|
||||
const archive = createArchive('zip');
|
||||
archive.on('error', reject);
|
||||
archive.on('warning', w => console.warn('WARN', w));
|
||||
archive.pipe(outStream);
|
||||
|
||||
for (let i = 0; i < fileCount; i++) {
|
||||
// Workaround for https://github.com/archiverjs/node-archiver/issues/542
|
||||
// TODO(Rugvip): Without this workaround the archive entries end up with an uncompressed size of 0.
|
||||
// That in turn causes yauzl to hang on extraction, because the internal transform error is ignored.
|
||||
const stream = new Readable();
|
||||
stream.push(Buffer.alloc(fileSize, i));
|
||||
stream.push(null);
|
||||
archive.append(stream, { name: `file-${i}.data` });
|
||||
}
|
||||
|
||||
archive.finalize();
|
||||
});
|
||||
|
||||
const stream = createReadStream(filePath);
|
||||
|
||||
const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag');
|
||||
|
||||
targetDir.addContent({ sub: {} });
|
||||
const sub = targetDir.resolve('sub');
|
||||
const dir = await res.dir({
|
||||
targetDir: sub,
|
||||
});
|
||||
|
||||
const files = await fs.readdir(dir);
|
||||
expect(files).toHaveLength(fileCount);
|
||||
|
||||
for (const file of files) {
|
||||
const stat = await fs.stat(resolvePath(dir, file));
|
||||
expect(stat.size).toBe(fileSize);
|
||||
}
|
||||
});
|
||||
|
||||
it('should throw on invalid archive', async () => {
|
||||
const stream = createReadStream(
|
||||
sourceDir.resolve('test-archive-corrupted.zip'),
|
||||
);
|
||||
|
||||
const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag');
|
||||
const filesPromise = res.files();
|
||||
|
||||
await expect(filesPromise).rejects.toThrow(
|
||||
/Invalid comment length. Expected: 55. Found: 0/,
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw on entries with a path outside the destination dir', async () => {
|
||||
const stream = createReadStream(
|
||||
sourceDir.resolve('test-archive-malicious.zip'),
|
||||
);
|
||||
|
||||
const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag');
|
||||
await expect(res.files()).rejects.toThrow(
|
||||
'invalid relative path: ../side.txt',
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw on entries that attempt to write outside destination dir', async () => {
|
||||
const stream = createReadStream(
|
||||
sourceDir.resolve('test-archive-malicious.zip'),
|
||||
);
|
||||
|
||||
const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag');
|
||||
await expect(res.dir()).rejects.toThrow(
|
||||
'invalid relative path: ../side.txt',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,215 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import archiver from 'archiver';
|
||||
import yauzl, { Entry } from 'yauzl';
|
||||
import fs from 'fs-extra';
|
||||
import platformPath from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import {
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseDirOptions,
|
||||
ReadTreeResponseFile,
|
||||
} from '../types';
|
||||
import { streamToBuffer } from './util';
|
||||
import { resolveSafeChildPath } from '@backstage/backend-plugin-api';
|
||||
|
||||
/**
|
||||
* Wraps a zip archive stream into a tree response reader.
|
||||
*/
|
||||
export class ZipArchiveResponse implements ReadTreeResponse {
|
||||
private read = false;
|
||||
|
||||
constructor(
|
||||
private readonly stream: Readable,
|
||||
private readonly subPath: string,
|
||||
private readonly workDir: string,
|
||||
public readonly etag: string,
|
||||
private readonly filter?: (path: string, info: { size: number }) => boolean,
|
||||
) {
|
||||
if (subPath) {
|
||||
if (!subPath.endsWith('/')) {
|
||||
this.subPath += '/';
|
||||
}
|
||||
if (subPath.startsWith('/')) {
|
||||
throw new TypeError(
|
||||
`ZipArchiveResponse subPath must not start with a /, got '${subPath}'`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
this.etag = etag;
|
||||
}
|
||||
|
||||
// Make sure the input stream is only read once
|
||||
private onlyOnce() {
|
||||
if (this.read) {
|
||||
throw new Error('Response has already been read');
|
||||
}
|
||||
this.read = true;
|
||||
}
|
||||
|
||||
// File path relative to the root extracted directory or a sub directory if subpath is set.
|
||||
private getInnerPath(path: string): string {
|
||||
return path.slice(this.subPath.length);
|
||||
}
|
||||
|
||||
private shouldBeIncluded(entry: Entry): boolean {
|
||||
if (this.subPath) {
|
||||
if (!entry.fileName.startsWith(this.subPath)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (this.filter) {
|
||||
return this.filter(this.getInnerPath(entry.fileName), {
|
||||
size: entry.uncompressedSize,
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private async streamToTemporaryFile(
|
||||
stream: Readable,
|
||||
): Promise<{ fileName: string; cleanup: () => Promise<void> }> {
|
||||
const tmpDir = await fs.mkdtemp(
|
||||
platformPath.join(this.workDir, 'backstage-tmp'),
|
||||
);
|
||||
const tmpFile = platformPath.join(tmpDir, 'tmp.zip');
|
||||
|
||||
const writeStream = fs.createWriteStream(tmpFile);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
writeStream.on('error', reject);
|
||||
writeStream.on('finish', () => {
|
||||
writeStream.end();
|
||||
resolve({
|
||||
fileName: tmpFile,
|
||||
cleanup: () => fs.rm(tmpDir, { recursive: true }),
|
||||
});
|
||||
});
|
||||
stream.pipe(writeStream);
|
||||
});
|
||||
}
|
||||
|
||||
private forEveryZipEntry(
|
||||
zip: string,
|
||||
callback: (entry: Entry, content: Readable) => Promise<void>,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
yauzl.open(zip, { lazyEntries: true }, (err, zipfile) => {
|
||||
if (err || !zipfile) {
|
||||
reject(err || new Error(`Failed to open zip file ${zip}`));
|
||||
return;
|
||||
}
|
||||
|
||||
zipfile.on('entry', async (entry: Entry) => {
|
||||
// Check that the file is not a directory, and that is matches the filter.
|
||||
if (!entry.fileName.endsWith('/') && this.shouldBeIncluded(entry)) {
|
||||
zipfile.openReadStream(entry, async (openErr, readStream) => {
|
||||
if (openErr || !readStream) {
|
||||
reject(
|
||||
openErr ||
|
||||
new Error(`Failed to open zip entry ${entry.fileName}`),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await callback(entry, readStream);
|
||||
zipfile.readEntry();
|
||||
});
|
||||
} else {
|
||||
zipfile.readEntry();
|
||||
}
|
||||
});
|
||||
zipfile.once('end', () => resolve());
|
||||
zipfile.on('error', e => reject(e));
|
||||
zipfile.readEntry();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async files(): Promise<ReadTreeResponseFile[]> {
|
||||
this.onlyOnce();
|
||||
const files = Array<ReadTreeResponseFile>();
|
||||
const temporary = await this.streamToTemporaryFile(this.stream);
|
||||
|
||||
await this.forEveryZipEntry(temporary.fileName, async (entry, content) => {
|
||||
files.push({
|
||||
path: this.getInnerPath(entry.fileName),
|
||||
content: async () => await streamToBuffer(content),
|
||||
lastModifiedAt: entry.lastModFileTime
|
||||
? new Date(entry.lastModFileTime)
|
||||
: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
await temporary.cleanup();
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
async archive(): Promise<Readable> {
|
||||
this.onlyOnce();
|
||||
|
||||
if (!this.subPath) {
|
||||
return this.stream;
|
||||
}
|
||||
|
||||
const archive = archiver('zip');
|
||||
const temporary = await this.streamToTemporaryFile(this.stream);
|
||||
|
||||
await this.forEveryZipEntry(temporary.fileName, async (entry, content) => {
|
||||
archive.append(await streamToBuffer(content), {
|
||||
name: this.getInnerPath(entry.fileName),
|
||||
});
|
||||
});
|
||||
|
||||
archive.finalize();
|
||||
|
||||
await temporary.cleanup();
|
||||
|
||||
return archive;
|
||||
}
|
||||
|
||||
async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
|
||||
this.onlyOnce();
|
||||
const dir =
|
||||
options?.targetDir ??
|
||||
(await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-')));
|
||||
|
||||
const temporary = await this.streamToTemporaryFile(this.stream);
|
||||
|
||||
await this.forEveryZipEntry(temporary.fileName, async (entry, content) => {
|
||||
const entryPath = this.getInnerPath(entry.fileName);
|
||||
const dirname = platformPath.dirname(entryPath);
|
||||
|
||||
if (dirname) {
|
||||
await fs.mkdirp(resolveSafeChildPath(dir, dirname));
|
||||
}
|
||||
return new Promise(async (resolve, reject) => {
|
||||
const file = fs.createWriteStream(resolveSafeChildPath(dir, entryPath));
|
||||
file.on('finish', resolve);
|
||||
|
||||
content.on('error', reject);
|
||||
content.pipe(file);
|
||||
});
|
||||
});
|
||||
|
||||
await temporary.cleanup();
|
||||
|
||||
return dir;
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { DefaultReadTreeResponseFactory } from './ReadTreeResponseFactory';
|
||||
@@ -1,41 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Readable, pipeline as pipelineCb } from 'stream';
|
||||
import { promisify } from 'util';
|
||||
import concatStream from 'concat-stream';
|
||||
|
||||
const pipeline = promisify(pipelineCb);
|
||||
|
||||
// Matches a directory name + one `/` at the start of any string,
|
||||
// containing any character except `/` one or more times, and ending with a `/`
|
||||
// e.g. Will match `dirA/` in `dirA/dirB/file.ext`
|
||||
const directoryNameRegex = /^[^\/]+\//;
|
||||
// Removes the first segment of a forward-slash-separated path
|
||||
export function stripFirstDirectoryFromPath(path: string): string {
|
||||
return path.replace(directoryNameRegex, '');
|
||||
}
|
||||
|
||||
// Collect the stream into a buffer and return
|
||||
export const streamToBuffer = (stream: Readable): Promise<Buffer> => {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
try {
|
||||
await pipeline(stream, concatStream(resolve));
|
||||
} catch (ex) {
|
||||
reject(ex);
|
||||
}
|
||||
});
|
||||
};
|
||||
@@ -1,133 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Readable } from 'stream';
|
||||
import { Config } from '@backstage/config';
|
||||
import {
|
||||
UrlReaderService,
|
||||
ReadTreeResponse,
|
||||
LoggerService,
|
||||
} from '@backstage/backend-plugin-api';
|
||||
|
||||
export type {
|
||||
UrlReaderService as UrlReader,
|
||||
ReadTreeOptions,
|
||||
ReadTreeResponse,
|
||||
ReadTreeResponseDirOptions,
|
||||
ReadTreeResponseFile,
|
||||
ReadUrlResponse,
|
||||
ReadUrlOptions,
|
||||
SearchOptions,
|
||||
SearchResponse,
|
||||
SearchResponseFile,
|
||||
} from '@backstage/backend-plugin-api';
|
||||
|
||||
/**
|
||||
* A predicate that decides whether a specific {@link @backstage/backend-plugin-api#UrlReaderService} can handle a
|
||||
* given URL.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type UrlReaderPredicateTuple = {
|
||||
predicate: (url: URL) => boolean;
|
||||
reader: UrlReaderService;
|
||||
};
|
||||
|
||||
/**
|
||||
* A factory function that can read config to construct zero or more
|
||||
* {@link @backstage/backend-plugin-api#UrlReaderService}s along with a predicate for when it should be used.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type ReaderFactory = (options: {
|
||||
config: Config;
|
||||
logger: LoggerService;
|
||||
treeResponseFactory: ReadTreeResponseFactory;
|
||||
}) => UrlReaderPredicateTuple[];
|
||||
|
||||
/**
|
||||
* An options object for {@link ReadUrlResponseFactory} factory methods.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type ReadUrlResponseFactoryFromStreamOptions = {
|
||||
etag?: string;
|
||||
lastModifiedAt?: Date;
|
||||
};
|
||||
|
||||
/**
|
||||
* Options that control execution of {@link ReadTreeResponseFactory} methods.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type ReadTreeResponseFactoryOptions = {
|
||||
// A binary stream of a tar archive.
|
||||
stream: Readable;
|
||||
// If unset, the files at the root of the tree will be read.
|
||||
// subpath must not contain the name of the top level directory.
|
||||
subpath?: string;
|
||||
// etag of the blob
|
||||
etag: string;
|
||||
// Filter passed on from the ReadTreeOptions
|
||||
filter?: (path: string, info?: { size: number }) => boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Options that control {@link ReadTreeResponseFactory.fromReadableArray}
|
||||
* execution.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type FromReadableArrayOptions = Array<{
|
||||
/**
|
||||
* The raw data itself.
|
||||
*/
|
||||
data: Readable;
|
||||
|
||||
/**
|
||||
* The filepath of the data.
|
||||
*/
|
||||
path: string;
|
||||
|
||||
/**
|
||||
* Last modified date of the file contents.
|
||||
*/
|
||||
lastModifiedAt?: Date;
|
||||
}>;
|
||||
|
||||
/**
|
||||
* A factory for response factories that handle the unpacking and inspection of
|
||||
* complex responses such as archive data.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export interface ReadTreeResponseFactory {
|
||||
fromTarArchive(
|
||||
options: ReadTreeResponseFactoryOptions & {
|
||||
/**
|
||||
* Strip the first parent directory of a tar archive.
|
||||
* Defaults to true.
|
||||
*/
|
||||
stripFirstDirectory?: boolean;
|
||||
},
|
||||
): Promise<ReadTreeResponse>;
|
||||
fromZipArchive(
|
||||
options: ReadTreeResponseFactoryOptions,
|
||||
): Promise<ReadTreeResponse>;
|
||||
fromReadableArray(
|
||||
options: FromReadableArrayOptions,
|
||||
): Promise<ReadTreeResponse>;
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export function parseLastModified(value: string | null | undefined) {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return new Date(value);
|
||||
}
|
||||
Reference in New Issue
Block a user