paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 37 Packages Projects Releases Wiki Activity

Merge pull request #3283 from backstage/rugvip/retree

Browse Source 
backend-common: refactor readTree to only accept URL and options + add response helper
This commit is contained in:
Patrik Oldsberg
2020-11-13 10:53:53 +01:00
committed by GitHub
parent d21e4172b5 bd1cef1847
commit 484385c658
21 changed files with 597 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/odd-camels-begin.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/backend-common': minor
---
Refactored UrlReader.readTree to be required and accept (url, options)
packages/backend-common/src/reading/AzureUrlReader.test.ts
+15 -2
View File
@@ -20,9 +20,14 @@ import { ConfigReader } from '@backstage/config';
import { getVoidLogger } from '../logging';
import { AzureUrlReader } from './AzureUrlReader';
import { msw } from '@backstage/test-utils';
import { ReadTreeResponseFactory } from './tree';
const logger = getVoidLogger();
const treeResponseFactory = ReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
describe('AzureUrlReader', () => {
const worker = setupServer();
msw.setupDefaultHandlers(worker);
@@ -87,7 +92,11 @@ describe('AzureUrlReader', () => {
}),
},
])('should handle happy path %#', async ({ url, config, response }) => {
const [{ reader }] = AzureUrlReader.factory({ config, logger });
const [{ reader }] = AzureUrlReader.factory({
config,
logger,
treeResponseFactory,
});
const data = await reader.read(url);
const res = await JSON.parse(data.toString('utf-8'));
@@ -115,7 +124,11 @@ describe('AzureUrlReader', () => {
},
])('should handle error path %#', async ({ url, config, error }) => {
await expect(async () => {
const [{ reader }] = AzureUrlReader.factory({ config, logger });
const [{ reader }] = AzureUrlReader.factory({
config,
logger,
treeResponseFactory,
});
await reader.read(url);
}).rejects.toThrow(error);
});
packages/backend-common/src/reading/AzureUrlReader.ts
+5 -1
View File
@@ -17,7 +17,7 @@
import fetch from 'cross-fetch';
import { Config } from '@backstage/config';
import { NotFoundError } from '../errors';
import { ReaderFactory, UrlReader } from './types';
import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
type Options = {
// TODO: added here for future support, but we only allow dev.azure.com for now
@@ -86,6 +86,10 @@ export class AzureUrlReader implements UrlReader {
throw new Error(message);
}
readTree(): Promise<ReadTreeResponse> {
throw new Error('AzureUrlReader does not implement readTree');
}
// Converts
// from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents
// to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch}
packages/backend-common/src/reading/BitbucketUrlReader.ts
+5 -1
View File
@@ -18,7 +18,7 @@ import { Config } from '@backstage/config';
import parseGitUri from 'git-url-parse';
import fetch from 'cross-fetch';
import { NotFoundError } from '../errors';
import { ReaderFactory, UrlReader } from './types';
import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
const DEFAULT_BASE_URL = 'https://api.bitbucket.org/2.0';
@@ -209,6 +209,10 @@ export class BitbucketUrlReader implements UrlReader {
throw new Error(message);
}
readTree(): Promise<ReadTreeResponse> {
throw new Error('BitbucketUrlReader does not implement readTree');
}
toString() {
const { host, token, username, appPassword } = this.config;
let authed = Boolean(token);
packages/backend-common/src/reading/FetchUrlReader.ts
+5 -1
View File
@@ -16,7 +16,7 @@
import fetch from 'cross-fetch';
import { NotFoundError } from '../errors';
import { UrlReader } from './types';
import { ReadTreeResponse, UrlReader } from './types';
/**
* A UrlReader that does a plain fetch of the URL.
@@ -41,6 +41,10 @@ export class FetchUrlReader implements UrlReader {
throw new Error(message);
}
readTree(): Promise<ReadTreeResponse> {
throw new Error('FetchUrlReader does not implement readTree');
}
toString() {
return 'fetch{}';
}
packages/backend-common/src/reading/GithubUrlReader.test.ts
+51 -37
View File
@@ -29,8 +29,11 @@ import {
} from './GithubUrlReader';
import fs from 'fs';
import path from 'path';
import mockfs from 'mock-fs';
import recursive from 'recursive-readdir';
import { ReadTreeResponseFactory } from './tree';
const treeResponseFactory = ReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
describe('GithubUrlReader', () => {
describe('getApiRequestOptions', () => {
@@ -226,10 +229,13 @@ describe('GithubUrlReader', () => {
describe('implementation', () => {
it('rejects unknown targets', async () => {
const processor = new GithubUrlReader({
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
});
const processor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory },
);
await expect(
processor.read('https://not.github.com/apa'),
).rejects.toThrow(
@@ -250,7 +256,7 @@ describe('GithubUrlReader', () => {
beforeEach(() => {
worker.use(
rest.get(
'https://github.com/spotify/mock/archive/repo.tar.gz',
'https://github.com/backstage/mock/archive/repo.tar.gz',
(_, res, ctx) =>
res(
ctx.status(200),
@@ -262,18 +268,20 @@ describe('GithubUrlReader', () => {
});
it('returns the wanted files from an archive', async () => {
const processor = new GithubUrlReader({
host: 'github.com',
});
const response = await processor.readTree(
'https://github.com/spotify/mock',
'repo',
['mkdocs.yml', 'docs'],
const processor = new GithubUrlReader(
{
host: 'github.com',
},
{ treeResponseFactory },
);
const files = response.files();
const response = await processor.readTree(
'https://github.com/backstage/mock/tree/repo',
);
const files = await response.files();
expect(files.length).toBe(2);
const mkDocsFile = await files[0].content();
const indexMarkdownFile = await files[1].content();
@@ -281,33 +289,39 @@ describe('GithubUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('returns a folder path from an archive', async () => {
const processor = new GithubUrlReader({
host: 'github.com',
});
it('must specify a branch', async () => {
const processor = new GithubUrlReader(
{
host: 'github.com',
},
{ treeResponseFactory },
);
await expect(
processor.readTree('https://github.com/backstage/mock'),
).rejects.toThrow(
'GitHub URL must contain branch to be able to fetch tree',
);
});
it('returns the wanted files from an archive with a subpath', async () => {
const processor = new GithubUrlReader(
{
host: 'github.com',
},
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://github.com/spotify/mock',
'repo',
['mkdocs.yml', 'docs'],
'https://github.com/backstage/mock/tree/repo/docs',
);
mockfs();
const directory = await response.dir('/tmp/fs');
const files = await response.files();
const writtenToDirectory = fs.existsSync(directory);
const paths = await recursive(directory);
mockfs.restore();
expect(files.length).toBe(1);
const indexMarkdownFile = await files[0].content();
expect(writtenToDirectory).toBe(true);
expect(paths.sort()).toEqual(
[
'/tmp/fs/mock-repo/docs/index.md',
'/tmp/fs/mock-repo/mkdocs.yml',
].sort(),
);
worker.resetHandlers();
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
});
});
packages/backend-common/src/reading/GithubUrlReader.ts
+51 -91
View File
@@ -17,13 +17,15 @@
import { Config } from '@backstage/config';
import parseGitUri from 'git-url-parse';
import fetch from 'cross-fetch';
import { NotFoundError } from '../errors';
import { ReaderFactory, ReadTreeResponse, UrlReader, File } from './types';
import tar from 'tar';
import fs from 'fs-extra';
import concatStream from 'concat-stream';
import path from 'path';
import os from 'os';
import { Readable } from 'stream';
import { InputError, NotFoundError } from '../errors';
import {
ReaderFactory,
ReadTreeResponse,
UrlReader,
ReadTreeOptions,
} from './types';
import { ReadTreeResponseFactory } from './tree';
/**
* The configuration parameters for a single GitHub API provider.
@@ -192,19 +194,18 @@ export function readConfig(config: Config): ProviderConfig[] {
* the one exposed by GitHub itself.
*/
export class GithubUrlReader implements UrlReader {
private config: ProviderConfig;
static factory: ReaderFactory = ({ config }) => {
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
return readConfig(config).map(provider => {
const reader = new GithubUrlReader(provider);
const reader = new GithubUrlReader(provider, { treeResponseFactory });
const predicate = (url: URL) => url.host === provider.host;
return { reader, predicate };
});
};
constructor(config: ProviderConfig) {
this.config = config;
}
constructor(
private readonly config: ProviderConfig,
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
) {}
async read(url: string): Promise<Buffer> {
const useApi =
@@ -234,89 +235,48 @@ export class GithubUrlReader implements UrlReader {
throw new Error(message);
}
private async getRepositoryArchive(
repoUrl: string,
branchName: string,
): Promise<Response> {
return fetch(new URL(`${repoUrl}/archive/${branchName}.tar.gz`).toString());
}
private async writeBufferToFile(
filePath: string,
content: Buffer,
): Promise<void> {
await fs.outputFile(filePath, content.toString());
}
async readTree(
repoUrl: string,
branchName: string,
paths: Array<string>,
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const { name: repoName } = parseGitUri(repoUrl);
const {
name: repoName,
ref,
protocol,
source,
full_name,
filepath,
} = parseGitUri(url);
const repoArchive = await this.getRepositoryArchive(repoUrl, branchName);
if (!ref) {
// TODO(Rugvip): We should add support for defaulting to the default branch
throw new InputError(
'GitHub URL must contain branch to be able to fetch tree',
);
}
const files: File[] = [];
return new Promise(resolve => {
const parser = new (tar.Parse as any)({
filter: (path: string) =>
!!paths.filter(file => {
return path.startsWith(`${repoName}-${branchName}/${file}`);
}).length,
onentry: (entry: tar.ReadEntry) => {
if (entry.type === 'Directory') {
entry.resume();
return;
}
// TODO(Rugvip): use API to fetch URL instead
const response = await fetch(
new URL(
`${protocol}://${source}/${full_name}/archive/${ref}.tar.gz`,
).toString(),
);
if (!response.ok) {
const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const contentPromise: Promise<Buffer> = new Promise(res => {
entry.pipe(concatStream(res));
});
const path = `${repoName}-${ref}/${filepath}`;
files.push({
path: entry.path,
content: () => contentPromise,
});
entry.resume();
},
});
// @ts-ignore Typescript doesn't consider .pipe a method on ReadableStream. Don't know why.
repoArchive.body?.pipe(parser).on('finish', () => {
resolve({
files: () => {
return files;
},
archive: () => {
return new Promise(resolve =>
resolve(Buffer.from('Archive is not yet implemented')),
);
},
dir: (outDir: string | undefined) => {
const targetDirectory =
outDir || fs.mkdtempSync(path.join(os.tmpdir(), 'backstage-'));
return new Promise((res, rej) => {
Promise.all(
files.map(async file => {
return this.writeBufferToFile(
`${targetDirectory}/${file.path}`,
await file.content(),
);
}),
)
.then(() => {
res(`${targetDirectory}/${repoName}-${branchName}`);
})
.catch(err => {
rej(err);
});
});
},
});
});
return this.deps.treeResponseFactory.fromArchive({
// TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want
// to stick to using that in exclusively backend code.
stream: (response.body as unknown) as Readable,
path,
filter: options?.filter,
});
}
packages/backend-common/src/reading/GitlabUrlReader.test.ts
+15 -2
View File
@@ -20,9 +20,14 @@ import { ConfigReader } from '@backstage/config';
import { getVoidLogger } from '../logging';
import { GitlabUrlReader } from './GitlabUrlReader';
import { msw } from '@backstage/test-utils';
import { ReadTreeResponseFactory } from './tree';
const logger = getVoidLogger();
const treeResponseFactory = ReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
describe('GitlabUrlReader', () => {
const worker = setupServer();
@@ -98,7 +103,11 @@ describe('GitlabUrlReader', () => {
}),
},
])('should handle happy path %#', async ({ url, config, response }) => {
const [{ reader }] = GitlabUrlReader.factory({ config, logger });
const [{ reader }] = GitlabUrlReader.factory({
config,
logger,
treeResponseFactory,
});
const data = await reader.read(url);
const res = await JSON.parse(data.toString('utf-8'));
@@ -114,7 +123,11 @@ describe('GitlabUrlReader', () => {
},
])('should handle error path %#', async ({ url, config, error }) => {
await expect(async () => {
const [{ reader }] = GitlabUrlReader.factory({ config, logger });
const [{ reader }] = GitlabUrlReader.factory({
config,
logger,
treeResponseFactory,
});
await reader.read(url);
}).rejects.toThrow(error);
});
packages/backend-common/src/reading/GitlabUrlReader.ts
+5 -1
View File
@@ -17,7 +17,7 @@
import fetch from 'cross-fetch';
import { Config } from '@backstage/config';
import { NotFoundError } from '../errors';
import { ReaderFactory, UrlReader } from './types';
import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
type Options = {
host: string;
@@ -87,6 +87,10 @@ export class GitlabUrlReader implements UrlReader {
throw new Error(message);
}
readTree(): Promise<ReadTreeResponse> {
throw new Error('GitlabUrlReader does not implement readTree');
}
// Converts
// from: https://gitlab.example.com/a/b/blob/master/c.yaml
// to: https://gitlab.example.com/a/b/raw/master/c.yaml
packages/backend-common/src/reading/UrlReaderPredicateMux.ts
+11 -17
View File
@@ -14,7 +14,12 @@
* limitations under the License.
*/
import { ReadTreeResponse, UrlReader, UrlReaderPredicateTuple } from './types';
import {
ReadTreeOptions,
ReadTreeResponse,
UrlReader,
UrlReaderPredicateTuple,
} from './types';
type Options = {
// UrlReader to fall back to if no other reader is matched
@@ -53,31 +58,20 @@ export class UrlReaderPredicateMux implements UrlReader {
throw new Error(`No reader found that could handle '${url}'`);
}
readTree(
repoUrl: string,
branchName: string,
paths: Array<string>,
): Promise<ReadTreeResponse> {
const parsed = new URL(repoUrl);
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse> {
const parsed = new URL(url);
for (const { predicate, reader } of this.readers) {
if (predicate(parsed)) {
if (reader.readTree) return reader.readTree(repoUrl, branchName, paths);
throw new Error(
`Trying to call readTree on UrlReader which does not support the feature.`,
);
return reader.readTree(url, options);
}
}
if (this.fallback) {
if (this.fallback.readTree)
return this.fallback.readTree(repoUrl, branchName, paths);
throw new Error(
`Trying to call readTree on UrlReader which does not support the feature.`,
);
return this.fallback.readTree(url, options);
}
throw new Error(`No reader found that could handle '${repoUrl}'`);
throw new Error(`No reader found that could handle '${url}'`);
}
toString() {
packages/backend-common/src/reading/UrlReaders.ts
+3 -1
View File
@@ -23,6 +23,7 @@ import { BitbucketUrlReader } from './BitbucketUrlReader';
import { GithubUrlReader } from './GithubUrlReader';
import { GitlabUrlReader } from './GitlabUrlReader';
import { FetchUrlReader } from './FetchUrlReader';
import { ReadTreeResponseFactory } from './tree';
type CreateOptions = {
/** Root config object */
@@ -49,9 +50,10 @@ export class UrlReaders {
fallback,
}: CreateOptions): UrlReader {
const mux = new UrlReaderPredicateMux({ fallback: fallback });
const treeResponseFactory = ReadTreeResponseFactory.create({ config });
for (const factory of factories ?? []) {
const tuples = factory({ config, logger: logger });
const tuples = factory({ config, logger: logger, treeResponseFactory });
for (const tuple of tuples) {
mux.register(tuple);
packages/backend-common/src/reading/tree/ArchiveResponse.test.ts
+151
View File
@@ -0,0 +1,151 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import { resolve as resolvePath } from 'path';
import { ArchiveResponse } from './ArchiveResponse';
const archiveData = fs.readFileSync(
resolvePath(__filename, '../../__fixtures__/repo.tar.gz'),
);
describe('ArchiveResponse', () => {
beforeEach(() => {
mockFs({
'/test-archive.tar.gz': archiveData,
'/tmp': mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
it('should read files', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp');
const files = await res.files();
expect(files).toEqual([
{
path: 'mkdocs.yml',
content: expect.any(Function),
},
{
path: 'docs/index.md',
content: expect.any(Function),
},
]);
const contents = await Promise.all(files.map(f => f.content()));
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
'site_name: Test',
'# Test',
]);
});
it('should read files with filter', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
);
const files = await res.files();
expect(files).toEqual([
{
path: 'mkdocs.yml',
content: expect.any(Function),
},
]);
const content = await files[0].content();
expect(content.toString('utf8').trim()).toEqual('site_name: Test');
});
it('should read as archive and files', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp');
const buffer = await res.archive();
await expect(res.archive()).rejects.toThrow(
'Response has already been read',
);
const res2 = new ArchiveResponse(buffer, '', '/tmp');
const files = await res2.files();
expect(files).toEqual([
{
path: 'mkdocs.yml',
content: expect.any(Function),
},
{
path: 'docs/index.md',
content: expect.any(Function),
},
]);
const contents = await Promise.all(files.map(f => f.content()));
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
'site_name: Test',
'# Test',
]);
});
it('should extract entire archive into directory', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new ArchiveResponse(stream, '', '/tmp');
const dir = await res.dir();
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('should extract archive into directory with a subpath', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new ArchiveResponse(stream, 'mock-repo/docs/', '/tmp');
const dir = await res.dir();
expect(dir).toMatch(/^\/tmp\/.*$/);
await expect(
fs.readFile(resolvePath(dir, 'index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('should extract archive into directory with a subpath and filter', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
);
const dir = await res.dir({ targetDir: '/tmp' });
expect(dir).toBe('/tmp');
await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe(
true,
);
await expect(
fs.pathExists(resolvePath(dir, 'docs/index.md')),
).resolves.toBe(false);
});
});
packages/backend-common/src/reading/tree/ArchiveResponse.ts
+160
View File
@@ -0,0 +1,160 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import tar, { Parse, ParseStream, ReadEntry } from 'tar';
import path from 'path';
import fs from 'fs-extra';
import { Readable, pipeline as pipelineCb } from 'stream';
import { promisify } from 'util';
import concatStream from 'concat-stream';
import {
ReadTreeResponse,
ReadTreeResponseFile,
ReadTreeResponseDirOptions,
} from '../types';
// Tar types for `Parse` is not a proper constructor, but it should be
const TarParseStream = (Parse as unknown) as { new (): ParseStream };
const pipeline = promisify(pipelineCb);
/**
* Wraps a tar archive stream into a tree response reader.
*/
export class ArchiveResponse implements ReadTreeResponse {
private read = false;
constructor(
private readonly stream: Readable,
private readonly subPath: string,
private readonly workDir: string,
private readonly filter?: (path: string) => boolean,
) {
if (subPath) {
if (!subPath.endsWith('/')) {
this.subPath += '/';
}
if (subPath.startsWith('/')) {
throw new TypeError(
`ArchiveResponse subPath must not start with a /, got '${subPath}'`,
);
}
}
}
// Make sure the input stream is only read once
private onlyOnce() {
if (this.read) {
throw new Error('Response has already been read');
}
this.read = true;
}
async files(): Promise<ReadTreeResponseFile[]> {
this.onlyOnce();
const files = Array<ReadTreeResponseFile>();
const parser = new TarParseStream();
parser.on('entry', (entry: ReadEntry & Readable) => {
if (entry.type === 'Directory') {
entry.resume();
return;
}
if (this.subPath) {
if (!entry.path.startsWith(this.subPath)) {
entry.resume();
return;
}
}
const path = entry.path.slice(this.subPath.length);
if (this.filter) {
if (!this.filter(path)) {
entry.resume();
return;
}
}
const content = new Promise<Buffer>(async resolve => {
await pipeline(entry, concatStream(resolve));
});
files.push({ path, content: () => content });
entry.resume();
});
await pipeline(this.stream, parser);
return files;
}
async archive(): Promise<Readable> {
if (!this.subPath) {
this.onlyOnce();
return this.stream;
}
// TODO(Rugvip): method for repacking a tar with a subpath is to simply extract into a
// tmp dir and recreate the archive. Would be nicer to stream things instead.
const tmpDir = await this.dir();
try {
const data = await new Promise<Buffer>(async resolve => {
await pipeline(
tar.create({ cwd: tmpDir }, ['']),
concatStream(resolve),
);
});
return Readable.from(data);
} finally {
await fs.remove(tmpDir);
}
}
async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
this.onlyOnce();
const dir =
options?.targetDir ??
(await fs.mkdtemp(path.join(this.workDir, 'backstage-')));
const strip = this.subPath ? this.subPath.split('/').length - 1 : 0;
await pipeline(
this.stream,
tar.extract({
strip,
cwd: dir,
filter: path => {
if (this.subPath && !path.startsWith(this.subPath)) {
return false;
}
if (this.filter) {
const innerPath = path.split('/').slice(strip).join('/');
return this.filter(innerPath);
}
return true;
},
}),
);
return dir;
}
}
packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts
+50
View File
@@ -0,0 +1,50 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import { Readable } from 'stream';
import { Config } from '@backstage/config';
import { ReadTreeResponse } from '../types';
import { ArchiveResponse } from './ArchiveResponse';
type FromArchiveOptions = {
// A binary stream of a tar archive.
stream: Readable;
// If set, the root of the tree will be set to the given directory path.
path?: string;
// Filter passed on from the ReadTreeOptions
filter?: (path: string) => boolean;
};
export class ReadTreeResponseFactory {
static create(options: { config: Config }): ReadTreeResponseFactory {
return new ReadTreeResponseFactory(
options.config.getOptionalString('backend.workingDirectory') ??
os.tmpdir(),
);
}
constructor(private readonly workDir: string) {}
async fromArchive(options: FromArchiveOptions): Promise<ReadTreeResponse> {
return new ArchiveResponse(
options.stream,
options.path ?? '',
this.workDir,
options.filter,
);
}
}
packages/backend-common/src/reading/tree/index.ts
+17
View File
@@ -0,0 +1,17 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { ReadTreeResponseFactory } from './ReadTreeResponseFactory';
packages/backend-common/src/reading/types.ts
+29 -10
View File
@@ -16,17 +16,30 @@
import { Logger } from 'winston';
import { Config } from '@backstage/config';
import { ReadTreeResponseFactory } from './tree';
export type ReadTreeOptions = {
/**
* A filter that can be used to select which files should be included.
*
* The path passed to the filter function is the relative path from the URL
* that the file tree is fetched from, without any leading '/'.
*
* For example, given the URL https://github.com/my/repo/tree/master/my-dir, a file
* at https://github.com/my/repo/blob/master/my-dir/my-subdir/my-file.txt will
* be represented as my-subdir/my-file.txt
*
* If no filter is provided all files are extracted.
*/
filter?(path: string): boolean;
};
/**
* A generic interface for fetching plain data from URLs.
*/
export type UrlReader = {
read(url: string): Promise<Buffer>;
readTree?(
repoUrl: string,
branchName: string,
paths: Array<string>,
): Promise<ReadTreeResponse>;
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
};
export type UrlReaderPredicateTuple = {
@@ -41,15 +54,21 @@ export type UrlReaderPredicateTuple = {
export type ReaderFactory = (options: {
config: Config;
logger: Logger;
treeResponseFactory: ReadTreeResponseFactory;
}) => UrlReaderPredicateTuple[];
export type File = {
export type ReadTreeResponseFile = {
path: string;
content(): Promise<Buffer>;
};
export type ReadTreeResponse = {
files(): File[];
archive(): Promise<Buffer>;
dir(outDir?: string): Promise<string>;
export type ReadTreeResponseDirOptions = {
/** The directory to write files to. Defaults to the OS tmpdir or `backend.workingDirectory` if set in config */
targetDir?: string;
};
export type ReadTreeResponse = {
files(): Promise<ReadTreeResponseFile[]>;
archive(): Promise<NodeJS.ReadableStream>;
dir(options?: ReadTreeResponseDirOptions): Promise<string>;
};
plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts
+6 -6
View File
@@ -157,14 +157,14 @@ describe('CodeOwnersProcessor', () => {
const read = jest
.fn()
.mockResolvedValue(mockReadResult({ data: ownersText }));
const reader = { read };
const reader = { read, readTree: jest.fn() };
const result = await findRawCodeOwners(mockLocation(), reader);
expect(result).toEqual(ownersText);
});
it('should raise error when no codeowner', async () => {
const read = jest.fn().mockRejectedValue(mockReadResult());
const reader = { read };
const reader = { read, readTree: jest.fn() };
await expect(
findRawCodeOwners(mockLocation(), reader),
@@ -178,7 +178,7 @@ describe('CodeOwnersProcessor', () => {
.mockImplementationOnce(() => mockReadResult({ error: 'foo' }))
.mockImplementationOnce(() => mockReadResult({ error: 'bar' }))
.mockResolvedValue(mockReadResult({ data: ownersText }));
const reader = { read };
const reader = { read, readTree: jest.fn() };
const result = await findRawCodeOwners(mockLocation(), reader);
@@ -197,7 +197,7 @@ describe('CodeOwnersProcessor', () => {
const read = jest
.fn()
.mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() }));
const reader = { read };
const reader = { read, readTree: jest.fn() };
const owner = await resolveCodeOwner(mockLocation(), reader);
expect(owner).toBe('backstage-core');
@@ -207,7 +207,7 @@ describe('CodeOwnersProcessor', () => {
const read = jest
.fn()
.mockImplementation(() => mockReadResult({ error: 'error: foo' }));
const reader = { read };
const reader = { read, readTree: jest.fn() };
await expect(
resolveCodeOwner(mockLocation(), reader),
@@ -221,7 +221,7 @@ describe('CodeOwnersProcessor', () => {
const read = jest
.fn()
.mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() }));
const reader = { read };
const reader = { read, readTree: jest.fn() };
const processor = new CodeOwnersProcessor({ reader });
return { entity, processor, read };
plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts
+1 -1
View File
@@ -27,7 +27,7 @@ import {
describe('PlaceholderProcessor', () => {
const read: jest.MockedFunction<ResolverRead> = jest.fn();
const reader: UrlReader = { read };
const reader: UrlReader = { read, readTree: jest.fn() };
beforeEach(() => {
jest.resetAllMocks();
plugins/catalog-backend/src/service/CatalogBuilder.test.ts
+4 -1
View File
@@ -40,7 +40,10 @@ const dummyEntityYaml = yaml.stringify(dummyEntity);
describe('CatalogBuilder', () => {
let db: Knex<any, unknown[]>;
const reader: jest.Mocked<UrlReader> = { read: jest.fn() };
const reader: jest.Mocked<UrlReader> = {
read: jest.fn(),
readTree: jest.fn(),
};
const env: CatalogEnvironment = {
logger: getVoidLogger(),
database: { getClient: async () => db },
plugins/techdocs-backend/src/helpers.test.ts
+6 -53
View File
@@ -14,12 +14,13 @@
* limitations under the License.
*/
import { Readable } from 'stream';
import { getDocFilesFromRepository } from './helpers';
import { UrlReader, ReadTreeResponse } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
describe('getDocFilesFromRepository', () => {
it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in root', async () => {
it('should read a remote directory using UrlReader.readTree', async () => {
class MockUrlReader implements UrlReader {
async read() {
return Buffer.from('mock');
@@ -30,11 +31,11 @@ describe('getDocFilesFromRepository', () => {
dir: async () => {
return '/tmp/testfolder';
},
files: () => {
files: async () => {
return [];
},
archive: async () => {
return Buffer.from('');
return Readable.from('');
},
};
}
@@ -45,7 +46,7 @@ describe('getDocFilesFromRepository', () => {
namespace: 'default',
annotations: {
'backstage.io/techdocs-ref':
'url:https://github.com/backstage/backstage/blob/master/mkdocs.yml',
'url:https://github.com/backstage/backstage/blob/master/subfolder/',
},
name: 'mytestcomponent',
description: 'A component for testing',
@@ -64,54 +65,6 @@ describe('getDocFilesFromRepository', () => {
mockEntity,
);
expect(output).toBe('/tmp/testfolder/.');
});
it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in a subfolder', async () => {
class MockUrlReader implements UrlReader {
async read() {
return Buffer.from('mock');
}
async readTree(): Promise<ReadTreeResponse> {
return {
dir: async () => {
return '/tmp/testfolder';
},
files: () => {
return [];
},
archive: async () => {
return Buffer.from('');
},
};
}
}
const mockEntity: Entity = {
metadata: {
namespace: 'default',
annotations: {
'backstage.io/techdocs-ref':
'url:https://github.com/backstage/backstage/blob/master/subfolder/mkdocs.yml',
},
name: 'mytestcomponent',
description: 'A component for testing',
},
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
spec: {
type: 'documentation',
lifecycle: 'experimental',
owner: 'testuser',
},
};
const output = await getDocFilesFromRepository(
new MockUrlReader(),
mockEntity,
);
expect(output).toBe('/tmp/testfolder/subfolder');
expect(output).toBe('/tmp/testfolder');
});
});
plugins/techdocs-backend/src/helpers.ts
+2 -19
View File
@@ -179,24 +179,7 @@ export const getDocFilesFromRepository = async (
entity,
);
const { ref, filepath: mkdocsPath } = parseGitUrl(target);
const response = await reader.readTree(target);
const docsRootPath = path.dirname(mkdocsPath);
const docsFolderPath = path.join(docsRootPath, 'docs');
if (reader.readTree) {
const readTreeResponse = await reader.readTree(
parseGitUrl(target).toString(),
ref,
[mkdocsPath, docsFolderPath],
);
const tmpDir = await readTreeResponse.dir();
return `${tmpDir}/${docsRootPath}`;
}
throw new Error(
`No readTree method available on the UrlReader for ${target}`,
);
return await response.dir();
};