paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/spotify/backstage into lintMod

Browse Source
This commit is contained in:
Debajyoti Halder
2021-01-29 14:05:27 +05:30
parent c9d6493bef 373b620f54
commit 4dad818e01
244 changed files with 4425 additions and 1286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend-common/CHANGELOG.md
+30
View File
@@ -1,5 +1,35 @@
# @backstage/backend-common
## 0.5.1
### Patch Changes
- 26a3a6cf0: Honor the branch ref in the url when cloning.
This fixes a bug in the scaffolder prepare stage where a non-default branch
was specified in the scaffolder URL but the default branch was cloned.
For example, even though the `other` branch is specified in this example, the
`master` branch was actually cloned:
```yaml
catalog:
locations:
- type: url
target: https://github.com/backstage/backstage/blob/other/plugins/scaffolder-backend/sample-templates/docs-template/template.yaml
```
This also fixes a 404 in the prepare stage for GitLab URLs.
- 664dd08c9: URL Reader's readTree: Fix bug with github.com URLs.
- 9dd057662: Upgrade [git-url-parse](https://www.npmjs.com/package/git-url-parse) to [v11.4.4](https://github.com/IonicaBizau/git-url-parse/pull/125) which fixes parsing an Azure DevOps branch ref.
- Updated dependencies [6800da78d]
- Updated dependencies [9dd057662]
- Updated dependencies [ef7957be4]
- Updated dependencies [ef7957be4]
- Updated dependencies [ef7957be4]
- @backstage/integration@0.3.1
- @backstage/config-loader@0.5.0
## 0.5.0
### Minor Changes
packages/backend-common/package.json
+5 -5
View File
@@ -1,7 +1,7 @@
{
"name": "@backstage/backend-common",
"description": "Common functionality library for Backstage backends",
"version": "0.5.0",
"version": "0.5.1",
"main": "src/index.ts",
"types": "src/index.ts",
"private": false,
@@ -31,8 +31,8 @@
"dependencies": {
"@backstage/cli-common": "^0.1.1",
"@backstage/config": "^0.1.2",
"@backstage/config-loader": "^0.4.1",
"@backstage/integration": "^0.3.0",
"@backstage/config-loader": "^0.5.0",
"@backstage/integration": "^0.3.1",
"@types/cors": "^2.8.6",
"@types/express": "^4.17.6",
"archiver": "^5.0.2",
@@ -43,7 +43,7 @@
"express": "^4.17.1",
"express-promise-router": "^3.0.3",
"fs-extra": "^9.0.1",
"git-url-parse": "^11.4.3",
"git-url-parse": "^11.4.4",
"helmet": "^4.0.0",
"isomorphic-git": "^1.8.0",
"knex": "^0.21.6",
@@ -66,7 +66,7 @@
}
},
"devDependencies": {
"@backstage/cli": "^0.4.7",
"@backstage/cli": "^0.5.0",
"@backstage/test-utils": "^0.1.5",
"@types/archiver": "^5.1.0",
"@types/compression": "^1.7.0",
packages/backend-common/src/reading/AzureUrlReader.test.ts
+30 -1
View File
@@ -14,7 +14,9 @@
* limitations under the License.
*/
import fs from 'fs';
import * as os from 'os';
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import path from 'path';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
@@ -31,6 +33,8 @@ const treeResponseFactory = ReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
describe('AzureUrlReader', () => {
const worker = setupServer();
msw.setupDefaultHandlers(worker);
@@ -139,6 +143,16 @@ describe('AzureUrlReader', () => {
});
describe('readTree', () => {
beforeEach(() => {
mockFs({
[tmpDir]: mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const repoBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'mock-main.zip'),
);
@@ -200,6 +214,21 @@ describe('AzureUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('creates a directory with the wanted files', async () => {
const response = await processor.readTree(
'https://dev.azure.com/organization/project/_git/repository',
);
const dir = await response.dir({ targetDir: tmpDir });
await expect(
fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(path.join(dir, 'docs', 'index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnAzure = async () => {
await processor.readTree(
packages/backend-common/src/reading/BitbucketUrlReader.test.ts
+41 -3
View File
@@ -16,7 +16,8 @@
import { ConfigReader } from '@backstage/config';
import { msw } from '@backstage/test-utils';
import fs from 'fs';
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import path from 'path';
@@ -53,6 +54,16 @@ describe('BitbucketUrlReader', () => {
});
describe('readTree', () => {
beforeEach(() => {
mockFs({
'/tmp': mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const worker = setupServer();
msw.setupDefaultHandlers(worker);
@@ -126,12 +137,12 @@ describe('BitbucketUrlReader', () => {
),
),
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/repositories/backstage/mock/commits/some-branch',
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/commits',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
values: [{ id: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
@@ -155,6 +166,21 @@ describe('BitbucketUrlReader', () => {
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
});
it('creates a directory with the wanted files', async () => {
const response = await bitbucketProcessor.readTree(
'https://bitbucket.org/backstage/mock',
);
const dir = await response.dir({ targetDir: '/tmp' });
await expect(
fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(path.join(dir, 'docs', 'index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('uses private bitbucket host', async () => {
const response = await hostedBitbucketProcessor.readTree(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch',
@@ -185,6 +211,18 @@ describe('BitbucketUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('creates a directory with the wanted files with a subpath', async () => {
const response = await bitbucketProcessor.readTree(
'https://bitbucket.org/backstage/mock/src/master/docs',
);
const dir = await response.dir({ targetDir: '/tmp' });
await expect(
fs.readFile(path.join(dir, 'index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnBitbucket = async () => {
await bitbucketProcessor.readTree(
packages/backend-common/src/reading/BitbucketUrlReader.ts
+27 -32
View File
@@ -121,31 +121,9 @@ export class BitbucketUrlReader implements UrlReader {
throw new Error(message);
}
// Get the filename of archive from the header of the response
const contentDispositionHeader = archiveBitbucketResponse.headers.get(
'content-disposition',
) as string;
if (!contentDispositionHeader) {
throw new Error(
`Failed to read tree from ${url}. ` +
'Bitbucket API response for downloading archive does not contain content-disposition header ',
);
}
const fileNameRegEx = new RegExp(
/^attachment; filename=(?<fileName>.*).zip$/,
);
const archiveFileName = contentDispositionHeader.match(fileNameRegEx)
?.groups?.fileName;
if (!archiveFileName) {
throw new Error(
`Failed to read tree from ${url}. Bitbucket API response for downloading archive has an unexpected ` +
`format of content-disposition header ${contentDispositionHeader} `,
);
}
return await this.treeResponseFactory.fromZipArchive({
stream: (archiveBitbucketResponse.body as unknown) as Readable,
path: `${archiveFileName}/${filepath}`,
subpath: filepath,
etag: lastCommitShortHash,
filter: options?.filter,
});
@@ -161,13 +139,18 @@ export class BitbucketUrlReader implements UrlReader {
}
private async getLastCommitShortHash(url: string): Promise<string> {
const { name: repoName, owner: project, ref } = parseGitUrl(url);
const { resource, name: repoName, owner: project, ref } = parseGitUrl(url);
let branch = ref;
if (!branch) {
branch = await getBitbucketDefaultBranch(url, this.config);
}
const commitsApiUrl = `${this.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`;
const isHosted = resource === 'bitbucket.org';
// Bitbucket Server https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp222
const commitsApiUrl = isHosted
? `${this.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`
: `${this.config.apiBaseUrl}/projects/${project}/repos/${repoName}/commits`;
const commitsResponse = await fetch(
commitsApiUrl,
@@ -182,14 +165,26 @@ export class BitbucketUrlReader implements UrlReader {
}
const commits = await commitsResponse.json();
if (
commits &&
commits.values &&
commits.values.length > 0 &&
commits.values[0].hash
) {
return commits.values[0].hash.substring(0, 12);
if (isHosted) {
if (
commits &&
commits.values &&
commits.values.length > 0 &&
commits.values[0].hash
) {
return commits.values[0].hash.substring(0, 12);
}
} else {
if (
commits &&
commits.values &&
commits.values.length > 0 &&
commits.values[0].id
) {
return commits.values[0].id.substring(0, 12);
}
}
throw new Error(`Failed to read response from ${commitsApiUrl}`);
}
}
packages/backend-common/src/reading/GithubUrlReader.test.ts
+39 -1
View File
@@ -17,7 +17,8 @@
import { ConfigReader } from '@backstage/config';
import { GithubCredentialsProvider } from '@backstage/integration';
import { msw } from '@backstage/test-utils';
import fs from 'fs';
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import path from 'path';
@@ -107,6 +108,16 @@ describe('GithubUrlReader', () => {
});
describe('readTree', () => {
beforeEach(() => {
mockFs({
'/tmp': mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const repoBuffer = fs.readFileSync(
path.resolve(
'src',
@@ -227,6 +238,21 @@ describe('GithubUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('creates a directory with the wanted files', async () => {
const response = await githubProcessor.readTree(
'https://github.com/backstage/mock',
);
const dir = await response.dir({ targetDir: '/tmp' });
await expect(
fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(path.join(dir, 'docs', 'index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('should use the headers from the credentials provider to the fetch request', async () => {
expect.assertions(2);
@@ -293,6 +319,18 @@ describe('GithubUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('creates a directory with the wanted files with subpath', async () => {
const response = await githubProcessor.readTree(
'https://github.com/backstage/mock/tree/main/docs',
);
const dir = await response.dir({ targetDir: '/tmp' });
await expect(
fs.readFile(path.join(dir, 'index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnGithub = async () => {
await githubProcessor.readTree('https://github.com/backstage/mock', {
packages/backend-common/src/reading/GithubUrlReader.ts
+1 -27
View File
@@ -166,37 +166,11 @@ export class GithubUrlReader implements UrlReader {
throw new Error(message);
}
// Get the filename of archive from the header of the response
const contentDispositionHeader = archive.headers.get(
'content-disposition',
) as string;
if (!contentDispositionHeader) {
throw new Error(
`Failed to read tree from ${url}. ` +
'GitHub API response for downloading archive does not contain content-disposition header ',
);
}
const fileNameRegEx = new RegExp(
/^attachment; filename=(?<fileName>.*).tar.gz$/,
);
const archiveFileName = contentDispositionHeader.match(fileNameRegEx)
?.groups?.fileName;
if (!archiveFileName) {
throw new Error(
`Failed to read tree from ${url}. GitHub API response for downloading archive has an unexpected ` +
`format of content-disposition header ${contentDispositionHeader} `,
);
}
// The path includes the name of the directory inside the tarball and a sub path
// if requested in readTree.
const path = `${archiveFileName}/${filepath}`;
return await this.deps.treeResponseFactory.fromTarArchive({
// TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want
// to stick to using that in exclusively backend code.
stream: (archive.body as unknown) as Readable,
path,
subpath: filepath,
etag: commitSha,
filter: options?.filter,
});
packages/backend-common/src/reading/GitlabUrlReader.test.ts
+39 -1
View File
@@ -16,7 +16,8 @@
import { ConfigReader } from '@backstage/config';
import { msw } from '@backstage/test-utils';
import fs from 'fs';
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import path from 'path';
@@ -153,6 +154,16 @@ describe('GitlabUrlReader', () => {
});
describe('readTree', () => {
beforeEach(() => {
mockFs({
'/tmp': mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const archiveBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.zip'),
);
@@ -254,6 +265,21 @@ describe('GitlabUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('creates a directory with the wanted files', async () => {
const response = await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock',
);
const dir = await response.dir({ targetDir: '/tmp' });
await expect(
fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(path.join(dir, 'docs', 'index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('returns the wanted files from hosted gitlab', async () => {
worker.use(
rest.get(
@@ -296,6 +322,18 @@ describe('GitlabUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('creates a directory with the wanted files with subpath', async () => {
const response = await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock/tree/main/docs',
);
const dir = await response.dir({ targetDir: '/tmp' });
await expect(
fs.readFile(path.join(dir, 'index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnGitlab = async () => {
await gitlabProcessor.readTree('https://gitlab.com/backstage/mock', {
packages/backend-common/src/reading/GitlabUrlReader.ts
+1 -25
View File
@@ -140,33 +140,9 @@ export class GitlabUrlReader implements UrlReader {
throw new Error(message);
}
// Get the filename of archive from the header of the response
const contentDispositionHeader = archiveGitLabResponse.headers.get(
'content-disposition',
) as string;
if (!contentDispositionHeader) {
throw new Error(
`Failed to read tree from ${url}. ` +
'GitLab API response for downloading archive does not contain content-disposition header ',
);
}
const fileNameRegEx = new RegExp(
/^attachment; filename="(?<fileName>.*).zip"$/,
);
const archiveFileName = contentDispositionHeader.match(fileNameRegEx)
?.groups?.fileName;
if (!archiveFileName) {
throw new Error(
`Failed to read tree from ${url}. GitLab API response for downloading archive has an unexpected ` +
`format of content-disposition header ${contentDispositionHeader} `,
);
}
const path = filepath ? `${archiveFileName}/${filepath}/` : '';
return await this.treeResponseFactory.fromZipArchive({
stream: (archiveGitLabResponse.body as unknown) as Readable,
path,
subpath: filepath,
etag: commitSha,
filter: options?.filter,
});
packages/backend-common/src/reading/integration.test.ts
+155
View File
@@ -0,0 +1,155 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { getVoidLogger } from '../logging';
import { UrlReaders } from './UrlReaders';
const reader = UrlReaders.default({
logger: getVoidLogger(),
config: new ConfigReader({
// The tokens in this config provide read only access to the backstage-verification repos
integrations: {
github: [
{
host: 'github.com',
token: `${86}af${617}d9c3c8bf958b37a${630691452765}bb0b0a`,
},
],
gitlab: [
{
host: 'gitlab.com',
token: 'tveGtSHDBJM9ZRHZNRfm',
},
],
bitbucket: [
{
host: 'bitbucket.org',
username: 'backstage-verification',
appPassword: 'H79MAAhtbZwCafkVTrrQ',
},
],
azure: [
{
host: 'dev.azure.com',
// lasts until 2022-01-27
token: 'bhs5cbukiuxrkc3ftuyt5h3eqewtkj37lmf3jx5aoajivq3f5jmq',
},
],
},
}),
});
function withRetries(count: number, fn: () => Promise<void>) {
return async () => {
let error;
for (let i = 0; i < count; i++) {
try {
await fn();
return;
} catch (err) {
error = err;
}
}
throw error;
};
}
describe('UrlReaders', () => {
it(
'should read data from azure',
withRetries(3, async () => {
const data = await reader.read(
'https://dev.azure.com/backstage-verification/test-templates/_git/test-templates?path=%2Ftemplate.yaml',
);
expect(data.toString()).toContain('test-template-azure');
const res = await reader.readTree(
'https://dev.azure.com/backstage-verification/test-templates/_git/test-templates?path=%2F{{cookiecutter.name}}',
);
const files = await res.files();
expect(files).toEqual([
{
path: 'catalog-info.yaml',
content: expect.any(Function),
},
]);
}),
);
it(
'should read data from gitlab',
withRetries(3, async () => {
const data = await reader.read(
'https://gitlab.com/backstage-verification/test-templates/-/blob/master/template.yaml',
);
expect(data.toString()).toContain('test-template-gitlab');
const res = await reader.readTree(
'https://gitlab.com/backstage-verification/test-templates/-/tree/master/{{cookiecutter.name}}',
);
const files = await res.files();
expect(files).toEqual([
{
path: 'catalog-info.yaml',
content: expect.any(Function),
},
]);
}),
);
it(
'should read data from bitbucket',
withRetries(3, async () => {
const data = await reader.read(
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
);
expect(data.toString()).toContain('test-template-bitbucket');
const res = await reader.readTree(
'https://bitbucket.org/backstage-verification/test-template/src/master/{{cookiecutter.name}}',
);
const files = await res.files();
expect(files).toEqual([
{
path: 'catalog-info.yaml',
content: expect.any(Function),
},
]);
}),
);
it(
'should read data from github',
withRetries(3, async () => {
const data = await reader.read(
'https://github.com/backstage-verification/test-templates/blob/master/template.yaml',
);
expect(data.toString()).toContain('test-template-github');
const res = await reader.readTree(
'https://github.com/backstage-verification/test-templates/tree/master/{{cookiecutter.name}}',
);
const files = await res.files();
expect(files).toEqual([
{
path: 'catalog-info.yaml',
content: expect.any(Function),
},
]);
}),
);
});
packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts
+5 -4
View File
@@ -24,8 +24,9 @@ import { ZipArchiveResponse } from './ZipArchiveResponse';
type FromArchiveOptions = {
// A binary stream of a tar archive.
stream: Readable;
// If set, the root of the tree will be set to the given directory path.
path?: string;
// If unset, the files at the root of the tree will be read.
// subpath must not contain the name of the top level directory.
subpath?: string;
// etag of the blob
etag: string;
// Filter passed on from the ReadTreeOptions
@@ -45,7 +46,7 @@ export class ReadTreeResponseFactory {
async fromTarArchive(options: FromArchiveOptions): Promise<ReadTreeResponse> {
return new TarArchiveResponse(
options.stream,
options.path ?? '',
options.subpath ?? '',
this.workDir,
options.etag,
options.filter,
@@ -55,7 +56,7 @@ export class ReadTreeResponseFactory {
async fromZipArchive(options: FromArchiveOptions): Promise<ReadTreeResponse> {
return new ZipArchiveResponse(
options.stream,
options.path ?? '',
options.subpath ?? '',
this.workDir,
options.etag,
options.filter,
packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts
+9 -23
View File
@@ -38,7 +38,7 @@ describe('TarArchiveResponse', () => {
it('should read files', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const res = new TarArchiveResponse(stream, '', '/tmp', 'etag');
const files = await res.files();
expect(files).toEqual([
@@ -61,12 +61,8 @@ describe('TarArchiveResponse', () => {
it('should read files with filter', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
const res = new TarArchiveResponse(stream, '', '/tmp', 'etag', path =>
path.endsWith('.yml'),
);
const files = await res.files();
@@ -83,7 +79,7 @@ describe('TarArchiveResponse', () => {
it('should read as archive and files', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const res = new TarArchiveResponse(stream, '', '/tmp', 'etag');
const buffer = await res.archive();
await expect(res.archive()).rejects.toThrow(
@@ -115,24 +111,18 @@ describe('TarArchiveResponse', () => {
const res = new TarArchiveResponse(stream, '', '/tmp', 'etag');
const dir = await res.dir();
await expect(
fs.readFile(resolvePath(dir, 'mock-main/mkdocs.yml'), 'utf8'),
fs.readFile(resolvePath(dir, 'mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(resolvePath(dir, 'mock-main/docs/index.md'), 'utf8'),
fs.readFile(resolvePath(dir, 'docs/index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('should extract archive into directory with a subpath', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(
stream,
'mock-main/docs/',
'/tmp',
'etag',
);
const res = new TarArchiveResponse(stream, 'docs', '/tmp', 'etag');
const dir = await res.dir();
expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/);
@@ -144,12 +134,8 @@ describe('TarArchiveResponse', () => {
it('should extract archive into directory with a subpath and filter', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
const res = new TarArchiveResponse(stream, '', '/tmp', 'etag', path =>
path.endsWith('.yml'),
);
const dir = await res.dir({ targetDir: '/tmp' });
packages/backend-common/src/reading/tree/TarArchiveResponse.ts
+21 -5
View File
@@ -30,6 +30,10 @@ import {
const TarParseStream = (Parse as unknown) as { new (): ParseStream };
const pipeline = promisify(pipelineCb);
// Matches a directory name + one `/` at the start of any string,
// containing any character except `/` one or more times, and ending with a `/`
// e.g. Will match `dirA/` in `dirA/dirB/file.ext`
const directoryNameRegex = /^[^\/]+\//;
/**
* Wraps a tar archive stream into a tree response reader.
@@ -78,14 +82,18 @@ export class TarArchiveResponse implements ReadTreeResponse {
return;
}
// File path relative to the root extracted directory. Will remove the
// top level dir name from the path since its name is hard to predetermine.
const relativePath = entry.path.replace(directoryNameRegex, '');
if (this.subPath) {
if (!entry.path.startsWith(this.subPath)) {
if (!relativePath.startsWith(this.subPath)) {
entry.resume();
return;
}
}
const path = entry.path.slice(this.subPath.length);
const path = relativePath.slice(this.subPath.length);
if (this.filter) {
if (!this.filter(path)) {
entry.resume();
@@ -97,7 +105,10 @@ export class TarArchiveResponse implements ReadTreeResponse {
await pipeline(entry, concatStream(resolve));
});
files.push({ path, content: () => content });
files.push({
path,
content: () => content,
});
entry.resume();
});
@@ -138,7 +149,9 @@ export class TarArchiveResponse implements ReadTreeResponse {
options?.targetDir ??
(await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-')));
const strip = this.subPath ? this.subPath.split('/').length - 1 : 0;
// Equivalent of tar --strip-components=N
// When no subPath is given, remove just 1 top level directory
const strip = this.subPath ? this.subPath.split('/').length : 1;
await pipeline(
this.stream,
@@ -146,7 +159,10 @@ export class TarArchiveResponse implements ReadTreeResponse {
strip,
cwd: dir,
filter: path => {
if (this.subPath && !path.startsWith(this.subPath)) {
// File path relative to the root extracted directory. Will remove the
// top level dir name from the path since its name is hard to predetermine.
const relativePath = path.replace(directoryNameRegex, '');
if (this.subPath && !relativePath.startsWith(this.subPath)) {
return false;
}
if (this.filter) {
packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts
+9 -22
View File
@@ -38,7 +38,7 @@ describe('ZipArchiveResponse', () => {
it('should read files', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag');
const files = await res.files();
expect(files).toEqual([
@@ -61,12 +61,8 @@ describe('ZipArchiveResponse', () => {
it('should read files with filter', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag', path =>
path.endsWith('.yml'),
);
const files = await res.files();
@@ -83,7 +79,7 @@ describe('ZipArchiveResponse', () => {
it('should read as archive and files', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag');
const buffer = await res.archive();
await expect(res.archive()).rejects.toThrow(
@@ -117,22 +113,17 @@ describe('ZipArchiveResponse', () => {
const dir = await res.dir();
await expect(
fs.readFile(resolvePath(dir, 'mock-main/mkdocs.yml'), 'utf8'),
fs.readFile(resolvePath(dir, 'mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(resolvePath(dir, 'mock-main/docs/index.md'), 'utf8'),
fs.readFile(resolvePath(dir, 'docs/index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('should extract archive into directory with a subpath', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(
stream,
'mock-main/docs/',
'/tmp',
'etag',
);
const res = new ZipArchiveResponse(stream, 'docs/', '/tmp', 'etag');
const dir = await res.dir();
expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/);
@@ -144,12 +135,8 @@ describe('ZipArchiveResponse', () => {
it('should extract archive into directory with a subpath and filter', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag', path =>
path.endsWith('.yml'),
);
const dir = await res.dir({ targetDir: '/tmp' });
packages/backend-common/src/reading/tree/ZipArchiveResponse.ts
+22 -7
View File
@@ -25,6 +25,11 @@ import {
ReadTreeResponseDirOptions,
} from '../types';
// Matches a directory name + one `/` at the start of any string,
// containing any character except / one or more times, and ending with a `/`
// e.g. Will match `dirA/` in `dirA/dirB/file.ext`
const directoryNameRegex = /^[^\/]+\//;
/**
* Wraps a zip archive stream into a tree response reader.
*/
@@ -60,18 +65,26 @@ export class ZipArchiveResponse implements ReadTreeResponse {
this.read = true;
}
private getPath(entry: Entry): string {
return entry.path.slice(this.subPath.length);
// Will remove the top level dir name from the path since its name is hard to predetermine.
private stripTopDirectory(path: string): string {
return path.replace(directoryNameRegex, '');
}
// File path relative to the root extracted directory or a sub directory if subpath is set.
private getInnerPath(path: string): string {
return path.slice(this.subPath.length);
}
private shouldBeIncluded(entry: Entry): boolean {
const strippedPath = this.stripTopDirectory(entry.path);
if (this.subPath) {
if (!entry.path.startsWith(this.subPath)) {
if (!strippedPath.startsWith(this.subPath)) {
return false;
}
}
if (this.filter) {
return this.filter(this.getPath(entry));
return this.filter(this.getInnerPath(entry.path));
}
return true;
}
@@ -91,7 +104,7 @@ export class ZipArchiveResponse implements ReadTreeResponse {
if (this.shouldBeIncluded(entry)) {
files.push({
path: this.getPath(entry),
path: this.getInnerPath(this.stripTopDirectory(entry.path)),
content: () => entry.buffer(),
});
} else {
@@ -115,7 +128,7 @@ export class ZipArchiveResponse implements ReadTreeResponse {
.pipe(unzipper.Parse())
.on('entry', (entry: Entry) => {
if (entry.type === 'File' && this.shouldBeIncluded(entry)) {
archive.append(entry, { name: this.getPath(entry) });
archive.append(entry, { name: this.getInnerPath(entry.path) });
} else {
entry.autodrain();
}
@@ -139,7 +152,9 @@ export class ZipArchiveResponse implements ReadTreeResponse {
// Ignore directory entries since we handle that with the file entries
// as a zip can have files with directories without directory entries
if (entry.type === 'File' && this.shouldBeIncluded(entry)) {
const entryPath = this.getPath(entry);
const entryPath = this.getInnerPath(
this.stripTopDirectory(entry.path),
);
const dirname = platformPath.dirname(entryPath);
if (dirname) {
await fs.mkdirp(platformPath.join(dir, dirname));
packages/backend-common/src/reading/types.ts
+3
View File
@@ -81,6 +81,9 @@ export type ReadTreeResponseDirOptions = {
};
export type ReadTreeResponse = {
/**
* files() returns an array of all the files inside the tree and corresponding functions to read their content.
*/
files(): Promise<ReadTreeResponseFile[]>;
archive(): Promise<NodeJS.ReadableStream>;
packages/backend-common/src/scm/git.ts
+10 -1
View File
@@ -86,13 +86,22 @@ export class Git {
return git.commit({ fs, dir, message, author, committer });
}
async clone({ url, dir }: { url: string; dir: string }): Promise<void> {
async clone({
url,
dir,
ref,
}: {
url: string;
dir: string;
ref?: string;
}): Promise<void> {
this.config.logger?.info(`Cloning repo {dir=${dir},url=${url}}`);
return git.clone({
fs,
http,
url,
dir,
ref,
singleBranch: true,
depth: 1,
onProgress: this.onProgressHandler(),