paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 31 Packages Projects Releases Wiki Activity

Merge pull request #4120 from backstage/orkohunter/url-reader-readTree-with-sha

Browse Source
This commit is contained in:
Himanshu Mishra
2021-01-20 14:49:47 +01:00
committed by GitHub
parent cb296b898c 662d9b9a22
commit e8e6bd92b6
33 changed files with 852 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend-common/src/errors.ts
+5
View File
@@ -75,3 +75,8 @@ export class NotFoundError extends CustomErrorBase {}
* resource.
*/
export class ConflictError extends CustomErrorBase {}
/**
* The requested resource has not changed since last request.
*/
export class NotModifiedError extends CustomErrorBase {}
packages/backend-common/src/middleware/errorHandler.test.ts
+4
View File
@@ -72,6 +72,9 @@ describe('errorHandler', () => {
it('handles well-known error classes', async () => {
const app = express();
app.use('/NotModifiedError', () => {
throw new errors.NotModifiedError();
});
app.use('/InputError', () => {
throw new errors.InputError();
});
@@ -90,6 +93,7 @@ describe('errorHandler', () => {
app.use(errorHandler());
const r = request(app);
expect((await r.get('/NotModifiedError')).status).toBe(304);
expect((await r.get('/InputError')).status).toBe(400);
expect((await r.get('/AuthenticationError')).status).toBe(401);
expect((await r.get('/NotAllowedError')).status).toBe(403);
packages/backend-common/src/middleware/errorHandler.ts
+2
View File
@@ -101,6 +101,8 @@ function getStatusCode(error: Error): number {
// Handle well-known error types
switch (error.name) {
case errors.NotModifiedError.name:
return 304;
case errors.InputError.name:
return 400;
case errors.AuthenticationError.name:
packages/backend-common/src/reading/AzureUrlReader.test.ts
+60 -8
View File
@@ -23,6 +23,7 @@ import { getVoidLogger } from '../logging';
import { AzureUrlReader } from './AzureUrlReader';
import { msw } from '@backstage/test-utils';
import { ReadTreeResponseFactory } from './tree';
import { NotModifiedError } from '../errors';
const logger = getVoidLogger();
@@ -139,7 +140,12 @@ describe('AzureUrlReader', () => {
describe('readTree', () => {
const repoBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'repo.zip'),
path.resolve('src', 'reading', '__fixtures__', 'mock-main.zip'),
);
const processor = new AzureUrlReader(
{ host: 'dev.azure.com' },
{ treeResponseFactory },
);
beforeEach(() => {
@@ -153,24 +159,70 @@ describe('AzureUrlReader', () => {
ctx.body(repoBuffer),
),
),
rest.get(
// https://docs.microsoft.com/en-us/rest/api/azure/devops/git/commits/get%20commits?view=azure-devops-rest-6.0#on-a-branch
'https://dev.azure.com/organization/project/_apis/git/repositories/repository/commits',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
count: 2,
value: [
{
commitId: '123abc2',
comment: 'second commit',
},
{
commitId: '123abc1',
comment: 'first commit',
},
],
}),
),
),
);
});
it('returns the wanted files from an archive', async () => {
const processor = new AzureUrlReader(
{ host: 'dev.azure.com' },
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://dev.azure.com/organization/project/_git/repository',
);
expect(response.etag).toBe('123abc2');
const files = await response.files();
expect(files.length).toBe(2);
const mkDocsFile = await files[1].content();
const indexMarkdownFile = await files[0].content();
const mkDocsFile = await files[0].content();
const indexMarkdownFile = await files[1].content();
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnAzure = async () => {
await processor.readTree(
'https://dev.azure.com/organization/project/_git/repository',
{ etag: '123abc2' },
);
};
await expect(fnAzure).rejects.toThrow(NotModifiedError);
});
it('should not throw a NotModifiedError when given an outdated etag in options', async () => {
const response = await processor.readTree(
'https://dev.azure.com/organization/project/_git/repository',
{ etag: 'outdated123abc' },
);
expect(response.etag).toBe('123abc2');
const files = await response.files();
expect(files.length).toBe(2);
const mkDocsFile = await files[0].content();
const indexMarkdownFile = await files[1].content();
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
expect(indexMarkdownFile.toString()).toBe('# Test\n');
packages/backend-common/src/reading/AzureUrlReader.ts
+30 -9
View File
@@ -20,10 +20,11 @@ import {
getAzureFileFetchUrl,
getAzureDownloadUrl,
getAzureRequestOptions,
getAzureCommitsUrl,
} from '@backstage/integration';
import fetch from 'cross-fetch';
import { Readable } from 'stream';
import { NotFoundError } from '../errors';
import { NotFoundError, NotModifiedError } from '../errors';
import {
ReaderFactory,
ReadTreeOptions,
@@ -75,20 +76,40 @@ export class AzureUrlReader implements UrlReader {
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const response = await fetch(
getAzureDownloadUrl(url),
getAzureRequestOptions(this.options, { Accept: 'application/zip' }),
// Get latest commit SHA
const commitsAzureResponse = await fetch(
getAzureCommitsUrl(url),
getAzureRequestOptions(this.options),
);
if (!response.ok) {
const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
if (!commitsAzureResponse.ok) {
const message = `Failed to read tree from ${url}, ${commitsAzureResponse.status} ${commitsAzureResponse.statusText}`;
if (commitsAzureResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
return this.deps.treeResponseFactory.fromZipArchive({
stream: (response.body as unknown) as Readable,
const commitSha = (await commitsAzureResponse.json()).value[0].commitId;
if (options?.etag && options.etag === commitSha) {
throw new NotModifiedError();
}
const archiveAzureResponse = await fetch(
getAzureDownloadUrl(url),
getAzureRequestOptions(this.options, { Accept: 'application/zip' }),
);
if (!archiveAzureResponse.ok) {
const message = `Failed to read tree from ${url}, ${archiveAzureResponse.status} ${archiveAzureResponse.statusText}`;
if (archiveAzureResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
return await this.deps.treeResponseFactory.fromZipArchive({
stream: (archiveAzureResponse.body as unknown) as Readable,
etag: commitSha,
filter: options?.filter,
});
}
packages/backend-common/src/reading/BitbucketUrlReader.test.ts
+88 -69
View File
@@ -20,6 +20,7 @@ import fs from 'fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import path from 'path';
import { NotModifiedError } from '../errors';
import { BitbucketUrlReader } from './BitbucketUrlReader';
import { ReadTreeResponseFactory } from './tree';
@@ -27,15 +28,24 @@ const treeResponseFactory = ReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
const bitbucketProcessor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
{ treeResponseFactory },
);
const hostedBitbucketProcessor = new BitbucketUrlReader(
{
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
},
{ treeResponseFactory },
);
describe('BitbucketUrlReader', () => {
describe('implementation', () => {
it('rejects unknown targets', async () => {
const processor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
{ treeResponseFactory },
);
await expect(
processor.read('https://not.bitbucket.com/apa'),
bitbucketProcessor.read('https://not.bitbucket.com/apa'),
).rejects.toThrow(
'Incorrect URL: https://not.bitbucket.com/apa, Error: Invalid Bitbucket URL or file path',
);
@@ -55,8 +65,30 @@ describe('BitbucketUrlReader', () => {
),
);
it('returns the wanted files from an archive', async () => {
const privateBitbucketRepoBuffer = fs.readFileSync(
path.resolve(
'src',
'reading',
'__fixtures__',
'bitbucket-server-repo.zip',
),
);
beforeEach(() => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
mainbranch: {
type: 'branch',
name: 'master',
},
}),
),
),
rest.get(
'https://bitbucket.org/backstage/mock/get/master.zip',
(_, res, ctx) =>
@@ -76,17 +108,35 @@ describe('BitbucketUrlReader', () => {
}),
),
),
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(privateBitbucketRepoBuffer),
),
),
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/repositories/backstage/mock/commits/some-branch',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
});
const processor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
{ treeResponseFactory },
);
const response = await processor.readTree(
it('returns the wanted files from an archive', async () => {
const response = await bitbucketProcessor.readTree(
'https://bitbucket.org/backstage/mock/src/master',
);
expect(response.etag).toBe('12ab34cd56ef');
const files = await response.files();
expect(files.length).toBe(2);
@@ -98,38 +148,12 @@ describe('BitbucketUrlReader', () => {
});
it('uses private bitbucket host', async () => {
const privateBitbucketRepoBuffer = fs.readFileSync(
path.resolve(
'src',
'reading',
'__fixtures__',
'bitbucket-server-repo.zip',
),
);
worker.use(
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(privateBitbucketRepoBuffer),
),
),
);
const processor = new BitbucketUrlReader(
{
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
},
{ treeResponseFactory },
);
const response = await processor.readTree(
const response = await hostedBitbucketProcessor.readTree(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch',
);
expect(response.etag).toBe('12ab34cd56ef');
const files = await response.files();
expect(files.length).toBe(1);
@@ -139,37 +163,12 @@ describe('BitbucketUrlReader', () => {
});
it('returns the wanted files from an archive with a subpath', async () => {
worker.use(
rest.get(
'https://bitbucket.org/backstage/mock/get/master.zip',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(repoBuffer),
),
),
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
const processor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
{ treeResponseFactory },
);
const response = await processor.readTree(
const response = await bitbucketProcessor.readTree(
'https://bitbucket.org/backstage/mock/src/master/docs',
);
expect(response.etag).toBe('12ab34cd56ef');
const files = await response.files();
expect(files.length).toBe(1);
@@ -177,5 +176,25 @@ describe('BitbucketUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnBitbucket = async () => {
await bitbucketProcessor.readTree(
'https://bitbucket.org/backstage/mock',
{ etag: '12ab34cd56ef' },
);
};
await expect(fnBitbucket).rejects.toThrow(NotModifiedError);
});
it('should not throw a NotModifiedError when given an outdated etag in options', async () => {
const response = await bitbucketProcessor.readTree(
'https://bitbucket.org/backstage/mock',
{ etag: 'outdatedetag123abc' },
);
expect(response.etag).toBe('12ab34cd56ef');
});
});
});
packages/backend-common/src/reading/BitbucketUrlReader.ts
+14 -9
View File
@@ -25,7 +25,7 @@ import {
import fetch from 'cross-fetch';
import parseGitUrl from 'git-url-parse';
import { Readable } from 'stream';
import { NotFoundError } from '../errors';
import { NotFoundError, NotModifiedError } from '../errors';
import { ReadTreeResponseFactory } from './tree';
import {
ReaderFactory,
@@ -105,16 +105,21 @@ export class BitbucketUrlReader implements UrlReader {
url,
);
const lastCommitShortHash = await this.getLastCommitShortHash(url);
if (options?.etag && options.etag === lastCommitShortHash) {
throw new NotModifiedError();
}
const isHosted = resource === 'bitbucket.org';
const downloadUrl = await getBitbucketDownloadUrl(url, this.config);
const response = await fetch(
const archiveBitbucketResponse = await fetch(
downloadUrl,
getBitbucketRequestOptions(this.config),
);
if (!response.ok) {
const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
if (!archiveBitbucketResponse.ok) {
const message = `Failed to read tree from ${url}, ${archiveBitbucketResponse.status} ${archiveBitbucketResponse.statusText}`;
if (archiveBitbucketResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
@@ -122,13 +127,13 @@ export class BitbucketUrlReader implements UrlReader {
let folderPath = `${project}-${repoName}`;
if (isHosted) {
const lastCommitShortHash = await this.getLastCommitShortHash(url);
folderPath = `${project}-${repoName}-${lastCommitShortHash}`;
}
return this.treeResponseFactory.fromZipArchive({
stream: (response.body as unknown) as Readable,
return await this.treeResponseFactory.fromZipArchive({
stream: (archiveBitbucketResponse.body as unknown) as Readable,
path: `${folderPath}/${filepath}`,
etag: lastCommitShortHash,
filter: options?.filter,
});
}
@@ -142,7 +147,7 @@ export class BitbucketUrlReader implements UrlReader {
return `bitbucket{host=${host},authed=${authed}}`;
}
private async getLastCommitShortHash(url: string): Promise<String> {
private async getLastCommitShortHash(url: string): Promise<string> {
const { name: repoName, owner: project, ref } = parseGitUrl(url);
let branch = ref;
packages/backend-common/src/reading/GithubUrlReader.test.ts
+172 -105
View File
@@ -21,6 +21,7 @@ import fs from 'fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import path from 'path';
import { NotFoundError, NotModifiedError } from '../errors';
import { GithubUrlReader } from './GithubUrlReader';
import { ReadTreeResponseFactory } from './tree';
@@ -28,11 +29,27 @@ const treeResponseFactory = ReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
describe('GithubUrlReader', () => {
const mockCredentialsProvider = ({
getCredentials: jest.fn().mockResolvedValue({ headers: {} }),
} as unknown) as GithubCredentialsProvider;
const mockCredentialsProvider = ({
getCredentials: jest.fn().mockResolvedValue({ headers: {} }),
} as unknown) as GithubCredentialsProvider;
const githubProcessor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
const gheProcessor = new GithubUrlReader(
{
host: 'ghe.github.com',
apiBaseUrl: 'https://ghe.github.com/api/v3',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
describe('GithubUrlReader', () => {
const worker = setupServer();
msw.setupDefaultHandlers(worker);
@@ -43,15 +60,8 @@ describe('GithubUrlReader', () => {
describe('implementation', () => {
it('rejects unknown targets', async () => {
const processor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
await expect(
processor.read('https://not.github.com/apa'),
githubProcessor.read('https://not.github.com/apa'),
).rejects.toThrow(
'Incorrect URL: https://not.github.com/apa, Error: Invalid GitHub URL or file path',
);
@@ -73,7 +83,7 @@ describe('GithubUrlReader', () => {
worker.use(
rest.get(
'https://api.github.com/repos/backstage/mock/tree/contents/?ref=repo',
'https://api.github.com/repos/backstage/mock/tree/contents/?ref=main',
(req, res, ctx) => {
expect(req.headers.get('authorization')).toBe(
mockHeaders.Authorization,
@@ -90,28 +100,67 @@ describe('GithubUrlReader', () => {
),
);
const processor = new GithubUrlReader(
{
host: 'ghe.github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
await processor.read(
'https://ghe.github.com/backstage/mock/tree/blob/repo',
await githubProcessor.read(
'https://github.com/backstage/mock/tree/blob/main',
);
});
});
describe('readTree', () => {
const repoBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'repo.tar.gz'),
path.resolve(
'src',
'reading',
'__fixtures__',
'backstage-mock-etag123.tar.gz',
),
);
const reposGithubApiResponse = {
id: '123',
full_name: 'backstage/mock',
default_branch: 'main',
branches_url:
'https://api.github.com/repos/backstage/mock/branches{/branch}',
archive_url:
'https://api.github.com/repos/backstage/mock/{archive_format}{/ref}',
};
const reposGheApiResponse = {
...reposGithubApiResponse,
branches_url:
'https://ghe.github.com/api/v3/repos/backstage/mock/branches{/branch}',
archive_url:
'https://ghe.github.com/api/v3/repos/backstage/mock/{archive_format}{/ref}',
};
const branchesApiResponse = {
name: 'main',
commit: {
sha: 'etag123abc',
},
};
beforeEach(() => {
worker.use(
rest.get('https://api.github.com/repos/backstage/mock', (_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(reposGithubApiResponse),
),
),
rest.get(
'https://github.com/backstage/mock/archive/repo.tar.gz',
'https://api.github.com/repos/backstage/mock/branches/main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(branchesApiResponse),
),
),
rest.get(
'https://api.github.com/repos/backstage/mock/tarball/etag123abc',
(_, res, ctx) =>
res(
ctx.status(200),
@@ -119,21 +168,46 @@ describe('GithubUrlReader', () => {
ctx.body(repoBuffer),
),
),
rest.get(
'https://api.github.com/repos/backstage/mock/branches/branchDoesNotExist',
(_, res, ctx) => res(ctx.status(404)),
),
rest.get(
'https://ghe.github.com/api/v3/repos/backstage/mock/tarball/etag123abc',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/x-gzip'),
ctx.body(repoBuffer),
),
),
rest.get(
'https://ghe.github.com/api/v3/repos/backstage/mock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(reposGheApiResponse),
),
),
rest.get(
'https://ghe.github.com/api/v3/repos/backstage/mock/branches/main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(branchesApiResponse),
),
),
);
});
it('returns the wanted files from an archive', async () => {
const processor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
const response = await githubProcessor.readTree(
'https://github.com/backstage/mock/tree/main',
);
const response = await processor.readTree(
'https://github.com/backstage/mock/tree/repo',
);
expect(response.etag).toBe('etag123abc');
const files = await response.files();
@@ -145,40 +219,6 @@ describe('GithubUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('includes the subdomain in the github url', async () => {
worker.resetHandlers();
worker.use(
rest.get(
'https://ghe.github.com/backstage/mock/archive/repo.tar.gz',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/x-gzip'),
ctx.body(repoBuffer),
),
),
);
const processor = new GithubUrlReader(
{
host: 'ghe.github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
const response = await processor.readTree(
'https://ghe.github.com/backstage/mock/tree/repo/docs',
);
const files = await response.files();
expect(files.length).toBe(1);
const indexMarkdownFile = await files[0].content();
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('should use the headers from the credentials provider to the fetch request', async () => {
expect.assertions(2);
@@ -193,7 +233,7 @@ describe('GithubUrlReader', () => {
worker.use(
rest.get(
'https://ghe.github.com/backstage/mock/archive/repo.tar.gz',
'https://ghe.github.com/api/v3/repos/backstage/mock/tarball/etag123abc',
(req, res, ctx) => {
expect(req.headers.get('authorization')).toBe(
mockHeaders.Authorization,
@@ -210,46 +250,14 @@ describe('GithubUrlReader', () => {
),
);
const processor = new GithubUrlReader(
{
host: 'ghe.github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
await processor.readTree(
'https://ghe.github.com/backstage/mock/tree/repo/docs',
await gheProcessor.readTree(
'https://ghe.github.com/backstage/mock/tree/main',
);
});
it('must specify a branch', async () => {
const processor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
await expect(
processor.readTree('https://github.com/backstage/mock'),
).rejects.toThrow(
'GitHub URL must contain branch to be able to fetch tree',
);
});
it('returns the wanted files from an archive with a subpath', async () => {
const processor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
const response = await processor.readTree(
'https://github.com/backstage/mock/tree/repo/docs',
it('includes the subdomain in the github url', async () => {
const response = await gheProcessor.readTree(
'https://ghe.github.com/backstage/mock/tree/main/docs',
);
const files = await response.files();
@@ -259,5 +267,64 @@ describe('GithubUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('returns the wanted files from an archive with a subpath', async () => {
const response = await githubProcessor.readTree(
'https://github.com/backstage/mock/tree/main/docs',
);
const files = await response.files();
expect(files.length).toBe(1);
const indexMarkdownFile = await files[0].content();
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnGithub = async () => {
await githubProcessor.readTree('https://github.com/backstage/mock', {
etag: 'etag123abc',
});
};
const fnGhe = async () => {
await gheProcessor.readTree(
'https://ghe.github.com/backstage/mock/tree/main/docs',
{
etag: 'etag123abc',
},
);
};
await expect(fnGithub).rejects.toThrow(NotModifiedError);
await expect(fnGhe).rejects.toThrow(NotModifiedError);
});
it('should not throw error when given an outdated etag in options', async () => {
const response = await githubProcessor.readTree(
'https://github.com/backstage/mock/tree/main',
{
etag: 'outdatedetag123abc',
},
);
expect((await response.files()).length).toBe(2);
});
it('should detect the default branch', async () => {
const response = await githubProcessor.readTree(
'https://github.com/backstage/mock',
);
expect((await response.files()).length).toBe(2);
});
it('should throw error on missing branch', async () => {
const fnGithub = async () => {
await githubProcessor.readTree(
'https://github.com/backstage/mock/tree/branchDoesNotExist',
);
};
await expect(fnGithub).rejects.toThrow(NotFoundError);
});
});
});
packages/backend-common/src/reading/GithubUrlReader.ts
+72 -30
View File
@@ -23,7 +23,7 @@ import {
import fetch from 'cross-fetch';
import parseGitUrl from 'git-url-parse';
import { Readable } from 'stream';
import { InputError, NotFoundError } from '../errors';
import { NotFoundError, NotModifiedError } from '../errors';
import { ReadTreeResponseFactory } from './tree';
import {
ReaderFactory,
@@ -98,51 +98,93 @@ export class GithubUrlReader implements UrlReader {
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const {
name: repoName,
ref,
protocol,
resource,
full_name,
filepath,
} = parseGitUrl(url);
if (!ref) {
// TODO(Rugvip): We should add support for defaulting to the default branch
throw new InputError(
'GitHub URL must contain branch to be able to fetch tree',
);
}
const { ref, filepath, full_name } = parseGitUrl(url);
// Caveat: The ref will totally be incorrect if the branch name includes a /
// Thus, readTree can not work on url containing branch name that has a /
const { headers } = await this.deps.credentialsProvider.getCredentials({
url,
});
// TODO(Rugvip): use API to fetch URL instead
const response = await fetch(
new URL(
`${protocol}://${resource}/${full_name}/archive/${ref}.tar.gz`,
).toString(),
// Get GitHub API urls for the repository
const repoGitHubResponse = await fetch(
new URL(`${this.config.apiBaseUrl}/repos/${full_name}`).toString(),
{
headers: {
...headers,
},
headers,
},
);
if (!response.ok) {
const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
if (!repoGitHubResponse.ok) {
const message = `Failed to read tree (repository) from ${url}, ${repoGitHubResponse.status} ${repoGitHubResponse.statusText}`;
if (repoGitHubResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const path = `${repoName}-${ref}/${filepath}`;
const repoResponseJson = await repoGitHubResponse.json();
return this.deps.treeResponseFactory.fromTarArchive({
// ref is an empty string if no branch is set in provided url to readTree.
// Use GitHub API to get the default branch of the repository.
const branch = ref || repoResponseJson.default_branch;
const branchesApiUrl = repoResponseJson.branches_url;
const archiveApiUrl = repoResponseJson.archive_url;
// Fetch the latest commit in the provided or default branch to compare against
// the provided sha.
const branchGitHubResponse = await fetch(
// branchesApiUrl looks like "https://api.github.com/repos/owner/repo/branches{/branch}"
branchesApiUrl.replace('{/branch}', `/${branch}`),
{
headers,
},
);
if (!branchGitHubResponse.ok) {
const message = `Failed to read tree (branch) from ${url}, ${branchGitHubResponse.status} ${branchGitHubResponse.statusText}`;
if (branchGitHubResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const commitSha = (await branchGitHubResponse.json()).commit.sha;
if (options?.etag && options.etag === commitSha) {
throw new NotModifiedError();
}
const archive = await fetch(
// archiveApiUrl looks like "https://api.github.com/repos/owner/repo/{archive_format}{/ref}"
archiveApiUrl
.replace('{archive_format}', 'tarball')
.replace('{/ref}', `/${commitSha}`),
{ headers },
);
if (!archive.ok) {
const message = `Failed to read tree (archive) from ${url}, ${archive.status} ${archive.statusText}`;
if (archive.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
// Note that repoResponseJson.full_name must be used over full_name because the path
// is case sensitive and full_name may not be inq the correct case.
// TODO(OrkoHunter): The directory name inside the tarball should be retrieved from the tar
// instead of being constructed here. Same goes for GitLab, Bitbucket and Azure.
const extractedDirName = `${repoResponseJson.full_name.replace(
'/',
'-',
)}-${commitSha.substr(0, 7)}`;
// The path includes the name of the directory inside the tarball and a sub path
// if requested in readTree.
const path = `${extractedDirName}/${filepath}`;
return await this.deps.treeResponseFactory.fromTarArchive({
// TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want
// to stick to using that in exclusively backend code.
stream: (response.body as unknown) as Readable,
stream: (archive.body as unknown) as Readable,
path,
etag: commitSha,
filter: options?.filter,
});
}
packages/backend-common/src/reading/GitlabUrlReader.test.ts
+137 -42
View File
@@ -23,6 +23,7 @@ import path from 'path';
import { getVoidLogger } from '../logging';
import { GitlabUrlReader } from './GitlabUrlReader';
import { ReadTreeResponseFactory } from './tree';
import { NotModifiedError, NotFoundError } from '../errors';
const logger = getVoidLogger();
@@ -30,6 +31,22 @@ const treeResponseFactory = ReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
const gitlabProcessor = new GitlabUrlReader(
{
host: 'gitlab.com',
apiBaseUrl: 'https://gitlab.com/api/v4',
},
{ treeResponseFactory },
);
const hostedGitlabProcessor = new GitlabUrlReader(
{
host: 'gitlab.mycompany.com',
apiBaseUrl: 'https://gitlab.mycompany.com/api/v4',
},
{ treeResponseFactory },
);
describe('GitlabUrlReader', () => {
const worker = setupServer();
msw.setupDefaultHandlers(worker);
@@ -136,39 +153,94 @@ describe('GitlabUrlReader', () => {
});
describe('readTree', () => {
const repoBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'repo.zip'),
const archiveBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.zip'),
);
const projectGitlabApiResponse = {
id: 11111111,
default_branch: 'main',
};
const branchGitlabApiResponse = {
commit: {
id: 'sha123abc',
},
};
beforeEach(() => {
worker.use(
rest.get(
'https://gitlab.com/backstage/mock/-/archive/repo/mock-repo.zip',
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive.zip?sha=main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(repoBuffer),
ctx.body(archiveBuffer),
),
),
rest.get(
'https://gitlab.com/api/v4/projects/backstage%2Fmock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(projectGitlabApiResponse),
),
),
rest.get(
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/branches/main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(branchGitlabApiResponse),
),
),
rest.get(
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/branches/branchDoesNotExist',
(_, res, ctx) => res(ctx.status(404)),
),
rest.get(
'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(projectGitlabApiResponse),
),
),
rest.get(
'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/branches/main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(branchGitlabApiResponse),
),
),
rest.get(
'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/archive.zip?sha=main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(archiveBuffer),
),
),
);
});
it('returns the wanted files from an archive', async () => {
const processor = new GitlabUrlReader(
{ host: 'gitlab.com' },
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://gitlab.com/backstage/mock/tree/repo',
const response = await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock/tree/main',
);
const files = await response.files();
expect(files.length).toBe(2);
const indexMarkdownFile = await files[0].content();
const mkDocsFile = await files[1].content();
const mkDocsFile = await files[0].content();
const indexMarkdownFile = await files[1].content();
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
expect(indexMarkdownFile.toString()).toBe('# Test\n');
@@ -177,23 +249,18 @@ describe('GitlabUrlReader', () => {
it('returns the wanted files from hosted gitlab', async () => {
worker.use(
rest.get(
'https://git.mycompany.com/backstage/mock/-/archive/repo/mock-repo.zip',
'https://gitlab.mycompany.com/backstage/mock/-/archive/main.zip',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(repoBuffer),
ctx.body(archiveBuffer),
),
),
);
const processor = new GitlabUrlReader(
{ host: 'git.mycompany.com' },
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://git.mycompany.com/backstage/mock/tree/repo/docs',
const response = await hostedGitlabProcessor.readTree(
'https://gitlab.mycompany.com/backstage/mock/tree/main/docs',
);
const files = await response.files();
@@ -204,27 +271,9 @@ describe('GitlabUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('throws an error when branch is not specified', async () => {
const processor = new GitlabUrlReader(
{ host: 'gitlab.com' },
{ treeResponseFactory },
);
await expect(
processor.readTree('https://gitlab.com/backstage/mock'),
).rejects.toThrow(
'GitLab URL must contain a branch to be able to fetch its tree',
);
});
it('returns the wanted files from an archive with a subpath', async () => {
const processor = new GitlabUrlReader(
{ host: 'gitlab.com' },
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://gitlab.com/backstage/mock/tree/repo/docs',
const response = await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock/tree/main/docs',
);
const files = await response.files();
@@ -234,5 +283,51 @@ describe('GitlabUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnGitlab = async () => {
await gitlabProcessor.readTree('https://gitlab.com/backstage/mock', {
etag: 'sha123abc',
});
};
const fnHostedGitlab = async () => {
await hostedGitlabProcessor.readTree(
'https://gitlab.mycompany.com/backstage/mock',
{
etag: 'sha123abc',
},
);
};
await expect(fnGitlab).rejects.toThrow(NotModifiedError);
await expect(fnHostedGitlab).rejects.toThrow(NotModifiedError);
});
it('should not throw error when given an outdated etag in options', async () => {
const response = await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock/tree/main',
{
etag: 'outdatedsha123abc',
},
);
expect((await response.files()).length).toBe(2);
});
it('should detect the default branch', async () => {
const response = await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock',
);
expect((await response.files()).length).toBe(2);
});
it('should throw error on missing branch', async () => {
const fnGithub = async () => {
await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock/tree/branchDoesNotExist',
);
};
await expect(fnGithub).rejects.toThrow(NotFoundError);
});
});
});
packages/backend-common/src/reading/GitlabUrlReader.ts
+69 -32
View File
@@ -21,7 +21,7 @@ import {
readGitLabIntegrationConfigs,
} from '@backstage/integration';
import fetch from 'cross-fetch';
import { InputError, NotFoundError } from '../errors';
import { NotFoundError, NotModifiedError } from '../errors';
import { ReadTreeResponseFactory } from './tree';
import {
ReaderFactory,
@@ -39,26 +39,26 @@ export class GitlabUrlReader implements UrlReader {
const configs = readGitLabIntegrationConfigs(
config.getOptionalConfigArray('integrations.gitlab') ?? [],
);
return configs.map(options => {
const reader = new GitlabUrlReader(options, { treeResponseFactory });
const predicate = (url: URL) => url.host === options.host;
return configs.map(provider => {
const reader = new GitlabUrlReader(provider, { treeResponseFactory });
const predicate = (url: URL) => url.host === provider.host;
return { reader, predicate };
});
};
constructor(
private readonly options: GitLabIntegrationConfig,
private readonly config: GitLabIntegrationConfig,
deps: { treeResponseFactory: ReadTreeResponseFactory },
) {
this.treeResponseFactory = deps.treeResponseFactory;
}
async read(url: string): Promise<Buffer> {
const builtUrl = await getGitLabFileFetchUrl(url, this.options);
const builtUrl = await getGitLabFileFetchUrl(url, this.config);
let response: Response;
try {
response = await fetch(builtUrl, getGitLabRequestOptions(this.options));
response = await fetch(builtUrl, getGitLabRequestOptions(this.config));
} catch (e) {
throw new Error(`Unable to read ${url}, ${e}`);
}
@@ -78,45 +78,82 @@ export class GitlabUrlReader implements UrlReader {
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const {
name: repoName,
ref,
protocol,
resource,
full_name,
filepath,
} = parseGitUrl(url);
const { name: repoName, ref, full_name, filepath } = parseGitUrl(url);
if (!ref) {
throw new InputError(
'GitLab URL must contain a branch to be able to fetch its tree',
);
}
const archive = `${protocol}://${resource}/${full_name}/-/archive/${ref}/${repoName}-${ref}.zip`;
const response = await fetch(
archive,
getGitLabRequestOptions(this.options),
// Use GitLab API to get the default branch
// encodeURIComponent is required for GitLab API
// https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding
const projectGitlabResponse = await fetch(
new URL(
`${this.config.apiBaseUrl}/projects/${encodeURIComponent(full_name)}`,
).toString(),
getGitLabRequestOptions(this.config),
);
if (!response.ok) {
const msg = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
if (!projectGitlabResponse.ok) {
const msg = `Failed to read tree from ${url}, ${projectGitlabResponse.status} ${projectGitlabResponse.statusText}`;
if (projectGitlabResponse.status === 404) {
throw new NotFoundError(msg);
}
throw new Error(msg);
}
const projectGitlabResponseJson = await projectGitlabResponse.json();
const path = filepath ? `${repoName}-${ref}/${filepath}/` : '';
// ref is an empty string if no branch is set in provided url to readTree.
const branch = ref || projectGitlabResponseJson.default_branch;
return this.treeResponseFactory.fromZipArchive({
stream: (response.body as unknown) as Readable,
// Fetch the latest commit in the provided or default branch to compare against
// the provided sha.
const branchGitlabResponse = await fetch(
new URL(
`${this.config.apiBaseUrl}/projects/${encodeURIComponent(
full_name,
)}/repository/branches/${branch}`,
).toString(),
getGitLabRequestOptions(this.config),
);
if (!branchGitlabResponse.ok) {
const message = `Failed to read tree (branch) from ${url}, ${branchGitlabResponse.status} ${branchGitlabResponse.statusText}`;
if (branchGitlabResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const commitSha = (await branchGitlabResponse.json()).commit.id;
if (options?.etag && options.etag === commitSha) {
throw new NotModifiedError();
}
// https://docs.gitlab.com/ee/api/repositories.html#get-file-archive
const archiveGitLabResponse = await fetch(
`${this.config.apiBaseUrl}/projects/${encodeURIComponent(
full_name,
)}/repository/archive.zip?sha=${branch}`,
getGitLabRequestOptions(this.config),
);
if (!archiveGitLabResponse.ok) {
const message = `Failed to read tree (archive) from ${url}, ${archiveGitLabResponse.status} ${archiveGitLabResponse.statusText}`;
if (archiveGitLabResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const path = filepath
? `${repoName}-${branch}-${commitSha}/${filepath}/`
: '';
return await this.treeResponseFactory.fromZipArchive({
stream: (archiveGitLabResponse.body as unknown) as Readable,
path,
etag: commitSha,
filter: options?.filter,
});
}
toString() {
const { host, token } = this.options;
const { host, token } = this.config;
return `gitlab{host=${host},authed=${Boolean(token)}}`;
}
}
packages/backend-common/src/reading/UrlReaderPredicateMux.ts
+5 -2
View File
@@ -45,12 +45,15 @@ export class UrlReaderPredicateMux implements UrlReader {
throw new NotAllowedError(`Reading from '${url}' is not allowed`);
}
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse> {
async readTree(
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const parsed = new URL(url);
for (const { predicate, reader } of this.readers) {
if (predicate(parsed)) {
return reader.readTree(url, options);
return await reader.readTree(url, options);
}
}
packages/backend-common/src/reading/__fixtures__/backstage-mock-etag123.tar.gz
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/__fixtures__/gitlab-archive.zip
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/__fixtures__/mock-main.tar.gz
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/__fixtures__/mock-main.zip
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/__fixtures__/repo.tar.gz
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/__fixtures__/repo.zip
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts
+4
View File
@@ -26,6 +26,8 @@ type FromArchiveOptions = {
stream: Readable;
// If set, the root of the tree will be set to the given directory path.
path?: string;
// etag of the blob
etag: string;
// Filter passed on from the ReadTreeOptions
filter?: (path: string) => boolean;
};
@@ -45,6 +47,7 @@ export class ReadTreeResponseFactory {
options.stream,
options.path ?? '',
this.workDir,
options.etag,
options.filter,
);
}
@@ -54,6 +57,7 @@ export class ReadTreeResponseFactory {
options.stream,
options.path ?? '',
this.workDir,
options.etag,
options.filter,
);
}
packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts
+25 -12
View File
@@ -20,7 +20,7 @@ import { resolve as resolvePath } from 'path';
import { TarArchiveResponse } from './TarArchiveResponse';
const archiveData = fs.readFileSync(
resolvePath(__filename, '../../__fixtures__/repo.tar.gz'),
resolvePath(__filename, '../../__fixtures__/mock-main.tar.gz'),
);
describe('TarArchiveResponse', () => {
@@ -38,7 +38,7 @@ describe('TarArchiveResponse', () => {
it('should read files', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp');
const res = new TarArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const files = await res.files();
expect(files).toEqual([
@@ -61,8 +61,12 @@ describe('TarArchiveResponse', () => {
it('should read files with filter', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
const res = new TarArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
);
const files = await res.files();
@@ -79,14 +83,14 @@ describe('TarArchiveResponse', () => {
it('should read as archive and files', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp');
const res = new TarArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const buffer = await res.archive();
await expect(res.archive()).rejects.toThrow(
'Response has already been read',
);
const res2 = new TarArchiveResponse(buffer, '', '/tmp');
const res2 = new TarArchiveResponse(buffer, '', '/tmp', 'etag');
const files = await res2.files();
expect(files).toEqual([
@@ -109,21 +113,26 @@ describe('TarArchiveResponse', () => {
it('should extract entire archive into directory', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, '', '/tmp');
const res = new TarArchiveResponse(stream, '', '/tmp', 'etag');
const dir = await res.dir();
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'),
fs.readFile(resolvePath(dir, 'mock-main/mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'),
fs.readFile(resolvePath(dir, 'mock-main/docs/index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('should extract archive into directory with a subpath', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-repo/docs/', '/tmp');
const res = new TarArchiveResponse(
stream,
'mock-main/docs/',
'/tmp',
'etag',
);
const dir = await res.dir();
expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/);
@@ -135,8 +144,12 @@ describe('TarArchiveResponse', () => {
it('should extract archive into directory with a subpath and filter', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
const res = new TarArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
);
const dir = await res.dir({ targetDir: '/tmp' });
packages/backend-common/src/reading/tree/TarArchiveResponse.ts
+3
View File
@@ -41,6 +41,7 @@ export class TarArchiveResponse implements ReadTreeResponse {
private readonly stream: Readable,
private readonly subPath: string,
private readonly workDir: string,
public readonly etag: string,
private readonly filter?: (path: string) => boolean,
) {
if (subPath) {
@@ -53,6 +54,8 @@ export class TarArchiveResponse implements ReadTreeResponse {
);
}
}
this.etag = etag;
}
// Make sure the input stream is only read once
packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts
+31 -18
View File
@@ -20,7 +20,7 @@ import { resolve as resolvePath } from 'path';
import { ZipArchiveResponse } from './ZipArchiveResponse';
const archiveData = fs.readFileSync(
resolvePath(__filename, '../../__fixtures__/repo.zip'),
resolvePath(__filename, '../../__fixtures__/mock-main.zip'),
);
describe('ZipArchiveResponse', () => {
@@ -38,31 +38,35 @@ describe('ZipArchiveResponse', () => {
it('should read files', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp');
const res = new ZipArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const files = await res.files();
expect(files).toEqual([
{
path: 'docs/index.md',
path: 'mkdocs.yml',
content: expect.any(Function),
},
{
path: 'mkdocs.yml',
path: 'docs/index.md',
content: expect.any(Function),
},
]);
const contents = await Promise.all(files.map(f => f.content()));
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
'# Test',
'site_name: Test',
'# Test',
]);
});
it('should read files with filter', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
const res = new ZipArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
);
const files = await res.files();
@@ -79,51 +83,56 @@ describe('ZipArchiveResponse', () => {
it('should read as archive and files', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp');
const res = new ZipArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const buffer = await res.archive();
await expect(res.archive()).rejects.toThrow(
'Response has already been read',
);
const res2 = new ZipArchiveResponse(buffer, '', '/tmp');
const res2 = new ZipArchiveResponse(buffer, '', '/tmp', 'etag');
const files = await res2.files();
expect(files).toEqual([
{
path: 'docs/index.md',
path: 'mkdocs.yml',
content: expect.any(Function),
},
{
path: 'mkdocs.yml',
path: 'docs/index.md',
content: expect.any(Function),
},
]);
const contents = await Promise.all(files.map(f => f.content()));
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
'# Test',
'site_name: Test',
'# Test',
]);
});
it('should extract entire archive into directory', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, '', '/tmp');
const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag');
const dir = await res.dir();
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'),
fs.readFile(resolvePath(dir, 'mock-main/mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'),
fs.readFile(resolvePath(dir, 'mock-main/docs/index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('should extract archive into directory with a subpath', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/docs/', '/tmp');
const res = new ZipArchiveResponse(
stream,
'mock-main/docs/',
'/tmp',
'etag',
);
const dir = await res.dir();
expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/);
@@ -135,8 +144,12 @@ describe('ZipArchiveResponse', () => {
it('should extract archive into directory with a subpath and filter', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
const res = new ZipArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
);
const dir = await res.dir({ targetDir: '/tmp' });
packages/backend-common/src/reading/tree/ZipArchiveResponse.ts
+3
View File
@@ -35,6 +35,7 @@ export class ZipArchiveResponse implements ReadTreeResponse {
private readonly stream: Readable,
private readonly subPath: string,
private readonly workDir: string,
public readonly etag: string,
private readonly filter?: (path: string) => boolean,
) {
if (subPath) {
@@ -47,6 +48,8 @@ export class ZipArchiveResponse implements ReadTreeResponse {
);
}
}
this.etag = etag;
}
// Make sure the input stream is only read once
packages/backend-common/src/reading/types.ts
+22
View File
@@ -32,6 +32,19 @@ export type ReadTreeOptions = {
* If no filter is provided all files are extracted.
*/
filter?(path: string): boolean;
/**
* An etag can be provided to check whether readTree's response has changed from a previous execution.
*
* In the readTree() response, an etag is returned along with the tree blob. The etag is a unique identifer
* of the tree blob, usually the commit SHA or etag from the target.
*
* When a etag is given in ReadTreeOptions, readTree will first compare the etag against the etag
* on the target branch. If they match, readTree will throw a NotModifiedError indicating that the readTree
* response will not differ from the previous response which included this particular etag. If they mismatch,
* readTree will return the rest of ReadTreeResponse along with a new etag.
*/
etag?: string;
};
/**
@@ -70,5 +83,14 @@ export type ReadTreeResponseDirOptions = {
export type ReadTreeResponse = {
files(): Promise<ReadTreeResponseFile[]>;
archive(): Promise<NodeJS.ReadableStream>;
/**
* dir() extracts the tree response into a directory and returns the path of the directory.
*/
dir(options?: ReadTreeResponseDirOptions): Promise<string>;
/**
* A unique identifer of the tree blob, usually the commit SHA or etag from the target.
*/
etag: string;
};
packages/integration/src/azure/core.ts
+56
View File
@@ -108,6 +108,62 @@ export function getAzureDownloadUrl(url: string): string {
return `${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`;
}
/**
* Given a URL, return the API URL to fetch commits on the branch.
*
* @param url A URL pointing to a repository or a sub-path
*/
export function getAzureCommitsUrl(url: string): string {
try {
const parsedUrl = new URL(url);
const [
empty,
userOrOrg,
project,
srcKeyword,
repoName,
] = parsedUrl.pathname.split('/');
// Remove the "GB" from "GBmain" for example.
const ref = parsedUrl.searchParams.get('version')?.substr(2);
if (
!!empty ||
!userOrOrg ||
!project ||
srcKeyword !== '_git' ||
!repoName
) {
throw new Error('Wrong Azure Devops URL');
}
// transform to commits api
parsedUrl.pathname = [
empty,
userOrOrg,
project,
'_apis',
'git',
'repositories',
repoName,
'commits',
].join('/');
const queryParams = [];
if (ref) {
queryParams.push(`searchCriteria.itemVersion.version=${ref}`);
}
parsedUrl.search = queryParams.join('&');
parsedUrl.protocol = 'https';
return parsedUrl.toString();
} catch (e) {
throw new Error(`Incorrect URL: ${url}, ${e}`);
}
}
/**
* Gets the request options necessary to make requests to a given provider.
*
packages/integration/src/azure/index.ts
+1
View File
@@ -23,4 +23,5 @@ export {
getAzureDownloadUrl,
getAzureFileFetchUrl,
getAzureRequestOptions,
getAzureCommitsUrl,
} from './core';
packages/integration/src/gitlab/config.test.ts
+13 -1
View File
@@ -43,7 +43,18 @@ describe('readGitLabIntegrationConfig', () => {
const output = readGitLabIntegrationConfig(buildConfig({}));
expect(output).toEqual({
host: 'gitlab.com',
apiBaseUrl: 'gitlab.com/api/v4',
apiBaseUrl: 'https://gitlab.com/api/v4',
});
});
it('injects the correct GitLab API base URL when missing', () => {
const output = readGitLabIntegrationConfig(
buildConfig({ host: 'gitlab.com' }),
);
expect(output).toEqual({
host: 'gitlab.com',
apiBaseUrl: 'https://gitlab.com/api/v4',
});
});
@@ -86,6 +97,7 @@ describe('readGitLabIntegrationConfigs', () => {
expect(output).toEqual([
{
host: 'gitlab.com',
apiBaseUrl: 'https://gitlab.com/api/v4',
},
]);
});
packages/integration/src/gitlab/config.ts
+2 -2
View File
@@ -18,7 +18,7 @@ import { Config } from '@backstage/config';
import { isValidHost } from '../helpers';
const GITLAB_HOST = 'gitlab.com';
const GITLAB_API_BASE_URL = 'gitlab.com/api/v4';
const GITLAB_API_BASE_URL = 'https://gitlab.com/api/v4';
/**
* The configuration parameters for a single GitLab integration.
@@ -89,7 +89,7 @@ export function readGitLabIntegrationConfigs(
// As a convenience we always make sure there's at least an unauthenticated
// reader for public gitlab repos.
if (!result.some(c => c.host === GITLAB_HOST)) {
result.push({ host: GITLAB_HOST });
result.push({ host: GITLAB_HOST, apiBaseUrl: GITLAB_API_BASE_URL });
}
return result;
packages/techdocs-common/src/helpers.test.ts
+1
View File
@@ -135,6 +135,7 @@ describe('getDocFilesFromRepository', () => {
archive: async () => {
return Readable.from('');
},
etag: '',
};
}
}