paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 17 Packages Projects Releases Wiki Activity

Merge pull request #4120 from backstage/orkohunter/url-reader-readTree-with-sha

Browse Source
This commit is contained in:
Himanshu Mishra
2021-01-20 14:49:47 +01:00
committed by GitHub
parent cb296b898c 662d9b9a22
commit e8e6bd92b6
33 changed files with 852 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/khaki-icons-trade.md
+26
View File
@@ -0,0 +1,26 @@
---
'@backstage/backend-common': patch
---
1. URL Reader's `readTree` method now returns an `etag` in the response along with the blob. The etag is an identifier of the blob and will only change if the blob is modified on the target. Usually it is set to the latest commit SHA on the target.
`readTree` also takes an optional `etag` in its options and throws a `NotModifiedError` if the etag matches with the etag of the resource.
So, the `etag` can be used in building a cache when working with URL Reader.
An example -
```ts
const response = await reader.readTree(
'https://github.com/backstage/backstage',
);
const etag = response.etag;
// Will throw a new NotModifiedError (exported from @backstage/backstage-common)
await reader.readTree('https://github.com/backstage/backstage', {
etag,
});
```
2. URL Reader's readTree method can now detect the default branch. So, `url:https://github.com/org/repo/tree/master` can be replaced with `url:https://github.com/org/repo` in places like `backstage.io/techdocs-ref`.
.changeset/nervous-mails-repair.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/integration': patch
---
Fix GitLab API base URL and add it by default to the gitlab.com host
.github/styles/vocab.txt
+1
View File
@@ -67,6 +67,7 @@ Dominik
dtuite
dzolotusky
Ek
etag
env
Env
eslint
catalog-info.yaml
+1 -1
View File
@@ -6,7 +6,7 @@ metadata:
Backstage is an open-source developer portal that puts the developer experience first.
annotations:
github.com/project-slug: backstage/backstage
backstage.io/techdocs-ref: url:https://github.com/backstage/backstage/tree/master
backstage.io/techdocs-ref: url:https://github.com/backstage/backstage
lighthouse.com/website-url: https://backstage.io
spec:
type: library
packages/backend-common/src/errors.ts
+5
View File
@@ -75,3 +75,8 @@ export class NotFoundError extends CustomErrorBase {}
* resource.
*/
export class ConflictError extends CustomErrorBase {}
/**
* The requested resource has not changed since last request.
*/
export class NotModifiedError extends CustomErrorBase {}
packages/backend-common/src/middleware/errorHandler.test.ts
+4
View File
@@ -72,6 +72,9 @@ describe('errorHandler', () => {
it('handles well-known error classes', async () => {
const app = express();
app.use('/NotModifiedError', () => {
throw new errors.NotModifiedError();
});
app.use('/InputError', () => {
throw new errors.InputError();
});
@@ -90,6 +93,7 @@ describe('errorHandler', () => {
app.use(errorHandler());
const r = request(app);
expect((await r.get('/NotModifiedError')).status).toBe(304);
expect((await r.get('/InputError')).status).toBe(400);
expect((await r.get('/AuthenticationError')).status).toBe(401);
expect((await r.get('/NotAllowedError')).status).toBe(403);
packages/backend-common/src/middleware/errorHandler.ts
+2
View File
@@ -101,6 +101,8 @@ function getStatusCode(error: Error): number {
// Handle well-known error types
switch (error.name) {
case errors.NotModifiedError.name:
return 304;
case errors.InputError.name:
return 400;
case errors.AuthenticationError.name:
packages/backend-common/src/reading/AzureUrlReader.test.ts
+60 -8
View File
@@ -23,6 +23,7 @@ import { getVoidLogger } from '../logging';
import { AzureUrlReader } from './AzureUrlReader';
import { msw } from '@backstage/test-utils';
import { ReadTreeResponseFactory } from './tree';
import { NotModifiedError } from '../errors';
const logger = getVoidLogger();
@@ -139,7 +140,12 @@ describe('AzureUrlReader', () => {
describe('readTree', () => {
const repoBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'repo.zip'),
path.resolve('src', 'reading', '__fixtures__', 'mock-main.zip'),
);
const processor = new AzureUrlReader(
{ host: 'dev.azure.com' },
{ treeResponseFactory },
);
beforeEach(() => {
@@ -153,24 +159,70 @@ describe('AzureUrlReader', () => {
ctx.body(repoBuffer),
),
),
rest.get(
// https://docs.microsoft.com/en-us/rest/api/azure/devops/git/commits/get%20commits?view=azure-devops-rest-6.0#on-a-branch
'https://dev.azure.com/organization/project/_apis/git/repositories/repository/commits',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
count: 2,
value: [
{
commitId: '123abc2',
comment: 'second commit',
},
{
commitId: '123abc1',
comment: 'first commit',
},
],
}),
),
),
);
});
it('returns the wanted files from an archive', async () => {
const processor = new AzureUrlReader(
{ host: 'dev.azure.com' },
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://dev.azure.com/organization/project/_git/repository',
);
expect(response.etag).toBe('123abc2');
const files = await response.files();
expect(files.length).toBe(2);
const mkDocsFile = await files[1].content();
const indexMarkdownFile = await files[0].content();
const mkDocsFile = await files[0].content();
const indexMarkdownFile = await files[1].content();
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnAzure = async () => {
await processor.readTree(
'https://dev.azure.com/organization/project/_git/repository',
{ etag: '123abc2' },
);
};
await expect(fnAzure).rejects.toThrow(NotModifiedError);
});
it('should not throw a NotModifiedError when given an outdated etag in options', async () => {
const response = await processor.readTree(
'https://dev.azure.com/organization/project/_git/repository',
{ etag: 'outdated123abc' },
);
expect(response.etag).toBe('123abc2');
const files = await response.files();
expect(files.length).toBe(2);
const mkDocsFile = await files[0].content();
const indexMarkdownFile = await files[1].content();
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
expect(indexMarkdownFile.toString()).toBe('# Test\n');
packages/backend-common/src/reading/AzureUrlReader.ts
+30 -9
View File
@@ -20,10 +20,11 @@ import {
getAzureFileFetchUrl,
getAzureDownloadUrl,
getAzureRequestOptions,
getAzureCommitsUrl,
} from '@backstage/integration';
import fetch from 'cross-fetch';
import { Readable } from 'stream';
import { NotFoundError } from '../errors';
import { NotFoundError, NotModifiedError } from '../errors';
import {
ReaderFactory,
ReadTreeOptions,
@@ -75,20 +76,40 @@ export class AzureUrlReader implements UrlReader {
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const response = await fetch(
getAzureDownloadUrl(url),
getAzureRequestOptions(this.options, { Accept: 'application/zip' }),
// Get latest commit SHA
const commitsAzureResponse = await fetch(
getAzureCommitsUrl(url),
getAzureRequestOptions(this.options),
);
if (!response.ok) {
const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
if (!commitsAzureResponse.ok) {
const message = `Failed to read tree from ${url}, ${commitsAzureResponse.status} ${commitsAzureResponse.statusText}`;
if (commitsAzureResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
return this.deps.treeResponseFactory.fromZipArchive({
stream: (response.body as unknown) as Readable,
const commitSha = (await commitsAzureResponse.json()).value[0].commitId;
if (options?.etag && options.etag === commitSha) {
throw new NotModifiedError();
}
const archiveAzureResponse = await fetch(
getAzureDownloadUrl(url),
getAzureRequestOptions(this.options, { Accept: 'application/zip' }),
);
if (!archiveAzureResponse.ok) {
const message = `Failed to read tree from ${url}, ${archiveAzureResponse.status} ${archiveAzureResponse.statusText}`;
if (archiveAzureResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
return await this.deps.treeResponseFactory.fromZipArchive({
stream: (archiveAzureResponse.body as unknown) as Readable,
etag: commitSha,
filter: options?.filter,
});
}
packages/backend-common/src/reading/BitbucketUrlReader.test.ts
+88 -69
View File
@@ -20,6 +20,7 @@ import fs from 'fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import path from 'path';
import { NotModifiedError } from '../errors';
import { BitbucketUrlReader } from './BitbucketUrlReader';
import { ReadTreeResponseFactory } from './tree';
@@ -27,15 +28,24 @@ const treeResponseFactory = ReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
const bitbucketProcessor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
{ treeResponseFactory },
);
const hostedBitbucketProcessor = new BitbucketUrlReader(
{
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
},
{ treeResponseFactory },
);
describe('BitbucketUrlReader', () => {
describe('implementation', () => {
it('rejects unknown targets', async () => {
const processor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
{ treeResponseFactory },
);
await expect(
processor.read('https://not.bitbucket.com/apa'),
bitbucketProcessor.read('https://not.bitbucket.com/apa'),
).rejects.toThrow(
'Incorrect URL: https://not.bitbucket.com/apa, Error: Invalid Bitbucket URL or file path',
);
@@ -55,8 +65,30 @@ describe('BitbucketUrlReader', () => {
),
);
it('returns the wanted files from an archive', async () => {
const privateBitbucketRepoBuffer = fs.readFileSync(
path.resolve(
'src',
'reading',
'__fixtures__',
'bitbucket-server-repo.zip',
),
);
beforeEach(() => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
mainbranch: {
type: 'branch',
name: 'master',
},
}),
),
),
rest.get(
'https://bitbucket.org/backstage/mock/get/master.zip',
(_, res, ctx) =>
@@ -76,17 +108,35 @@ describe('BitbucketUrlReader', () => {
}),
),
),
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(privateBitbucketRepoBuffer),
),
),
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/repositories/backstage/mock/commits/some-branch',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
});
const processor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
{ treeResponseFactory },
);
const response = await processor.readTree(
it('returns the wanted files from an archive', async () => {
const response = await bitbucketProcessor.readTree(
'https://bitbucket.org/backstage/mock/src/master',
);
expect(response.etag).toBe('12ab34cd56ef');
const files = await response.files();
expect(files.length).toBe(2);
@@ -98,38 +148,12 @@ describe('BitbucketUrlReader', () => {
});
it('uses private bitbucket host', async () => {
const privateBitbucketRepoBuffer = fs.readFileSync(
path.resolve(
'src',
'reading',
'__fixtures__',
'bitbucket-server-repo.zip',
),
);
worker.use(
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(privateBitbucketRepoBuffer),
),
),
);
const processor = new BitbucketUrlReader(
{
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
},
{ treeResponseFactory },
);
const response = await processor.readTree(
const response = await hostedBitbucketProcessor.readTree(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch',
);
expect(response.etag).toBe('12ab34cd56ef');
const files = await response.files();
expect(files.length).toBe(1);
@@ -139,37 +163,12 @@ describe('BitbucketUrlReader', () => {
});
it('returns the wanted files from an archive with a subpath', async () => {
worker.use(
rest.get(
'https://bitbucket.org/backstage/mock/get/master.zip',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(repoBuffer),
),
),
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
const processor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
{ treeResponseFactory },
);
const response = await processor.readTree(
const response = await bitbucketProcessor.readTree(
'https://bitbucket.org/backstage/mock/src/master/docs',
);
expect(response.etag).toBe('12ab34cd56ef');
const files = await response.files();
expect(files.length).toBe(1);
@@ -177,5 +176,25 @@ describe('BitbucketUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnBitbucket = async () => {
await bitbucketProcessor.readTree(
'https://bitbucket.org/backstage/mock',
{ etag: '12ab34cd56ef' },
);
};
await expect(fnBitbucket).rejects.toThrow(NotModifiedError);
});
it('should not throw a NotModifiedError when given an outdated etag in options', async () => {
const response = await bitbucketProcessor.readTree(
'https://bitbucket.org/backstage/mock',
{ etag: 'outdatedetag123abc' },
);
expect(response.etag).toBe('12ab34cd56ef');
});
});
});
packages/backend-common/src/reading/BitbucketUrlReader.ts
+14 -9
View File
@@ -25,7 +25,7 @@ import {
import fetch from 'cross-fetch';
import parseGitUrl from 'git-url-parse';
import { Readable } from 'stream';
import { NotFoundError } from '../errors';
import { NotFoundError, NotModifiedError } from '../errors';
import { ReadTreeResponseFactory } from './tree';
import {
ReaderFactory,
@@ -105,16 +105,21 @@ export class BitbucketUrlReader implements UrlReader {
url,
);
const lastCommitShortHash = await this.getLastCommitShortHash(url);
if (options?.etag && options.etag === lastCommitShortHash) {
throw new NotModifiedError();
}
const isHosted = resource === 'bitbucket.org';
const downloadUrl = await getBitbucketDownloadUrl(url, this.config);
const response = await fetch(
const archiveBitbucketResponse = await fetch(
downloadUrl,
getBitbucketRequestOptions(this.config),
);
if (!response.ok) {
const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
if (!archiveBitbucketResponse.ok) {
const message = `Failed to read tree from ${url}, ${archiveBitbucketResponse.status} ${archiveBitbucketResponse.statusText}`;
if (archiveBitbucketResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
@@ -122,13 +127,13 @@ export class BitbucketUrlReader implements UrlReader {
let folderPath = `${project}-${repoName}`;
if (isHosted) {
const lastCommitShortHash = await this.getLastCommitShortHash(url);
folderPath = `${project}-${repoName}-${lastCommitShortHash}`;
}
return this.treeResponseFactory.fromZipArchive({
stream: (response.body as unknown) as Readable,
return await this.treeResponseFactory.fromZipArchive({
stream: (archiveBitbucketResponse.body as unknown) as Readable,
path: `${folderPath}/${filepath}`,
etag: lastCommitShortHash,
filter: options?.filter,
});
}
@@ -142,7 +147,7 @@ export class BitbucketUrlReader implements UrlReader {
return `bitbucket{host=${host},authed=${authed}}`;
}
private async getLastCommitShortHash(url: string): Promise<String> {
private async getLastCommitShortHash(url: string): Promise<string> {
const { name: repoName, owner: project, ref } = parseGitUrl(url);
let branch = ref;
packages/backend-common/src/reading/GithubUrlReader.test.ts
+172 -105
View File
@@ -21,6 +21,7 @@ import fs from 'fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import path from 'path';
import { NotFoundError, NotModifiedError } from '../errors';
import { GithubUrlReader } from './GithubUrlReader';
import { ReadTreeResponseFactory } from './tree';
@@ -28,11 +29,27 @@ const treeResponseFactory = ReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
describe('GithubUrlReader', () => {
const mockCredentialsProvider = ({
getCredentials: jest.fn().mockResolvedValue({ headers: {} }),
} as unknown) as GithubCredentialsProvider;
const mockCredentialsProvider = ({
getCredentials: jest.fn().mockResolvedValue({ headers: {} }),
} as unknown) as GithubCredentialsProvider;
const githubProcessor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
const gheProcessor = new GithubUrlReader(
{
host: 'ghe.github.com',
apiBaseUrl: 'https://ghe.github.com/api/v3',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
describe('GithubUrlReader', () => {
const worker = setupServer();
msw.setupDefaultHandlers(worker);
@@ -43,15 +60,8 @@ describe('GithubUrlReader', () => {
describe('implementation', () => {
it('rejects unknown targets', async () => {
const processor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
await expect(
processor.read('https://not.github.com/apa'),
githubProcessor.read('https://not.github.com/apa'),
).rejects.toThrow(
'Incorrect URL: https://not.github.com/apa, Error: Invalid GitHub URL or file path',
);
@@ -73,7 +83,7 @@ describe('GithubUrlReader', () => {
worker.use(
rest.get(
'https://api.github.com/repos/backstage/mock/tree/contents/?ref=repo',
'https://api.github.com/repos/backstage/mock/tree/contents/?ref=main',
(req, res, ctx) => {
expect(req.headers.get('authorization')).toBe(
mockHeaders.Authorization,
@@ -90,28 +100,67 @@ describe('GithubUrlReader', () => {
),
);
const processor = new GithubUrlReader(
{
host: 'ghe.github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
await processor.read(
'https://ghe.github.com/backstage/mock/tree/blob/repo',
await githubProcessor.read(
'https://github.com/backstage/mock/tree/blob/main',
);
});
});
describe('readTree', () => {
const repoBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'repo.tar.gz'),
path.resolve(
'src',
'reading',
'__fixtures__',
'backstage-mock-etag123.tar.gz',
),
);
const reposGithubApiResponse = {
id: '123',
full_name: 'backstage/mock',
default_branch: 'main',
branches_url:
'https://api.github.com/repos/backstage/mock/branches{/branch}',
archive_url:
'https://api.github.com/repos/backstage/mock/{archive_format}{/ref}',
};
const reposGheApiResponse = {
...reposGithubApiResponse,
branches_url:
'https://ghe.github.com/api/v3/repos/backstage/mock/branches{/branch}',
archive_url:
'https://ghe.github.com/api/v3/repos/backstage/mock/{archive_format}{/ref}',
};
const branchesApiResponse = {
name: 'main',
commit: {
sha: 'etag123abc',
},
};
beforeEach(() => {
worker.use(
rest.get('https://api.github.com/repos/backstage/mock', (_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(reposGithubApiResponse),
),
),
rest.get(
'https://github.com/backstage/mock/archive/repo.tar.gz',
'https://api.github.com/repos/backstage/mock/branches/main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(branchesApiResponse),
),
),
rest.get(
'https://api.github.com/repos/backstage/mock/tarball/etag123abc',
(_, res, ctx) =>
res(
ctx.status(200),
@@ -119,21 +168,46 @@ describe('GithubUrlReader', () => {
ctx.body(repoBuffer),
),
),
rest.get(
'https://api.github.com/repos/backstage/mock/branches/branchDoesNotExist',
(_, res, ctx) => res(ctx.status(404)),
),
rest.get(
'https://ghe.github.com/api/v3/repos/backstage/mock/tarball/etag123abc',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/x-gzip'),
ctx.body(repoBuffer),
),
),
rest.get(
'https://ghe.github.com/api/v3/repos/backstage/mock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(reposGheApiResponse),
),
),
rest.get(
'https://ghe.github.com/api/v3/repos/backstage/mock/branches/main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(branchesApiResponse),
),
),
);
});
it('returns the wanted files from an archive', async () => {
const processor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
const response = await githubProcessor.readTree(
'https://github.com/backstage/mock/tree/main',
);
const response = await processor.readTree(
'https://github.com/backstage/mock/tree/repo',
);
expect(response.etag).toBe('etag123abc');
const files = await response.files();
@@ -145,40 +219,6 @@ describe('GithubUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('includes the subdomain in the github url', async () => {
worker.resetHandlers();
worker.use(
rest.get(
'https://ghe.github.com/backstage/mock/archive/repo.tar.gz',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/x-gzip'),
ctx.body(repoBuffer),
),
),
);
const processor = new GithubUrlReader(
{
host: 'ghe.github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
const response = await processor.readTree(
'https://ghe.github.com/backstage/mock/tree/repo/docs',
);
const files = await response.files();
expect(files.length).toBe(1);
const indexMarkdownFile = await files[0].content();
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('should use the headers from the credentials provider to the fetch request', async () => {
expect.assertions(2);
@@ -193,7 +233,7 @@ describe('GithubUrlReader', () => {
worker.use(
rest.get(
'https://ghe.github.com/backstage/mock/archive/repo.tar.gz',
'https://ghe.github.com/api/v3/repos/backstage/mock/tarball/etag123abc',
(req, res, ctx) => {
expect(req.headers.get('authorization')).toBe(
mockHeaders.Authorization,
@@ -210,46 +250,14 @@ describe('GithubUrlReader', () => {
),
);
const processor = new GithubUrlReader(
{
host: 'ghe.github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
await processor.readTree(
'https://ghe.github.com/backstage/mock/tree/repo/docs',
await gheProcessor.readTree(
'https://ghe.github.com/backstage/mock/tree/main',
);
});
it('must specify a branch', async () => {
const processor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
await expect(
processor.readTree('https://github.com/backstage/mock'),
).rejects.toThrow(
'GitHub URL must contain branch to be able to fetch tree',
);
});
it('returns the wanted files from an archive with a subpath', async () => {
const processor = new GithubUrlReader(
{
host: 'github.com',
apiBaseUrl: 'https://api.github.com',
},
{ treeResponseFactory, credentialsProvider: mockCredentialsProvider },
);
const response = await processor.readTree(
'https://github.com/backstage/mock/tree/repo/docs',
it('includes the subdomain in the github url', async () => {
const response = await gheProcessor.readTree(
'https://ghe.github.com/backstage/mock/tree/main/docs',
);
const files = await response.files();
@@ -259,5 +267,64 @@ describe('GithubUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('returns the wanted files from an archive with a subpath', async () => {
const response = await githubProcessor.readTree(
'https://github.com/backstage/mock/tree/main/docs',
);
const files = await response.files();
expect(files.length).toBe(1);
const indexMarkdownFile = await files[0].content();
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnGithub = async () => {
await githubProcessor.readTree('https://github.com/backstage/mock', {
etag: 'etag123abc',
});
};
const fnGhe = async () => {
await gheProcessor.readTree(
'https://ghe.github.com/backstage/mock/tree/main/docs',
{
etag: 'etag123abc',
},
);
};
await expect(fnGithub).rejects.toThrow(NotModifiedError);
await expect(fnGhe).rejects.toThrow(NotModifiedError);
});
it('should not throw error when given an outdated etag in options', async () => {
const response = await githubProcessor.readTree(
'https://github.com/backstage/mock/tree/main',
{
etag: 'outdatedetag123abc',
},
);
expect((await response.files()).length).toBe(2);
});
it('should detect the default branch', async () => {
const response = await githubProcessor.readTree(
'https://github.com/backstage/mock',
);
expect((await response.files()).length).toBe(2);
});
it('should throw error on missing branch', async () => {
const fnGithub = async () => {
await githubProcessor.readTree(
'https://github.com/backstage/mock/tree/branchDoesNotExist',
);
};
await expect(fnGithub).rejects.toThrow(NotFoundError);
});
});
});
packages/backend-common/src/reading/GithubUrlReader.ts
+72 -30
View File
@@ -23,7 +23,7 @@ import {
import fetch from 'cross-fetch';
import parseGitUrl from 'git-url-parse';
import { Readable } from 'stream';
import { InputError, NotFoundError } from '../errors';
import { NotFoundError, NotModifiedError } from '../errors';
import { ReadTreeResponseFactory } from './tree';
import {
ReaderFactory,
@@ -98,51 +98,93 @@ export class GithubUrlReader implements UrlReader {
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const {
name: repoName,
ref,
protocol,
resource,
full_name,
filepath,
} = parseGitUrl(url);
if (!ref) {
// TODO(Rugvip): We should add support for defaulting to the default branch
throw new InputError(
'GitHub URL must contain branch to be able to fetch tree',
);
}
const { ref, filepath, full_name } = parseGitUrl(url);
// Caveat: The ref will totally be incorrect if the branch name includes a /
// Thus, readTree can not work on url containing branch name that has a /
const { headers } = await this.deps.credentialsProvider.getCredentials({
url,
});
// TODO(Rugvip): use API to fetch URL instead
const response = await fetch(
new URL(
`${protocol}://${resource}/${full_name}/archive/${ref}.tar.gz`,
).toString(),
// Get GitHub API urls for the repository
const repoGitHubResponse = await fetch(
new URL(`${this.config.apiBaseUrl}/repos/${full_name}`).toString(),
{
headers: {
...headers,
},
headers,
},
);
if (!response.ok) {
const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
if (!repoGitHubResponse.ok) {
const message = `Failed to read tree (repository) from ${url}, ${repoGitHubResponse.status} ${repoGitHubResponse.statusText}`;
if (repoGitHubResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const path = `${repoName}-${ref}/${filepath}`;
const repoResponseJson = await repoGitHubResponse.json();
return this.deps.treeResponseFactory.fromTarArchive({
// ref is an empty string if no branch is set in provided url to readTree.
// Use GitHub API to get the default branch of the repository.
const branch = ref || repoResponseJson.default_branch;
const branchesApiUrl = repoResponseJson.branches_url;
const archiveApiUrl = repoResponseJson.archive_url;
// Fetch the latest commit in the provided or default branch to compare against
// the provided sha.
const branchGitHubResponse = await fetch(
// branchesApiUrl looks like "https://api.github.com/repos/owner/repo/branches{/branch}"
branchesApiUrl.replace('{/branch}', `/${branch}`),
{
headers,
},
);
if (!branchGitHubResponse.ok) {
const message = `Failed to read tree (branch) from ${url}, ${branchGitHubResponse.status} ${branchGitHubResponse.statusText}`;
if (branchGitHubResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const commitSha = (await branchGitHubResponse.json()).commit.sha;
if (options?.etag && options.etag === commitSha) {
throw new NotModifiedError();
}
const archive = await fetch(
// archiveApiUrl looks like "https://api.github.com/repos/owner/repo/{archive_format}{/ref}"
archiveApiUrl
.replace('{archive_format}', 'tarball')
.replace('{/ref}', `/${commitSha}`),
{ headers },
);
if (!archive.ok) {
const message = `Failed to read tree (archive) from ${url}, ${archive.status} ${archive.statusText}`;
if (archive.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
// Note that repoResponseJson.full_name must be used over full_name because the path
// is case sensitive and full_name may not be inq the correct case.
// TODO(OrkoHunter): The directory name inside the tarball should be retrieved from the tar
// instead of being constructed here. Same goes for GitLab, Bitbucket and Azure.
const extractedDirName = `${repoResponseJson.full_name.replace(
'/',
'-',
)}-${commitSha.substr(0, 7)}`;
// The path includes the name of the directory inside the tarball and a sub path
// if requested in readTree.
const path = `${extractedDirName}/${filepath}`;
return await this.deps.treeResponseFactory.fromTarArchive({
// TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want
// to stick to using that in exclusively backend code.
stream: (response.body as unknown) as Readable,
stream: (archive.body as unknown) as Readable,
path,
etag: commitSha,
filter: options?.filter,
});
}
packages/backend-common/src/reading/GitlabUrlReader.test.ts
+137 -42
View File
@@ -23,6 +23,7 @@ import path from 'path';
import { getVoidLogger } from '../logging';
import { GitlabUrlReader } from './GitlabUrlReader';
import { ReadTreeResponseFactory } from './tree';
import { NotModifiedError, NotFoundError } from '../errors';
const logger = getVoidLogger();
@@ -30,6 +31,22 @@ const treeResponseFactory = ReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
const gitlabProcessor = new GitlabUrlReader(
{
host: 'gitlab.com',
apiBaseUrl: 'https://gitlab.com/api/v4',
},
{ treeResponseFactory },
);
const hostedGitlabProcessor = new GitlabUrlReader(
{
host: 'gitlab.mycompany.com',
apiBaseUrl: 'https://gitlab.mycompany.com/api/v4',
},
{ treeResponseFactory },
);
describe('GitlabUrlReader', () => {
const worker = setupServer();
msw.setupDefaultHandlers(worker);
@@ -136,39 +153,94 @@ describe('GitlabUrlReader', () => {
});
describe('readTree', () => {
const repoBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'repo.zip'),
const archiveBuffer = fs.readFileSync(
path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.zip'),
);
const projectGitlabApiResponse = {
id: 11111111,
default_branch: 'main',
};
const branchGitlabApiResponse = {
commit: {
id: 'sha123abc',
},
};
beforeEach(() => {
worker.use(
rest.get(
'https://gitlab.com/backstage/mock/-/archive/repo/mock-repo.zip',
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive.zip?sha=main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(repoBuffer),
ctx.body(archiveBuffer),
),
),
rest.get(
'https://gitlab.com/api/v4/projects/backstage%2Fmock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(projectGitlabApiResponse),
),
),
rest.get(
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/branches/main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(branchGitlabApiResponse),
),
),
rest.get(
'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/branches/branchDoesNotExist',
(_, res, ctx) => res(ctx.status(404)),
),
rest.get(
'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(projectGitlabApiResponse),
),
),
rest.get(
'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/branches/main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(branchGitlabApiResponse),
),
),
rest.get(
'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/archive.zip?sha=main',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(archiveBuffer),
),
),
);
});
it('returns the wanted files from an archive', async () => {
const processor = new GitlabUrlReader(
{ host: 'gitlab.com' },
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://gitlab.com/backstage/mock/tree/repo',
const response = await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock/tree/main',
);
const files = await response.files();
expect(files.length).toBe(2);
const indexMarkdownFile = await files[0].content();
const mkDocsFile = await files[1].content();
const mkDocsFile = await files[0].content();
const indexMarkdownFile = await files[1].content();
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
expect(indexMarkdownFile.toString()).toBe('# Test\n');
@@ -177,23 +249,18 @@ describe('GitlabUrlReader', () => {
it('returns the wanted files from hosted gitlab', async () => {
worker.use(
rest.get(
'https://git.mycompany.com/backstage/mock/-/archive/repo/mock-repo.zip',
'https://gitlab.mycompany.com/backstage/mock/-/archive/main.zip',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(repoBuffer),
ctx.body(archiveBuffer),
),
),
);
const processor = new GitlabUrlReader(
{ host: 'git.mycompany.com' },
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://git.mycompany.com/backstage/mock/tree/repo/docs',
const response = await hostedGitlabProcessor.readTree(
'https://gitlab.mycompany.com/backstage/mock/tree/main/docs',
);
const files = await response.files();
@@ -204,27 +271,9 @@ describe('GitlabUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('throws an error when branch is not specified', async () => {
const processor = new GitlabUrlReader(
{ host: 'gitlab.com' },
{ treeResponseFactory },
);
await expect(
processor.readTree('https://gitlab.com/backstage/mock'),
).rejects.toThrow(
'GitLab URL must contain a branch to be able to fetch its tree',
);
});
it('returns the wanted files from an archive with a subpath', async () => {
const processor = new GitlabUrlReader(
{ host: 'gitlab.com' },
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://gitlab.com/backstage/mock/tree/repo/docs',
const response = await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock/tree/main/docs',
);
const files = await response.files();
@@ -234,5 +283,51 @@ describe('GitlabUrlReader', () => {
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnGitlab = async () => {
await gitlabProcessor.readTree('https://gitlab.com/backstage/mock', {
etag: 'sha123abc',
});
};
const fnHostedGitlab = async () => {
await hostedGitlabProcessor.readTree(
'https://gitlab.mycompany.com/backstage/mock',
{
etag: 'sha123abc',
},
);
};
await expect(fnGitlab).rejects.toThrow(NotModifiedError);
await expect(fnHostedGitlab).rejects.toThrow(NotModifiedError);
});
it('should not throw error when given an outdated etag in options', async () => {
const response = await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock/tree/main',
{
etag: 'outdatedsha123abc',
},
);
expect((await response.files()).length).toBe(2);
});
it('should detect the default branch', async () => {
const response = await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock',
);
expect((await response.files()).length).toBe(2);
});
it('should throw error on missing branch', async () => {
const fnGithub = async () => {
await gitlabProcessor.readTree(
'https://gitlab.com/backstage/mock/tree/branchDoesNotExist',
);
};
await expect(fnGithub).rejects.toThrow(NotFoundError);
});
});
});
packages/backend-common/src/reading/GitlabUrlReader.ts
+69 -32
View File
@@ -21,7 +21,7 @@ import {
readGitLabIntegrationConfigs,
} from '@backstage/integration';
import fetch from 'cross-fetch';
import { InputError, NotFoundError } from '../errors';
import { NotFoundError, NotModifiedError } from '../errors';
import { ReadTreeResponseFactory } from './tree';
import {
ReaderFactory,
@@ -39,26 +39,26 @@ export class GitlabUrlReader implements UrlReader {
const configs = readGitLabIntegrationConfigs(
config.getOptionalConfigArray('integrations.gitlab') ?? [],
);
return configs.map(options => {
const reader = new GitlabUrlReader(options, { treeResponseFactory });
const predicate = (url: URL) => url.host === options.host;
return configs.map(provider => {
const reader = new GitlabUrlReader(provider, { treeResponseFactory });
const predicate = (url: URL) => url.host === provider.host;
return { reader, predicate };
});
};
constructor(
private readonly options: GitLabIntegrationConfig,
private readonly config: GitLabIntegrationConfig,
deps: { treeResponseFactory: ReadTreeResponseFactory },
) {
this.treeResponseFactory = deps.treeResponseFactory;
}
async read(url: string): Promise<Buffer> {
const builtUrl = await getGitLabFileFetchUrl(url, this.options);
const builtUrl = await getGitLabFileFetchUrl(url, this.config);
let response: Response;
try {
response = await fetch(builtUrl, getGitLabRequestOptions(this.options));
response = await fetch(builtUrl, getGitLabRequestOptions(this.config));
} catch (e) {
throw new Error(`Unable to read ${url}, ${e}`);
}
@@ -78,45 +78,82 @@ export class GitlabUrlReader implements UrlReader {
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const {
name: repoName,
ref,
protocol,
resource,
full_name,
filepath,
} = parseGitUrl(url);
const { name: repoName, ref, full_name, filepath } = parseGitUrl(url);
if (!ref) {
throw new InputError(
'GitLab URL must contain a branch to be able to fetch its tree',
);
}
const archive = `${protocol}://${resource}/${full_name}/-/archive/${ref}/${repoName}-${ref}.zip`;
const response = await fetch(
archive,
getGitLabRequestOptions(this.options),
// Use GitLab API to get the default branch
// encodeURIComponent is required for GitLab API
// https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding
const projectGitlabResponse = await fetch(
new URL(
`${this.config.apiBaseUrl}/projects/${encodeURIComponent(full_name)}`,
).toString(),
getGitLabRequestOptions(this.config),
);
if (!response.ok) {
const msg = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
if (!projectGitlabResponse.ok) {
const msg = `Failed to read tree from ${url}, ${projectGitlabResponse.status} ${projectGitlabResponse.statusText}`;
if (projectGitlabResponse.status === 404) {
throw new NotFoundError(msg);
}
throw new Error(msg);
}
const projectGitlabResponseJson = await projectGitlabResponse.json();
const path = filepath ? `${repoName}-${ref}/${filepath}/` : '';
// ref is an empty string if no branch is set in provided url to readTree.
const branch = ref || projectGitlabResponseJson.default_branch;
return this.treeResponseFactory.fromZipArchive({
stream: (response.body as unknown) as Readable,
// Fetch the latest commit in the provided or default branch to compare against
// the provided sha.
const branchGitlabResponse = await fetch(
new URL(
`${this.config.apiBaseUrl}/projects/${encodeURIComponent(
full_name,
)}/repository/branches/${branch}`,
).toString(),
getGitLabRequestOptions(this.config),
);
if (!branchGitlabResponse.ok) {
const message = `Failed to read tree (branch) from ${url}, ${branchGitlabResponse.status} ${branchGitlabResponse.statusText}`;
if (branchGitlabResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const commitSha = (await branchGitlabResponse.json()).commit.id;
if (options?.etag && options.etag === commitSha) {
throw new NotModifiedError();
}
// https://docs.gitlab.com/ee/api/repositories.html#get-file-archive
const archiveGitLabResponse = await fetch(
`${this.config.apiBaseUrl}/projects/${encodeURIComponent(
full_name,
)}/repository/archive.zip?sha=${branch}`,
getGitLabRequestOptions(this.config),
);
if (!archiveGitLabResponse.ok) {
const message = `Failed to read tree (archive) from ${url}, ${archiveGitLabResponse.status} ${archiveGitLabResponse.statusText}`;
if (archiveGitLabResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const path = filepath
? `${repoName}-${branch}-${commitSha}/${filepath}/`
: '';
return await this.treeResponseFactory.fromZipArchive({
stream: (archiveGitLabResponse.body as unknown) as Readable,
path,
etag: commitSha,
filter: options?.filter,
});
}
toString() {
const { host, token } = this.options;
const { host, token } = this.config;
return `gitlab{host=${host},authed=${Boolean(token)}}`;
}
}
packages/backend-common/src/reading/UrlReaderPredicateMux.ts
+5 -2
View File
@@ -45,12 +45,15 @@ export class UrlReaderPredicateMux implements UrlReader {
throw new NotAllowedError(`Reading from '${url}' is not allowed`);
}
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse> {
async readTree(
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const parsed = new URL(url);
for (const { predicate, reader } of this.readers) {
if (predicate(parsed)) {
return reader.readTree(url, options);
return await reader.readTree(url, options);
}
}
packages/backend-common/src/reading/__fixtures__/backstage-mock-etag123.tar.gz
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/__fixtures__/gitlab-archive.zip
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/__fixtures__/mock-main.tar.gz
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/__fixtures__/mock-main.zip
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/__fixtures__/repo.tar.gz
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/__fixtures__/repo.zip
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts
+4
View File
@@ -26,6 +26,8 @@ type FromArchiveOptions = {
stream: Readable;
// If set, the root of the tree will be set to the given directory path.
path?: string;
// etag of the blob
etag: string;
// Filter passed on from the ReadTreeOptions
filter?: (path: string) => boolean;
};
@@ -45,6 +47,7 @@ export class ReadTreeResponseFactory {
options.stream,
options.path ?? '',
this.workDir,
options.etag,
options.filter,
);
}
@@ -54,6 +57,7 @@ export class ReadTreeResponseFactory {
options.stream,
options.path ?? '',
this.workDir,
options.etag,
options.filter,
);
}
packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts
+25 -12
View File
@@ -20,7 +20,7 @@ import { resolve as resolvePath } from 'path';
import { TarArchiveResponse } from './TarArchiveResponse';
const archiveData = fs.readFileSync(
resolvePath(__filename, '../../__fixtures__/repo.tar.gz'),
resolvePath(__filename, '../../__fixtures__/mock-main.tar.gz'),
);
describe('TarArchiveResponse', () => {
@@ -38,7 +38,7 @@ describe('TarArchiveResponse', () => {
it('should read files', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp');
const res = new TarArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const files = await res.files();
expect(files).toEqual([
@@ -61,8 +61,12 @@ describe('TarArchiveResponse', () => {
it('should read files with filter', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
const res = new TarArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
);
const files = await res.files();
@@ -79,14 +83,14 @@ describe('TarArchiveResponse', () => {
it('should read as archive and files', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp');
const res = new TarArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const buffer = await res.archive();
await expect(res.archive()).rejects.toThrow(
'Response has already been read',
);
const res2 = new TarArchiveResponse(buffer, '', '/tmp');
const res2 = new TarArchiveResponse(buffer, '', '/tmp', 'etag');
const files = await res2.files();
expect(files).toEqual([
@@ -109,21 +113,26 @@ describe('TarArchiveResponse', () => {
it('should extract entire archive into directory', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, '', '/tmp');
const res = new TarArchiveResponse(stream, '', '/tmp', 'etag');
const dir = await res.dir();
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'),
fs.readFile(resolvePath(dir, 'mock-main/mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'),
fs.readFile(resolvePath(dir, 'mock-main/docs/index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('should extract archive into directory with a subpath', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-repo/docs/', '/tmp');
const res = new TarArchiveResponse(
stream,
'mock-main/docs/',
'/tmp',
'etag',
);
const dir = await res.dir();
expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/);
@@ -135,8 +144,12 @@ describe('TarArchiveResponse', () => {
it('should extract archive into directory with a subpath and filter', async () => {
const stream = fs.createReadStream('/test-archive.tar.gz');
const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
const res = new TarArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
);
const dir = await res.dir({ targetDir: '/tmp' });
packages/backend-common/src/reading/tree/TarArchiveResponse.ts
+3
View File
@@ -41,6 +41,7 @@ export class TarArchiveResponse implements ReadTreeResponse {
private readonly stream: Readable,
private readonly subPath: string,
private readonly workDir: string,
public readonly etag: string,
private readonly filter?: (path: string) => boolean,
) {
if (subPath) {
@@ -53,6 +54,8 @@ export class TarArchiveResponse implements ReadTreeResponse {
);
}
}
this.etag = etag;
}
// Make sure the input stream is only read once
packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts
+31 -18
View File
@@ -20,7 +20,7 @@ import { resolve as resolvePath } from 'path';
import { ZipArchiveResponse } from './ZipArchiveResponse';
const archiveData = fs.readFileSync(
resolvePath(__filename, '../../__fixtures__/repo.zip'),
resolvePath(__filename, '../../__fixtures__/mock-main.zip'),
);
describe('ZipArchiveResponse', () => {
@@ -38,31 +38,35 @@ describe('ZipArchiveResponse', () => {
it('should read files', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp');
const res = new ZipArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const files = await res.files();
expect(files).toEqual([
{
path: 'docs/index.md',
path: 'mkdocs.yml',
content: expect.any(Function),
},
{
path: 'mkdocs.yml',
path: 'docs/index.md',
content: expect.any(Function),
},
]);
const contents = await Promise.all(files.map(f => f.content()));
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
'# Test',
'site_name: Test',
'# Test',
]);
});
it('should read files with filter', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
const res = new ZipArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
);
const files = await res.files();
@@ -79,51 +83,56 @@ describe('ZipArchiveResponse', () => {
it('should read as archive and files', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp');
const res = new ZipArchiveResponse(stream, 'mock-main/', '/tmp', 'etag');
const buffer = await res.archive();
await expect(res.archive()).rejects.toThrow(
'Response has already been read',
);
const res2 = new ZipArchiveResponse(buffer, '', '/tmp');
const res2 = new ZipArchiveResponse(buffer, '', '/tmp', 'etag');
const files = await res2.files();
expect(files).toEqual([
{
path: 'docs/index.md',
path: 'mkdocs.yml',
content: expect.any(Function),
},
{
path: 'mkdocs.yml',
path: 'docs/index.md',
content: expect.any(Function),
},
]);
const contents = await Promise.all(files.map(f => f.content()));
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
'# Test',
'site_name: Test',
'# Test',
]);
});
it('should extract entire archive into directory', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, '', '/tmp');
const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag');
const dir = await res.dir();
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'),
fs.readFile(resolvePath(dir, 'mock-main/mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'),
fs.readFile(resolvePath(dir, 'mock-main/docs/index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('should extract archive into directory with a subpath', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/docs/', '/tmp');
const res = new ZipArchiveResponse(
stream,
'mock-main/docs/',
'/tmp',
'etag',
);
const dir = await res.dir();
expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/);
@@ -135,8 +144,12 @@ describe('ZipArchiveResponse', () => {
it('should extract archive into directory with a subpath and filter', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
const res = new ZipArchiveResponse(
stream,
'mock-main/',
'/tmp',
'etag',
path => path.endsWith('.yml'),
);
const dir = await res.dir({ targetDir: '/tmp' });
packages/backend-common/src/reading/tree/ZipArchiveResponse.ts
+3
View File
@@ -35,6 +35,7 @@ export class ZipArchiveResponse implements ReadTreeResponse {
private readonly stream: Readable,
private readonly subPath: string,
private readonly workDir: string,
public readonly etag: string,
private readonly filter?: (path: string) => boolean,
) {
if (subPath) {
@@ -47,6 +48,8 @@ export class ZipArchiveResponse implements ReadTreeResponse {
);
}
}
this.etag = etag;
}
// Make sure the input stream is only read once
packages/backend-common/src/reading/types.ts
+22
View File
@@ -32,6 +32,19 @@ export type ReadTreeOptions = {
* If no filter is provided all files are extracted.
*/
filter?(path: string): boolean;
/**
* An etag can be provided to check whether readTree's response has changed from a previous execution.
*
* In the readTree() response, an etag is returned along with the tree blob. The etag is a unique identifer
* of the tree blob, usually the commit SHA or etag from the target.
*
* When a etag is given in ReadTreeOptions, readTree will first compare the etag against the etag
* on the target branch. If they match, readTree will throw a NotModifiedError indicating that the readTree
* response will not differ from the previous response which included this particular etag. If they mismatch,
* readTree will return the rest of ReadTreeResponse along with a new etag.
*/
etag?: string;
};
/**
@@ -70,5 +83,14 @@ export type ReadTreeResponseDirOptions = {
export type ReadTreeResponse = {
files(): Promise<ReadTreeResponseFile[]>;
archive(): Promise<NodeJS.ReadableStream>;
/**
* dir() extracts the tree response into a directory and returns the path of the directory.
*/
dir(options?: ReadTreeResponseDirOptions): Promise<string>;
/**
* A unique identifer of the tree blob, usually the commit SHA or etag from the target.
*/
etag: string;
};
packages/integration/src/azure/core.ts
+56
View File
@@ -108,6 +108,62 @@ export function getAzureDownloadUrl(url: string): string {
return `${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`;
}
/**
* Given a URL, return the API URL to fetch commits on the branch.
*
* @param url A URL pointing to a repository or a sub-path
*/
export function getAzureCommitsUrl(url: string): string {
try {
const parsedUrl = new URL(url);
const [
empty,
userOrOrg,
project,
srcKeyword,
repoName,
] = parsedUrl.pathname.split('/');
// Remove the "GB" from "GBmain" for example.
const ref = parsedUrl.searchParams.get('version')?.substr(2);
if (
!!empty ||
!userOrOrg ||
!project ||
srcKeyword !== '_git' ||
!repoName
) {
throw new Error('Wrong Azure Devops URL');
}
// transform to commits api
parsedUrl.pathname = [
empty,
userOrOrg,
project,
'_apis',
'git',
'repositories',
repoName,
'commits',
].join('/');
const queryParams = [];
if (ref) {
queryParams.push(`searchCriteria.itemVersion.version=${ref}`);
}
parsedUrl.search = queryParams.join('&');
parsedUrl.protocol = 'https';
return parsedUrl.toString();
} catch (e) {
throw new Error(`Incorrect URL: ${url}, ${e}`);
}
}
/**
* Gets the request options necessary to make requests to a given provider.
*
packages/integration/src/azure/index.ts
+1
View File
@@ -23,4 +23,5 @@ export {
getAzureDownloadUrl,
getAzureFileFetchUrl,
getAzureRequestOptions,
getAzureCommitsUrl,
} from './core';
packages/integration/src/gitlab/config.test.ts
+13 -1
View File
@@ -43,7 +43,18 @@ describe('readGitLabIntegrationConfig', () => {
const output = readGitLabIntegrationConfig(buildConfig({}));
expect(output).toEqual({
host: 'gitlab.com',
apiBaseUrl: 'gitlab.com/api/v4',
apiBaseUrl: 'https://gitlab.com/api/v4',
});
});
it('injects the correct GitLab API base URL when missing', () => {
const output = readGitLabIntegrationConfig(
buildConfig({ host: 'gitlab.com' }),
);
expect(output).toEqual({
host: 'gitlab.com',
apiBaseUrl: 'https://gitlab.com/api/v4',
});
});
@@ -86,6 +97,7 @@ describe('readGitLabIntegrationConfigs', () => {
expect(output).toEqual([
{
host: 'gitlab.com',
apiBaseUrl: 'https://gitlab.com/api/v4',
},
]);
});
packages/integration/src/gitlab/config.ts
+2 -2
View File
@@ -18,7 +18,7 @@ import { Config } from '@backstage/config';
import { isValidHost } from '../helpers';
const GITLAB_HOST = 'gitlab.com';
const GITLAB_API_BASE_URL = 'gitlab.com/api/v4';
const GITLAB_API_BASE_URL = 'https://gitlab.com/api/v4';
/**
* The configuration parameters for a single GitLab integration.
@@ -89,7 +89,7 @@ export function readGitLabIntegrationConfigs(
// As a convenience we always make sure there's at least an unauthenticated
// reader for public gitlab repos.
if (!result.some(c => c.host === GITLAB_HOST)) {
result.push({ host: GITLAB_HOST });
result.push({ host: GITLAB_HOST, apiBaseUrl: GITLAB_API_BASE_URL });
}
return result;
packages/techdocs-common/src/helpers.test.ts
+1
View File
@@ -135,6 +135,7 @@ describe('getDocFilesFromRepository', () => {
archive: async () => {
return Readable.from('');
},
etag: '',
};
}
}