From 491f3a0ecf89119388e3c1508fc2bd02452956f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 8 Feb 2021 15:06:47 +0100 Subject: [PATCH] backend-common: implement UrlReader.search for the other providers too --- .changeset/angry-walls-mate.md | 5 + .changeset/dry-llamas-wave.md | 7 + .../src/reading/AzureUrlReader.test.ts | 117 ++++++++-- .../src/reading/AzureUrlReader.ts | 71 +++++-- .../src/reading/BitbucketUrlReader.test.ts | 201 ++++++++++++++++-- .../src/reading/BitbucketUrlReader.ts | 92 +++++--- .../src/reading/GithubUrlReader.test.ts | 76 ++++--- .../src/reading/GithubUrlReader.ts | 30 +-- .../src/reading/GitlabUrlReader.test.ts | 131 ++++++++++-- .../src/reading/GitlabUrlReader.ts | 79 ++++--- .../src/reading/tree/TarArchiveResponse.ts | 9 +- .../src/reading/tree/ZipArchiveResponse.ts | 17 +- .../backend-common/src/reading/tree/util.ts | 25 +++ packages/integration/src/ScmIntegrations.ts | 9 +- .../src/azure/AzureIntegration.test.ts | 10 + packages/integration/src/azure/index.ts | 3 +- .../src/bitbucket/BitbucketIntegration.ts | 6 +- packages/integration/src/bitbucket/index.ts | 1 + .../src/github/GitHubIntegration.ts | 6 +- packages/integration/src/github/index.ts | 1 + .../src/gitlab/GitLabIntegration.ts | 6 +- packages/integration/src/gitlab/index.ts | 1 + packages/integration/src/helpers.test.ts | 68 +++++- packages/integration/src/helpers.ts | 41 ++++ packages/integration/src/index.ts | 1 + packages/integration/src/types.ts | 27 ++- 26 files changed, 822 insertions(+), 218 deletions(-) create mode 100644 .changeset/angry-walls-mate.md create mode 100644 .changeset/dry-llamas-wave.md create mode 100644 packages/backend-common/src/reading/tree/util.ts diff --git a/.changeset/angry-walls-mate.md b/.changeset/angry-walls-mate.md new file mode 100644 index 0000000000..5902168b2c --- /dev/null +++ b/.changeset/angry-walls-mate.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': minor +--- + +Make `ScmIntegration.resolveUrl` mandatory. diff --git a/.changeset/dry-llamas-wave.md b/.changeset/dry-llamas-wave.md new file mode 100644 index 0000000000..fea9b122a6 --- /dev/null +++ b/.changeset/dry-llamas-wave.md @@ -0,0 +1,7 @@ +--- +'@backstage/backend-common': patch +--- + +Implement `UrlReader.search` for the other providers (Azure, Bitbucket, GitLab) as well. + +The `UrlReader` subclasses now are implemented in terms of the respective `Integration` class. diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index c937f8a40c..b52097baf5 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -14,18 +14,22 @@ * limitations under the License. */ -import * as os from 'os'; +import { ConfigReader } from '@backstage/config'; +import { + AzureIntegration, + readAzureIntegrationConfig, +} from '@backstage/integration'; +import { msw } from '@backstage/test-utils'; import fs from 'fs-extra'; import mockFs from 'mock-fs'; -import path from 'path'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import { ConfigReader } from '@backstage/config'; +import * as os from 'os'; +import path from 'path'; +import { NotModifiedError } from '../errors'; import { getVoidLogger } from '../logging'; import { AzureUrlReader } from './AzureUrlReader'; -import { msw } from '@backstage/test-utils'; import { ReadTreeResponseFactory } from './tree'; -import { NotModifiedError } from '../errors'; const logger = getVoidLogger(); @@ -36,6 +40,16 @@ const treeResponseFactory = ReadTreeResponseFactory.create({ const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; describe('AzureUrlReader', () => { + beforeEach(() => { + mockFs({ + [tmpDir]: mockFs.directory(), + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + const worker = setupServer(); msw.setupDefaultHandlers(worker); @@ -143,22 +157,18 @@ describe('AzureUrlReader', () => { }); describe('readTree', () => { - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - const repoBuffer = fs.readFileSync( path.resolve('src', 'reading', '__fixtures__', 'mock-main.zip'), ); const processor = new AzureUrlReader( - { host: 'dev.azure.com' }, + new AzureIntegration( + readAzureIntegrationConfig( + new ConfigReader({ + host: 'dev.azure.com', + }), + ), + ), { treeResponseFactory }, ); @@ -257,4 +267,79 @@ describe('AzureUrlReader', () => { expect(indexMarkdownFile.toString()).toBe('# Test\n'); }); }); + + describe('search', () => { + const repoBuffer = fs.readFileSync( + path.resolve('src', 'reading', '__fixtures__', 'mock-main.zip'), + ); + + const processor = new AzureUrlReader( + new AzureIntegration( + readAzureIntegrationConfig( + new ConfigReader({ + host: 'dev.azure.com', + }), + ), + ), + { treeResponseFactory }, + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + rest.get( + // https://docs.microsoft.com/en-us/rest/api/azure/devops/git/commits/get%20commits?view=azure-devops-rest-6.0#on-a-branch + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/commits', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + count: 2, + value: [ + { + commitId: '123abc2', + comment: 'second commit', + }, + { + commitId: '123abc1', + comment: 'first commit', + }, + ], + }), + ), + ), + ); + }); + + it('works for the naive case', async () => { + const result = await processor.search( + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=%2F**%2Findex.*&version=GBmaster', + ); + expect(result.etag).toBe('123abc2'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=%2Fdocs%2Findex.md&version=GBmaster', + ); + await expect(result.files[0].content()).resolves.toEqual( + Buffer.from('# Test\n'), + ); + }); + + it('throws NotModifiedError when same etag', async () => { + await expect( + processor.search( + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=**/index.*&version=GBmaster', + { etag: '123abc2' }, + ), + ).rejects.toThrow(NotModifiedError); + }); + }); }); diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index bc1b61a041..21b988e5b0 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -15,39 +15,41 @@ */ import { - AzureIntegrationConfig, - readAzureIntegrationConfigs, - getAzureFileFetchUrl, - getAzureDownloadUrl, - getAzureRequestOptions, + AzureIntegration, getAzureCommitsUrl, + getAzureDownloadUrl, + getAzureFileFetchUrl, + getAzureRequestOptions, + ScmIntegrations, } from '@backstage/integration'; import fetch from 'cross-fetch'; +import parseGitUrl from 'git-url-parse'; +import { Minimatch } from 'minimatch'; import { Readable } from 'stream'; import { NotFoundError, NotModifiedError } from '../errors'; +import { ReadTreeResponseFactory } from './tree'; +import { stripFirstDirectoryFromPath } from './tree/util'; import { ReaderFactory, ReadTreeOptions, ReadTreeResponse, + SearchOptions, SearchResponse, UrlReader, } from './types'; -import { ReadTreeResponseFactory } from './tree'; export class AzureUrlReader implements UrlReader { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { - const configs = readAzureIntegrationConfigs( - config.getOptionalConfigArray('integrations.azure') ?? [], - ); - return configs.map(options => { - const reader = new AzureUrlReader(options, { treeResponseFactory }); - const predicate = (url: URL) => url.host === options.host; + const integrations = ScmIntegrations.fromConfig(config); + return integrations.azure.list().map(integration => { + const reader = new AzureUrlReader(integration, { treeResponseFactory }); + const predicate = (url: URL) => url.host === integration.config.host; return { reader, predicate }; }); }; constructor( - private readonly options: AzureIntegrationConfig, + private readonly integration: AzureIntegration, private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, ) {} @@ -56,7 +58,10 @@ export class AzureUrlReader implements UrlReader { let response: Response; try { - response = await fetch(builtUrl, getAzureRequestOptions(this.options)); + response = await fetch( + builtUrl, + getAzureRequestOptions(this.integration.config), + ); } catch (e) { throw new Error(`Unable to read ${url}, ${e}`); } @@ -83,7 +88,7 @@ export class AzureUrlReader implements UrlReader { const commitsAzureResponse = await fetch( getAzureCommitsUrl(url), - getAzureRequestOptions(this.options), + getAzureRequestOptions(this.integration.config), ); if (!commitsAzureResponse.ok) { const message = `Failed to read tree from ${url}, ${commitsAzureResponse.status} ${commitsAzureResponse.statusText}`; @@ -100,7 +105,9 @@ export class AzureUrlReader implements UrlReader { const archiveAzureResponse = await fetch( getAzureDownloadUrl(url), - getAzureRequestOptions(this.options, { Accept: 'application/zip' }), + getAzureRequestOptions(this.integration.config, { + Accept: 'application/zip', + }), ); if (!archiveAzureResponse.ok) { const message = `Failed to read tree from ${url}, ${archiveAzureResponse.status} ${archiveAzureResponse.statusText}`; @@ -117,12 +124,38 @@ export class AzureUrlReader implements UrlReader { }); } - async search(): Promise { - throw new Error('AzureUrlReader does not implement search'); + async search(url: string, options?: SearchOptions): Promise { + const { filepath } = parseGitUrl(url); + const matcher = new Minimatch(filepath); + + // TODO(freben): For now, read the entire repo and filter through that. In + // a future improvement, we could be smart and try to deduce that non-glob + // prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used + // to get just that part of the repo. + const treeUrl = new URL(url); + treeUrl.searchParams.delete('path'); + treeUrl.pathname = treeUrl.pathname.replace(/\/+$/, ''); + + const tree = await this.readTree(treeUrl.toString(), { + etag: options?.etag, + filter: path => matcher.match(stripFirstDirectoryFromPath(path)), + }); + const files = await tree.files(); + + return { + etag: tree.etag, + files: files.map(file => ({ + url: this.integration.resolveUrl({ + url: `/${file.path}`, + base: url, + }), + content: file.content, + })), + }; } toString() { - const { host, token } = this.options; + const { host, token } = this.integration.config; return `azure{host=${host},authed=${Boolean(token)}}`; } } diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 974c84b2c2..df9febb352 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -15,11 +15,16 @@ */ import { ConfigReader } from '@backstage/config'; +import { + BitbucketIntegration, + readBitbucketIntegrationConfig, +} from '@backstage/integration'; import { msw } from '@backstage/test-utils'; import fs from 'fs-extra'; import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; +import os from 'os'; import path from 'path'; import { NotModifiedError } from '../errors'; import { BitbucketUrlReader } from './BitbucketUrlReader'; @@ -30,19 +35,45 @@ const treeResponseFactory = ReadTreeResponseFactory.create({ }); const bitbucketProcessor = new BitbucketUrlReader( - { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' }, + new BitbucketIntegration( + readBitbucketIntegrationConfig( + new ConfigReader({ + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + }), + ), + ), { treeResponseFactory }, ); const hostedBitbucketProcessor = new BitbucketUrlReader( - { - host: 'bitbucket.mycompany.net', - apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', - }, + new BitbucketIntegration( + readBitbucketIntegrationConfig( + new ConfigReader({ + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', + }), + ), + ), { treeResponseFactory }, ); +const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; + describe('BitbucketUrlReader', () => { + beforeEach(() => { + mockFs({ + [tmpDir]: mockFs.directory(), + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + const worker = setupServer(); + msw.setupDefaultHandlers(worker); + describe('implementation', () => { it('rejects unknown targets', async () => { await expect( @@ -54,19 +85,6 @@ describe('BitbucketUrlReader', () => { }); describe('readTree', () => { - beforeEach(() => { - mockFs({ - '/tmp': mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - - const worker = setupServer(); - msw.setupDefaultHandlers(worker); - const repoBuffer = fs.readFileSync( path.resolve( 'src', @@ -247,12 +265,153 @@ describe('BitbucketUrlReader', () => { expect(() => { /* eslint-disable no-new */ new BitbucketUrlReader( - { - host: 'bitbucket.mycompany.net', - }, + new BitbucketIntegration( + readBitbucketIntegrationConfig( + new ConfigReader({ + host: 'bitbucket.mycompany.net', + }), + ), + ), { treeResponseFactory }, ); }).toThrowError('must configure an explicit apiBaseUrl'); }); }); + + describe('search hosted', () => { + const repoBuffer = fs.readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'bitbucket-repo-with-commit-hash.zip', + ), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage/mock', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + mainbranch: { + type: 'branch', + name: 'master', + }, + }), + ), + ), + rest.get( + 'https://bitbucket.org/backstage/mock/get/master.zip', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-12ab34cd56ef.zip', + ), + ctx.body(repoBuffer), + ), + ), + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], + }), + ), + ), + ); + }); + + it('works for the naive case', async () => { + const result = await bitbucketProcessor.search( + 'https://bitbucket.org/backstage/mock/src/master/**/index.*', + ); + expect(result.etag).toBe('12ab34cd56ef'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.org/backstage/mock/src/master/docs/index.md', + ); + await expect(result.files[0].content()).resolves.toEqual( + Buffer.from('# Test\n'), + ); + }); + + it('throws NotModifiedError when same etag', async () => { + await expect( + bitbucketProcessor.search( + 'https://bitbucket.org/backstage/mock/src/master/**/index.*', + { etag: '12ab34cd56ef' }, + ), + ).rejects.toThrow(NotModifiedError); + }); + }); + + describe('search private', () => { + const privateBitbucketRepoBuffer = fs.readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'bitbucket-server-repo.zip', + ), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock.zip', + ), + ctx.body(privateBitbucketRepoBuffer), + ), + ), + rest.get( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/commits', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + values: [{ id: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], + }), + ), + ), + ); + }); + + it('works for the naive case', async () => { + const result = await hostedBitbucketProcessor.search( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master', + ); + expect(result.etag).toBe('12ab34cd56ef'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master', + ); + await expect(result.files[0].content()).resolves.toEqual( + Buffer.from('# Test\n'), + ); + }); + + it('throws NotModifiedError when same etag', async () => { + await expect( + hostedBitbucketProcessor.search( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master', + { etag: '12ab34cd56ef' }, + ), + ).rejects.toThrow(NotModifiedError); + }); + }); }); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index e51c03e562..3954e7ee4a 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -15,22 +15,25 @@ */ import { - BitbucketIntegrationConfig, + BitbucketIntegration, getBitbucketDefaultBranch, getBitbucketDownloadUrl, getBitbucketFileFetchUrl, getBitbucketRequestOptions, - readBitbucketIntegrationConfigs, + ScmIntegrations, } from '@backstage/integration'; import fetch from 'cross-fetch'; import parseGitUrl from 'git-url-parse'; +import { Minimatch } from 'minimatch'; import { Readable } from 'stream'; import { NotFoundError, NotModifiedError } from '../errors'; import { ReadTreeResponseFactory } from './tree'; +import { stripFirstDirectoryFromPath } from './tree/util'; import { ReaderFactory, ReadTreeOptions, ReadTreeResponse, + SearchOptions, SearchResponse, UrlReader, } from './types'; @@ -40,45 +43,43 @@ import { * the one exposed by Bitbucket Cloud itself. */ export class BitbucketUrlReader implements UrlReader { - private readonly config: BitbucketIntegrationConfig; - private readonly treeResponseFactory: ReadTreeResponseFactory; - static factory: ReaderFactory = ({ config, treeResponseFactory }) => { - const configs = readBitbucketIntegrationConfigs( - config.getOptionalConfigArray('integrations.bitbucket') ?? [], - ); - return configs.map(provider => { - const reader = new BitbucketUrlReader(provider, { treeResponseFactory }); - const predicate = (url: URL) => url.host === provider.host; + const integrations = ScmIntegrations.fromConfig(config); + return integrations.bitbucket.list().map(integration => { + const reader = new BitbucketUrlReader(integration, { + treeResponseFactory, + }); + const predicate = (url: URL) => url.host === integration.config.host; return { reader, predicate }; }); }; constructor( - config: BitbucketIntegrationConfig, - deps: { treeResponseFactory: ReadTreeResponseFactory }, + private readonly integration: BitbucketIntegration, + private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, ) { - const { host, apiBaseUrl, token, username, appPassword } = config; + const { + host, + apiBaseUrl, + token, + username, + appPassword, + } = integration.config; if (!apiBaseUrl) { throw new Error( `Bitbucket integration for '${host}' must configure an explicit apiBaseUrl`, ); - } - - if (!token && username && !appPassword) { + } else if (!token && username && !appPassword) { throw new Error( `Bitbucket integration for '${host}' has configured a username but is missing a required appPassword.`, ); } - - this.config = config; - this.treeResponseFactory = deps.treeResponseFactory; } async read(url: string): Promise { - const bitbucketUrl = getBitbucketFileFetchUrl(url, this.config); - const options = getBitbucketRequestOptions(this.config); + const bitbucketUrl = getBitbucketFileFetchUrl(url, this.integration.config); + const options = getBitbucketRequestOptions(this.integration.config); let response: Response; try { @@ -109,10 +110,13 @@ export class BitbucketUrlReader implements UrlReader { throw new NotModifiedError(); } - const downloadUrl = await getBitbucketDownloadUrl(url, this.config); + const downloadUrl = await getBitbucketDownloadUrl( + url, + this.integration.config, + ); const archiveBitbucketResponse = await fetch( downloadUrl, - getBitbucketRequestOptions(this.config), + getBitbucketRequestOptions(this.integration.config), ); if (!archiveBitbucketResponse.ok) { const message = `Failed to read tree from ${url}, ${archiveBitbucketResponse.status} ${archiveBitbucketResponse.statusText}`; @@ -122,7 +126,7 @@ export class BitbucketUrlReader implements UrlReader { throw new Error(message); } - return await this.treeResponseFactory.fromZipArchive({ + return await this.deps.treeResponseFactory.fromZipArchive({ stream: (archiveBitbucketResponse.body as unknown) as Readable, subpath: filepath, etag: lastCommitShortHash, @@ -130,12 +134,36 @@ export class BitbucketUrlReader implements UrlReader { }); } - async search(): Promise { - throw new Error('BitbucketUrlReader does not implement search'); + async search(url: string, options?: SearchOptions): Promise { + const { filepath } = parseGitUrl(url); + const matcher = new Minimatch(filepath); + + // TODO(freben): For now, read the entire repo and filter through that. In + // a future improvement, we could be smart and try to deduce that non-glob + // prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used + // to get just that part of the repo. + const treeUrl = url.replace(filepath, '').replace(/\/+$/, ''); + + const tree = await this.readTree(treeUrl, { + etag: options?.etag, + filter: path => matcher.match(stripFirstDirectoryFromPath(path)), + }); + const files = await tree.files(); + + return { + etag: tree.etag, + files: files.map(file => ({ + url: this.integration.resolveUrl({ + url: `/${file.path}`, + base: url, + }), + content: file.content, + })), + }; } toString() { - const { host, token, username, appPassword } = this.config; + const { host, token, username, appPassword } = this.integration.config; let authed = Boolean(token); if (!authed) { authed = Boolean(username && appPassword); @@ -148,18 +176,18 @@ export class BitbucketUrlReader implements UrlReader { let branch = ref; if (!branch) { - branch = await getBitbucketDefaultBranch(url, this.config); + branch = await getBitbucketDefaultBranch(url, this.integration.config); } const isHosted = resource === 'bitbucket.org'; // Bitbucket Server https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp222 const commitsApiUrl = isHosted - ? `${this.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}` - : `${this.config.apiBaseUrl}/projects/${project}/repos/${repoName}/commits`; + ? `${this.integration.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}` + : `${this.integration.config.apiBaseUrl}/projects/${project}/repos/${repoName}/commits`; const commitsResponse = await fetch( commitsApiUrl, - getBitbucketRequestOptions(this.config), + getBitbucketRequestOptions(this.integration.config), ); if (!commitsResponse.ok) { const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`; diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index d4e579b143..72fbf83f6d 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -15,12 +15,17 @@ */ import { ConfigReader } from '@backstage/config'; -import { GithubCredentialsProvider } from '@backstage/integration'; +import { + GithubCredentialsProvider, + GitHubIntegration, + readGitHubIntegrationConfig, +} from '@backstage/integration'; import { msw } from '@backstage/test-utils'; import fs from 'fs-extra'; import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; +import os from 'os'; import path from 'path'; import { NotFoundError, NotModifiedError } from '../errors'; import { @@ -41,26 +46,45 @@ const mockCredentialsProvider = ({ } as unknown) as GithubCredentialsProvider; const githubProcessor = new GithubUrlReader( - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }, + new GitHubIntegration( + readGitHubIntegrationConfig( + new ConfigReader({ + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + }), + ), + ), { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, ); const gheProcessor = new GithubUrlReader( - { - host: 'ghe.github.com', - apiBaseUrl: 'https://ghe.github.com/api/v3', - }, + new GitHubIntegration( + readGitHubIntegrationConfig( + new ConfigReader({ + host: 'ghe.github.com', + apiBaseUrl: 'https://ghe.github.com/api/v3', + }), + ), + ), { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, ); +const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; + describe('GithubUrlReader', () => { const worker = setupServer(); - msw.setupDefaultHandlers(worker); + beforeEach(() => { + mockFs({ + [tmpDir]: mockFs.directory(), + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + beforeEach(() => { jest.clearAllMocks(); }); @@ -94,7 +118,7 @@ describe('GithubUrlReader', () => { worker.use( rest.get( - 'https://api.github.com/repos/backstage/mock/tree/contents/?ref=main', + 'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/?ref=main', (req, res, ctx) => { expect(req.headers.get('authorization')).toBe( mockHeaders.Authorization, @@ -111,7 +135,7 @@ describe('GithubUrlReader', () => { ), ); - await githubProcessor.read( + await gheProcessor.read( 'https://github.com/backstage/mock/tree/blob/main', ); }); @@ -122,16 +146,6 @@ describe('GithubUrlReader', () => { */ describe('readTree', () => { - beforeEach(() => { - mockFs({ - '/tmp': mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - const repoBuffer = fs.readFileSync( path.resolve( 'src', @@ -397,9 +411,13 @@ describe('GithubUrlReader', () => { expect(() => { /* eslint-disable no-new */ new GithubUrlReader( - { - host: 'ghe.mycompany.net', - }, + new GitHubIntegration( + readGitHubIntegrationConfig( + new ConfigReader({ + host: 'ghe.mycompany.net', + }), + ), + ), { treeResponseFactory, credentialsProvider: mockCredentialsProvider, @@ -414,14 +432,6 @@ describe('GithubUrlReader', () => { */ describe('search', () => { - beforeEach(() => { - mockFs({ '/tmp': mockFs.directory() }); - }); - - afterEach(() => { - mockFs.restore(); - }); - const repoBuffer = fs.readFileSync( path.resolve( 'src', diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 291740a9fc..ac5a93bd5f 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -17,8 +17,8 @@ import { getGitHubFileFetchUrl, GithubCredentialsProvider, - GitHubIntegrationConfig, - readGitHubIntegrationConfigs, + GitHubIntegration, + ScmIntegrations, } from '@backstage/integration'; import { RestEndpointMethodTypes } from '@octokit/rest'; import fetch from 'cross-fetch'; @@ -48,36 +48,36 @@ export type GhBlobResponse = RestEndpointMethodTypes['git']['getBlob']['response */ export class GithubUrlReader implements UrlReader { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { - const configs = readGitHubIntegrationConfigs( - config.getOptionalConfigArray('integrations.github') ?? [], - ); - return configs.map(provider => { - const credentialsProvider = GithubCredentialsProvider.create(provider); - const reader = new GithubUrlReader(provider, { + const integrations = ScmIntegrations.fromConfig(config); + return integrations.github.list().map(integration => { + const credentialsProvider = GithubCredentialsProvider.create( + integration.config, + ); + const reader = new GithubUrlReader(integration, { treeResponseFactory, credentialsProvider, }); - const predicate = (url: URL) => url.host === provider.host; + const predicate = (url: URL) => url.host === integration.config.host; return { reader, predicate }; }); }; constructor( - private readonly config: GitHubIntegrationConfig, + private readonly integration: GitHubIntegration, private readonly deps: { treeResponseFactory: ReadTreeResponseFactory; credentialsProvider: GithubCredentialsProvider; }, ) { - if (!config.apiBaseUrl && !config.rawBaseUrl) { + if (!integration.config.apiBaseUrl && !integration.config.rawBaseUrl) { throw new Error( - `GitHub integration for '${config.host}' must configure an explicit apiBaseUrl and rawBaseUrl`, + `GitHub integration '${integration.title}' must configure an explicit apiBaseUrl or rawBaseUrl`, ); } } async read(url: string): Promise { - const ghUrl = getGitHubFileFetchUrl(url, this.config); + const ghUrl = getGitHubFileFetchUrl(url, this.integration.config); const { headers } = await this.deps.credentialsProvider.getCredentials({ url, }); @@ -155,7 +155,7 @@ export class GithubUrlReader implements UrlReader { } toString() { - const { host, token } = this.config; + const { host, token } = this.integration.config; return `github{host=${host},authed=${Boolean(token)}}`; } @@ -258,7 +258,7 @@ export class GithubUrlReader implements UrlReader { }); const repo: GhRepoResponse = await this.fetchJson( - `${this.config.apiBaseUrl}/repos/${full_name}`, + `${this.integration.config.apiBaseUrl}/repos/${full_name}`, { headers }, ); diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 5b0b4002a3..636540caa8 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -20,11 +20,16 @@ import fs from 'fs-extra'; import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; +import os from 'os'; import path from 'path'; import { getVoidLogger } from '../logging'; import { GitlabUrlReader } from './GitlabUrlReader'; import { ReadTreeResponseFactory } from './tree'; import { NotModifiedError, NotFoundError } from '../errors'; +import { + GitLabIntegration, + readGitLabIntegrationConfig, +} from '@backstage/integration'; const logger = getVoidLogger(); @@ -33,24 +38,44 @@ const treeResponseFactory = ReadTreeResponseFactory.create({ }); const gitlabProcessor = new GitlabUrlReader( - { - host: 'gitlab.com', - apiBaseUrl: 'https://gitlab.com/api/v4', - baseUrl: 'https://gitlab.com', - }, + new GitLabIntegration( + readGitLabIntegrationConfig( + new ConfigReader({ + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + baseUrl: 'https://gitlab.com', + }), + ), + ), { treeResponseFactory }, ); const hostedGitlabProcessor = new GitlabUrlReader( - { - host: 'gitlab.mycompany.com', - apiBaseUrl: 'https://gitlab.mycompany.com/api/v4', - baseUrl: 'https://gitlab.mycompany.com', - }, + new GitLabIntegration( + readGitLabIntegrationConfig( + new ConfigReader({ + host: 'gitlab.mycompany.com', + apiBaseUrl: 'https://gitlab.mycompany.com/api/v4', + baseUrl: 'https://gitlab.mycompany.com', + }), + ), + ), { treeResponseFactory }, ); +const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; + describe('GitlabUrlReader', () => { + beforeEach(() => { + mockFs({ + [tmpDir]: mockFs.directory(), + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + const worker = setupServer(); msw.setupDefaultHandlers(worker); @@ -156,16 +181,6 @@ describe('GitlabUrlReader', () => { }); describe('readTree', () => { - beforeEach(() => { - mockFs({ - '/tmp': mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - const archiveBuffer = fs.readFileSync( path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.zip'), ); @@ -382,4 +397,80 @@ describe('GitlabUrlReader', () => { await expect(fnGithub).rejects.toThrow(NotFoundError); }); }); + + describe('search', () => { + const archiveBuffer = fs.readFileSync( + path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.zip'), + ); + + const projectGitlabApiResponse = { + id: 11111111, + default_branch: 'main', + }; + + const branchGitlabApiResponse = { + commit: { + id: 'sha123abc', + }, + }; + + beforeEach(() => { + worker.use( + rest.get( + 'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive.zip?sha=main', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename="mock-main-sha123abc.zip"', + ), + ctx.body(archiveBuffer), + ), + ), + rest.get( + 'https://gitlab.com/api/v4/projects/backstage%2Fmock', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(projectGitlabApiResponse), + ), + ), + rest.get( + 'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/branches/main', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(branchGitlabApiResponse), + ), + ), + ); + }); + + it('works for the naive case', async () => { + const result = await gitlabProcessor.search( + 'https://gitlab.com/backstage/mock/tree/main/**/index.*', + ); + expect(result.etag).toBe('sha123abc'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://gitlab.com/backstage/mock/tree/main/docs/index.md', + ); + await expect(result.files[0].content()).resolves.toEqual( + Buffer.from('# Test\n'), + ); + }); + + it('throws NotModifiedError when same etag', async () => { + await expect( + gitlabProcessor.search( + 'https://gitlab.com/backstage/mock/tree/main/**/index.*', + { etag: 'sha123abc' }, + ), + ).rejects.toThrow(NotModifiedError); + }); + }); }); diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 66030e30d9..66ceae05c6 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -17,49 +17,51 @@ import { getGitLabFileFetchUrl, getGitLabRequestOptions, - GitLabIntegrationConfig, - readGitLabIntegrationConfigs, + GitLabIntegration, + ScmIntegrations, } from '@backstage/integration'; import fetch from 'cross-fetch'; import parseGitUrl from 'git-url-parse'; +import { Minimatch } from 'minimatch'; import { Readable } from 'stream'; import { NotFoundError, NotModifiedError } from '../errors'; import { ReadTreeResponseFactory } from './tree'; +import { stripFirstDirectoryFromPath } from './tree/util'; import { ReaderFactory, ReadTreeOptions, ReadTreeResponse, + SearchOptions, SearchResponse, UrlReader, } from './types'; export class GitlabUrlReader implements UrlReader { - private readonly treeResponseFactory: ReadTreeResponseFactory; - static factory: ReaderFactory = ({ config, treeResponseFactory }) => { - const configs = readGitLabIntegrationConfigs( - config.getOptionalConfigArray('integrations.gitlab') ?? [], - ); - return configs.map(provider => { - const reader = new GitlabUrlReader(provider, { treeResponseFactory }); - const predicate = (url: URL) => url.host === provider.host; + const integrations = ScmIntegrations.fromConfig(config); + return integrations.gitlab.list().map(integration => { + const reader = new GitlabUrlReader(integration, { + treeResponseFactory, + }); + const predicate = (url: URL) => url.host === integration.config.host; return { reader, predicate }; }); }; constructor( - private readonly config: GitLabIntegrationConfig, - deps: { treeResponseFactory: ReadTreeResponseFactory }, - ) { - this.treeResponseFactory = deps.treeResponseFactory; - } + private readonly integration: GitLabIntegration, + private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, + ) {} async read(url: string): Promise { - const builtUrl = await getGitLabFileFetchUrl(url, this.config); + const builtUrl = await getGitLabFileFetchUrl(url, this.integration.config); let response: Response; try { - response = await fetch(builtUrl, getGitLabRequestOptions(this.config)); + response = await fetch( + builtUrl, + getGitLabRequestOptions(this.integration.config), + ); } catch (e) { throw new Error(`Unable to read ${url}, ${e}`); } @@ -86,9 +88,11 @@ export class GitlabUrlReader implements UrlReader { // https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding const projectGitlabResponse = await fetch( new URL( - `${this.config.apiBaseUrl}/projects/${encodeURIComponent(full_name)}`, + `${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent( + full_name, + )}`, ).toString(), - getGitLabRequestOptions(this.config), + getGitLabRequestOptions(this.integration.config), ); if (!projectGitlabResponse.ok) { const msg = `Failed to read tree from ${url}, ${projectGitlabResponse.status} ${projectGitlabResponse.statusText}`; @@ -106,11 +110,11 @@ export class GitlabUrlReader implements UrlReader { // the provided sha. const branchGitlabResponse = await fetch( new URL( - `${this.config.apiBaseUrl}/projects/${encodeURIComponent( + `${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent( full_name, )}/repository/branches/${branch}`, ).toString(), - getGitLabRequestOptions(this.config), + getGitLabRequestOptions(this.integration.config), ); if (!branchGitlabResponse.ok) { const message = `Failed to read tree (branch) from ${url}, ${branchGitlabResponse.status} ${branchGitlabResponse.statusText}`; @@ -128,10 +132,10 @@ export class GitlabUrlReader implements UrlReader { // https://docs.gitlab.com/ee/api/repositories.html#get-file-archive const archiveGitLabResponse = await fetch( - `${this.config.apiBaseUrl}/projects/${encodeURIComponent( + `${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent( full_name, )}/repository/archive.zip?sha=${branch}`, - getGitLabRequestOptions(this.config), + getGitLabRequestOptions(this.integration.config), ); if (!archiveGitLabResponse.ok) { const message = `Failed to read tree (archive) from ${url}, ${archiveGitLabResponse.status} ${archiveGitLabResponse.statusText}`; @@ -141,7 +145,7 @@ export class GitlabUrlReader implements UrlReader { throw new Error(message); } - return await this.treeResponseFactory.fromZipArchive({ + return await this.deps.treeResponseFactory.fromZipArchive({ stream: (archiveGitLabResponse.body as unknown) as Readable, subpath: filepath, etag: commitSha, @@ -149,12 +153,33 @@ export class GitlabUrlReader implements UrlReader { }); } - async search(): Promise { - throw new Error('GitlabUrlReader does not implement search'); + async search(url: string, options?: SearchOptions): Promise { + const { filepath } = parseGitUrl(url); + const matcher = new Minimatch(filepath); + + // TODO(freben): For now, read the entire repo and filter through that. In + // a future improvement, we could be smart and try to deduce that non-glob + // prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used + // to get just that part of the repo. + const treeUrl = url.replace(filepath, '').replace(/\/+$/, ''); + + const tree = await this.readTree(treeUrl, { + etag: options?.etag, + filter: path => matcher.match(stripFirstDirectoryFromPath(path)), + }); + const files = await tree.files(); + + return { + etag: tree.etag, + files: files.map(file => ({ + url: this.integration.resolveUrl({ url: `/${file.path}`, base: url }), + content: file.content, + })), + }; } toString() { - const { host, token } = this.config; + const { host, token } = this.integration.config; return `gitlab{host=${host},authed=${Boolean(token)}}`; } } diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts index d22529293f..e61a1df645 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -25,15 +25,12 @@ import { ReadTreeResponseDirOptions, ReadTreeResponseFile, } from '../types'; +import { stripFirstDirectoryFromPath } from './util'; // Tar types for `Parse` is not a proper constructor, but it should be const TarParseStream = (Parse as unknown) as { new (): ParseStream }; const pipeline = promisify(pipelineCb); -// Matches a directory name + one `/` at the start of any string, -// containing any character except `/` one or more times, and ending with a `/` -// e.g. Will match `dirA/` in `dirA/dirB/file.ext` -const directoryNameRegex = /^[^\/]+\//; /** * Wraps a tar archive stream into a tree response reader. @@ -84,7 +81,7 @@ export class TarArchiveResponse implements ReadTreeResponse { // File path relative to the root extracted directory. Will remove the // top level dir name from the path since its name is hard to predetermine. - const relativePath = entry.path.replace(directoryNameRegex, ''); + const relativePath = stripFirstDirectoryFromPath(entry.path); if (this.subPath) { if (!relativePath.startsWith(this.subPath)) { @@ -161,7 +158,7 @@ export class TarArchiveResponse implements ReadTreeResponse { filter: path => { // File path relative to the root extracted directory. Will remove the // top level dir name from the path since its name is hard to predetermine. - const relativePath = path.replace(directoryNameRegex, ''); + const relativePath = stripFirstDirectoryFromPath(path); if (this.subPath && !relativePath.startsWith(this.subPath)) { return false; } diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 2547c7b0e4..4aebff5c84 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -24,11 +24,7 @@ import { ReadTreeResponseDirOptions, ReadTreeResponseFile, } from '../types'; - -// Matches a directory name + one `/` at the start of any string, -// containing any character except / one or more times, and ending with a `/` -// e.g. Will match `dirA/` in `dirA/dirB/file.ext` -const directoryNameRegex = /^[^\/]+\//; +import { stripFirstDirectoryFromPath } from './util'; /** * Wraps a zip archive stream into a tree response reader. @@ -65,18 +61,13 @@ export class ZipArchiveResponse implements ReadTreeResponse { this.read = true; } - // Will remove the top level dir name from the path since its name is hard to predetermine. - private stripTopDirectory(path: string): string { - return path.replace(directoryNameRegex, ''); - } - // File path relative to the root extracted directory or a sub directory if subpath is set. private getInnerPath(path: string): string { return path.slice(this.subPath.length); } private shouldBeIncluded(entry: Entry): boolean { - const strippedPath = this.stripTopDirectory(entry.path); + const strippedPath = stripFirstDirectoryFromPath(entry.path); if (this.subPath) { if (!strippedPath.startsWith(this.subPath)) { @@ -104,7 +95,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { if (this.shouldBeIncluded(entry)) { files.push({ - path: this.getInnerPath(this.stripTopDirectory(entry.path)), + path: this.getInnerPath(stripFirstDirectoryFromPath(entry.path)), content: () => entry.buffer(), }); } else { @@ -153,7 +144,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { // as a zip can have files with directories without directory entries if (entry.type === 'File' && this.shouldBeIncluded(entry)) { const entryPath = this.getInnerPath( - this.stripTopDirectory(entry.path), + stripFirstDirectoryFromPath(entry.path), ); const dirname = platformPath.dirname(entryPath); if (dirname) { diff --git a/packages/backend-common/src/reading/tree/util.ts b/packages/backend-common/src/reading/tree/util.ts new file mode 100644 index 0000000000..8e908a5e60 --- /dev/null +++ b/packages/backend-common/src/reading/tree/util.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Matches a directory name + one `/` at the start of any string, +// containing any character except `/` one or more times, and ending with a `/` +// e.g. Will match `dirA/` in `dirA/dirB/file.ext` +const directoryNameRegex = /^[^\/]+\//; + +// Removes the first segment of a forward-slash-separated path +export function stripFirstDirectoryFromPath(path: string): string { + return path.replace(directoryNameRegex, ''); +} diff --git a/packages/integration/src/ScmIntegrations.ts b/packages/integration/src/ScmIntegrations.ts index c20fdcef9c..dddf6ed5d0 100644 --- a/packages/integration/src/ScmIntegrations.ts +++ b/packages/integration/src/ScmIntegrations.ts @@ -19,6 +19,7 @@ import { AzureIntegration } from './azure/AzureIntegration'; import { BitbucketIntegration } from './bitbucket/BitbucketIntegration'; import { GitHubIntegration } from './github/GitHubIntegration'; import { GitLabIntegration } from './gitlab/GitLabIntegration'; +import { defaultScmResolveUrl } from './helpers'; import { ScmIntegration, ScmIntegrationRegistry, @@ -83,11 +84,11 @@ export class ScmIntegrations implements ScmIntegrationRegistry { } resolveUrl(options: { url: string; base: string }): string { - const resolve = this.byUrl(options.base)?.resolveUrl; - if (!resolve) { - return new URL(options.url, options.base).toString(); + const integration = this.byUrl(options.base); + if (!integration) { + return defaultScmResolveUrl(options); } - return resolve(options); + return integration.resolveUrl(options); } } diff --git a/packages/integration/src/azure/AzureIntegration.test.ts b/packages/integration/src/azure/AzureIntegration.test.ts index 8a10e40bcf..3f9e165b90 100644 --- a/packages/integration/src/azure/AzureIntegration.test.ts +++ b/packages/integration/src/azure/AzureIntegration.test.ts @@ -58,6 +58,16 @@ describe('AzureIntegration', () => { 'https://dev.azure.com/organization/project/_git/repository?path=%2Fa.yaml', ); + expect( + integration.resolveUrl({ + url: '/a.yaml', + base: + 'https://dev.azure.com/organization/project/_git/repository?path=%2Ffolder%2Fcatalog-info.yaml', + }), + ).toBe( + 'https://dev.azure.com/organization/project/_git/repository?path=%2Fa.yaml', + ); + expect( integration.resolveUrl({ url: './a.yaml', diff --git a/packages/integration/src/azure/index.ts b/packages/integration/src/azure/index.ts index 6d57437779..2d95ef19b9 100644 --- a/packages/integration/src/azure/index.ts +++ b/packages/integration/src/azure/index.ts @@ -14,14 +14,15 @@ * limitations under the License. */ +export { AzureIntegration } from './AzureIntegration'; export { readAzureIntegrationConfig, readAzureIntegrationConfigs, } from './config'; export type { AzureIntegrationConfig } from './config'; export { + getAzureCommitsUrl, getAzureDownloadUrl, getAzureFileFetchUrl, getAzureRequestOptions, - getAzureCommitsUrl, } from './core'; diff --git a/packages/integration/src/bitbucket/BitbucketIntegration.ts b/packages/integration/src/bitbucket/BitbucketIntegration.ts index f3e69b946a..10b69877e4 100644 --- a/packages/integration/src/bitbucket/BitbucketIntegration.ts +++ b/packages/integration/src/bitbucket/BitbucketIntegration.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { basicIntegrations } from '../helpers'; +import { basicIntegrations, defaultScmResolveUrl } from '../helpers'; import { ScmIntegration, ScmIntegrationsFactory } from '../types'; import { BitbucketIntegrationConfig, @@ -47,4 +47,8 @@ export class BitbucketIntegration implements ScmIntegration { get config(): BitbucketIntegrationConfig { return this.integrationConfig; } + + resolveUrl(options: { url: string; base: string }): string { + return defaultScmResolveUrl(options); + } } diff --git a/packages/integration/src/bitbucket/index.ts b/packages/integration/src/bitbucket/index.ts index 9df4d3d3fe..124fe4a2a2 100644 --- a/packages/integration/src/bitbucket/index.ts +++ b/packages/integration/src/bitbucket/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +export { BitbucketIntegration } from './BitbucketIntegration'; export { readBitbucketIntegrationConfig, readBitbucketIntegrationConfigs, diff --git a/packages/integration/src/github/GitHubIntegration.ts b/packages/integration/src/github/GitHubIntegration.ts index c103597d74..c60ab462c9 100644 --- a/packages/integration/src/github/GitHubIntegration.ts +++ b/packages/integration/src/github/GitHubIntegration.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { basicIntegrations } from '../helpers'; +import { basicIntegrations, defaultScmResolveUrl } from '../helpers'; import { ScmIntegration, ScmIntegrationsFactory } from '../types'; import { GitHubIntegrationConfig, @@ -45,4 +45,8 @@ export class GitHubIntegration implements ScmIntegration { get config(): GitHubIntegrationConfig { return this.integrationConfig; } + + resolveUrl(options: { url: string; base: string }): string { + return defaultScmResolveUrl(options); + } } diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index 6491e8dcc5..99d2f56d0f 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -21,3 +21,4 @@ export { export type { GitHubIntegrationConfig } from './config'; export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core'; export { GithubCredentialsProvider } from './GithubCredentialsProvider'; +export { GitHubIntegration } from './GitHubIntegration'; diff --git a/packages/integration/src/gitlab/GitLabIntegration.ts b/packages/integration/src/gitlab/GitLabIntegration.ts index d939917366..131825edb5 100644 --- a/packages/integration/src/gitlab/GitLabIntegration.ts +++ b/packages/integration/src/gitlab/GitLabIntegration.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { basicIntegrations } from '../helpers'; +import { basicIntegrations, defaultScmResolveUrl } from '../helpers'; import { ScmIntegration, ScmIntegrationsFactory } from '../types'; import { GitLabIntegrationConfig, @@ -45,4 +45,8 @@ export class GitLabIntegration implements ScmIntegration { get config(): GitLabIntegrationConfig { return this.integrationConfig; } + + resolveUrl(options: { url: string; base: string }): string { + return defaultScmResolveUrl(options); + } } diff --git a/packages/integration/src/gitlab/index.ts b/packages/integration/src/gitlab/index.ts index 8dc4e90764..8886e0e3ce 100644 --- a/packages/integration/src/gitlab/index.ts +++ b/packages/integration/src/gitlab/index.ts @@ -20,3 +20,4 @@ export { } from './config'; export type { GitLabIntegrationConfig } from './config'; export { getGitLabFileFetchUrl, getGitLabRequestOptions } from './core'; +export { GitLabIntegration } from './GitLabIntegration'; diff --git a/packages/integration/src/helpers.test.ts b/packages/integration/src/helpers.test.ts index 9f4c531ba6..2899f2ed98 100644 --- a/packages/integration/src/helpers.test.ts +++ b/packages/integration/src/helpers.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { isValidHost } from './helpers'; +import { defaultScmResolveUrl, isValidHost } from './helpers'; describe('isValidHost', () => { it.each([ @@ -51,3 +51,69 @@ describe('isValidHost', () => { expect(isValidHost(str)).toBe(expected); }); }); + +describe('defaultScmResolveUrl', () => { + it('works for relative paths and retains query params', () => { + expect( + defaultScmResolveUrl({ + url: './b.yaml', + base: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml', + }), + ).toBe( + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/b.yaml', + ); + + expect( + defaultScmResolveUrl({ + url: './b.yaml', + base: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml?at=master', + }), + ).toBe( + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/b.yaml?at=master', + ); + + expect( + defaultScmResolveUrl({ + url: 'b.yaml', + base: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml', + }), + ).toBe( + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/b.yaml', + ); + }); + + it('works for absolute paths and retains query params', () => { + expect( + defaultScmResolveUrl({ + url: '/other/b.yaml', + base: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml', + }), + ).toBe( + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/other/b.yaml', + ); + + expect( + defaultScmResolveUrl({ + url: '/other/b.yaml', + base: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml?at=master', + }), + ).toBe( + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/other/b.yaml?at=master', + ); + }); + + it('works for full urls and throws away query params', () => { + expect( + defaultScmResolveUrl({ + url: 'https://b.com/b.yaml', + base: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/folder/a.yaml?at=master', + }), + ).toBe('https://b.com/b.yaml'); + }); +}); diff --git a/packages/integration/src/helpers.ts b/packages/integration/src/helpers.ts index 73cef18297..c09f808011 100644 --- a/packages/integration/src/helpers.ts +++ b/packages/integration/src/helpers.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import parseGitUrl from 'git-url-parse'; import { ScmIntegration, ScmIntegrationsGroup } from './types'; /** Checks whether the given argument is a valid URL hostname */ @@ -51,3 +52,43 @@ export function basicIntegrations( }, }; } + +/** + * Default implementation of ScmIntegration.resolveUrl, that only works with + * URL pathname based providers. + */ +export function defaultScmResolveUrl(options: { + url: string; + base: string; +}): string { + const { url, base } = options; + + // If it is a fully qualified URL - then return it verbatim + try { + // eslint-disable-next-line no-new + new URL(url); + return url; + } catch { + // ignore intentionally + } + + let updated: URL; + + if (url.startsWith('/')) { + // If it is an absolute path, move relative to the repo root + const { filepath } = parseGitUrl(base); + updated = new URL(base); + const repoRootPath = updated.pathname + .substring(0, updated.pathname.length - filepath.length) + .replace(/\/+$/, ''); + updated.pathname = `${repoRootPath}${url}`; + } else { + // For relative URLs, just let the default URL constructor handle the + // resolving. Note that this essentially will treat the last segment of the + // base as a file - NOT a folder - unless the url ends in a slash. + updated = new URL(url, base); + } + + updated.search = new URL(base).search; + return updated.toString(); +} diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index fdcd7da676..c39608fe8e 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -18,5 +18,6 @@ export * from './azure'; export * from './bitbucket'; export * from './github'; export * from './gitlab'; +export { defaultScmResolveUrl } from './helpers'; export { ScmIntegrations } from './ScmIntegrations'; export type { ScmIntegration, ScmIntegrationRegistry } from './types'; diff --git a/packages/integration/src/types.ts b/packages/integration/src/types.ts index 7d5b0fbe74..c63a323dc3 100644 --- a/packages/integration/src/types.ts +++ b/packages/integration/src/types.ts @@ -36,16 +36,21 @@ export interface ScmIntegration { title: string; /** - * Works like the two-argument form of the URL constructor, resolving an - * absolute or relative URL in relation to a base URL. + * Resolves an absolute or relative URL in relation to a base URL. * - * If this method is not implemented, the URL constructor is used instead for - * URLs that match this integration. + * This method is adapted for use within SCM systems, so relative URLs are + * within the context of the root of the hierarchy pointed to by the base + * URL. + * + * For example, if the base URL is `/folder/a.yaml`, i.e. + * within the file tree of a certain repo, an absolute path of `/b.yaml` does + * not resolve to `https://hostname/b.yaml` but rather to + * `/b.yaml` inside the file tree of that same repo. * * @param options.url The (absolute or relative) URL or path to resolve * @param options.base The base URL onto which this resolution happens */ - resolveUrl?(options: { url: string; base: string }): string; + resolveUrl(options: { url: string; base: string }): string; } /** @@ -83,8 +88,16 @@ export interface ScmIntegrationRegistry gitlab: ScmIntegrationsGroup; /** - * Works like the two-argument form of the URL constructor, resolving an - * absolute or relative URL in relation to a base URL. + * Resolves an absolute or relative URL in relation to a base URL. + * + * This method is adapted for use within SCM systems, so relative URLs are + * within the context of the root of the hierarchy pointed to by the base + * URL. + * + * For example, if the base URL is `/folder/a.yaml`, i.e. + * within the file tree of a certain repo, an absolute path of `/b.yaml` does + * not resolve to `https://hostname/b.yaml` but rather to + * `/b.yaml` inside the file tree of that same repo. * * @param options.url The (absolute or relative) URL or path to resolve * @param options.base The base URL onto which this resolution happens