From 5a5163519744d5e2589ecf719f1812c2fa48f735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 4 Feb 2021 15:54:54 +0100 Subject: [PATCH] backend-common: implement UrlReader.search that does glob matching --- .changeset/shiny-shirts-worry.md | 5 + packages/backend-common/package.json | 1 + .../src/reading/AzureUrlReader.ts | 5 + .../src/reading/BitbucketUrlReader.ts | 5 + .../src/reading/FetchUrlReader.ts | 13 +- .../src/reading/GithubUrlReader.test.ts | 383 +++++++++++++++++- .../src/reading/GithubUrlReader.ts | 142 ++++++- .../src/reading/GitlabUrlReader.ts | 9 +- .../src/reading/UrlReaderPredicateMux.ts | 14 + packages/backend-common/src/reading/index.ts | 2 +- packages/backend-common/src/reading/types.ts | 49 +++ packages/techdocs-common/src/helpers.test.ts | 15 +- .../src/stages/prepare/dir.test.ts | 1 + .../src/ingestion/LocationReaders.ts | 10 + .../processors/CodeOwnersProcessor.test.ts | 12 +- .../processors/PlaceholderProcessor.test.ts | 2 +- .../processors/UrlReaderProcessor.test.ts | 38 +- .../processors/UrlReaderProcessor.ts | 35 +- .../src/service/CatalogBuilder.test.ts | 1 + .../src/service/standaloneServer.ts | 1 + 20 files changed, 696 insertions(+), 47 deletions(-) create mode 100644 .changeset/shiny-shirts-worry.md diff --git a/.changeset/shiny-shirts-worry.md b/.changeset/shiny-shirts-worry.md new file mode 100644 index 0000000000..b4797b61b4 --- /dev/null +++ b/.changeset/shiny-shirts-worry.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Implement `UrlReader.search` which implements glob matching. diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index faec36dab4..895d241c6a 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -50,6 +50,7 @@ "knex": "^0.21.6", "lodash": "^4.17.15", "logform": "^2.1.1", + "minimatch": "^3.0.4", "minimist": "^1.2.5", "morgan": "^1.10.0", "selfsigned": "^1.10.7", diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index 578db2ac92..bc1b61a041 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -29,6 +29,7 @@ import { ReaderFactory, ReadTreeOptions, ReadTreeResponse, + SearchResponse, UrlReader, } from './types'; import { ReadTreeResponseFactory } from './tree'; @@ -116,6 +117,10 @@ export class AzureUrlReader implements UrlReader { }); } + async search(): Promise { + throw new Error('AzureUrlReader does not implement search'); + } + toString() { const { host, token } = this.options; return `azure{host=${host},authed=${Boolean(token)}}`; diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 373c75588e..e51c03e562 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -31,6 +31,7 @@ import { ReaderFactory, ReadTreeOptions, ReadTreeResponse, + SearchResponse, UrlReader, } from './types'; @@ -129,6 +130,10 @@ export class BitbucketUrlReader implements UrlReader { }); } + async search(): Promise { + throw new Error('BitbucketUrlReader does not implement search'); + } + toString() { const { host, token, username, appPassword } = this.config; let authed = Boolean(token); diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts index 81f1dfa90e..0b26dfe911 100644 --- a/packages/backend-common/src/reading/FetchUrlReader.ts +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -16,7 +16,12 @@ import fetch from 'cross-fetch'; import { NotFoundError } from '../errors'; -import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; +import { + ReaderFactory, + ReadTreeResponse, + SearchResponse, + UrlReader, +} from './types'; /** * A UrlReader that does a plain fetch of the URL. @@ -68,10 +73,14 @@ export class FetchUrlReader implements UrlReader { throw new Error(message); } - readTree(): Promise { + async readTree(): Promise { throw new Error('FetchUrlReader does not implement readTree'); } + async search(): Promise { + throw new Error('FetchUrlReader does not implement search'); + } + toString() { return 'fetch{}'; } diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index fae3f3ba12..d4e579b143 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -23,7 +23,13 @@ import { rest } from 'msw'; import { setupServer } from 'msw/node'; import path from 'path'; import { NotFoundError, NotModifiedError } from '../errors'; -import { GithubUrlReader } from './GithubUrlReader'; +import { + GhBlobResponse, + GhBranchResponse, + GhRepoResponse, + GhTreeResponse, + GithubUrlReader, +} from './GithubUrlReader'; import { ReadTreeResponseFactory } from './tree'; const treeResponseFactory = ReadTreeResponseFactory.create({ @@ -69,6 +75,10 @@ describe('GithubUrlReader', () => { }); }); + /* + * read + */ + describe('read', () => { it('should use the headers from the credentials provider to the fetch request when doing read', async () => { expect.assertions(2); @@ -107,6 +117,10 @@ describe('GithubUrlReader', () => { }); }); + /* + * readTree + */ + describe('readTree', () => { beforeEach(() => { mockFs({ @@ -128,29 +142,31 @@ describe('GithubUrlReader', () => { ); const reposGithubApiResponse = { - id: '123', + id: 123, full_name: 'backstage/mock', default_branch: 'main', branches_url: 'https://api.github.com/repos/backstage/mock/branches{/branch}', archive_url: 'https://api.github.com/repos/backstage/mock/{archive_format}{/ref}', - }; + } as Partial; const reposGheApiResponse = { - ...reposGithubApiResponse, + id: 123, + full_name: 'backstage/mock', + default_branch: 'main', branches_url: 'https://ghe.github.com/api/v3/repos/backstage/mock/branches{/branch}', archive_url: 'https://ghe.github.com/api/v3/repos/backstage/mock/{archive_format}{/ref}', - }; + } as Partial; const branchesApiResponse = { name: 'main', commit: { sha: 'etag123abc', }, - }; + } as Partial; beforeEach(() => { worker.use( @@ -392,4 +408,359 @@ describe('GithubUrlReader', () => { }).toThrowError('must configure an explicit apiBaseUrl'); }); }); + + /* + * search + */ + + describe('search', () => { + beforeEach(() => { + mockFs({ '/tmp': mockFs.directory() }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + const repoBuffer = fs.readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'backstage-mock-etag123.tar.gz', + ), + ); + + const githubTreeContents: GhTreeResponse['tree'] = [ + { + path: 'mkdocs.yml', + type: 'blob', + url: 'https://api.github.com/repos/backstage/mock/git/blobs/1', + }, + { + path: 'docs', + type: 'tree', + url: 'https://api.github.com/repos/backstage/mock/git/trees/2', + }, + { + path: 'docs/index.md', + type: 'blob', + url: 'https://api.github.com/repos/backstage/mock/git/blobs/3', + }, + ]; + + const gheTreeContents: GhTreeResponse['tree'] = [ + { + path: 'mkdocs.yml', + type: 'blob', + url: 'https://ghe.github.com/api/v3/repos/backstage/mock/git/blobs/1', + }, + { + path: 'docs', + type: 'tree', + url: 'https://ghe.github.com/api/v3/repos/backstage/mock/git/trees/2', + }, + { + path: 'docs/index.md', + type: 'blob', + url: 'https://ghe.github.com/api/v3/repos/backstage/mock/git/blobs/3', + }, + ]; + + // Tarballs + beforeEach(() => { + worker.use( + rest.get( + 'https://api.github.com/repos/backstage/mock/tarball/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-etag123.tar.gz', + ), + ctx.body(repoBuffer), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/tarball/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-etag123.tar.gz', + ), + ctx.body(repoBuffer), + ), + ), + ); + }); + + // Repo details + beforeEach(() => { + const githubResponse = { + id: 123, + full_name: 'backstage/mock', + default_branch: 'main', + branches_url: + 'https://api.github.com/repos/backstage/mock/branches{/branch}', + archive_url: + 'https://api.github.com/repos/backstage/mock/{archive_format}{/ref}', + trees_url: + 'https://api.github.com/repos/backstage/mock/git/trees{/sha}', + } as Partial; + + const gheResponse = { + id: 123, + full_name: 'backstage/mock', + default_branch: 'main', + branches_url: + 'https://ghe.github.com/api/v3/repos/backstage/mock/branches{/branch}', + archive_url: + 'https://ghe.github.com/api/v3/repos/backstage/mock/{archive_format}{/ref}', + trees_url: + 'https://ghe.github.com/api/v3/repos/backstage/mock/git/trees{/sha}', + } as Partial; + + worker.use( + rest.get('https://api.github.com/repos/backstage/mock', (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(githubResponse), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(gheResponse), + ), + ), + ); + }); + + // Branch details + beforeEach(() => { + const response = { + name: 'main', + commit: { + sha: 'etag123abc', + }, + } as Partial; + + worker.use( + rest.get( + 'https://api.github.com/repos/backstage/mock/branches/main', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(response), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/branches/main', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(response), + ), + ), + rest.get( + 'https://api.github.com/repos/backstage/mock/branches/branchDoesNotExist', + (_, res, ctx) => res(ctx.status(404)), + ), + ); + }); + + // Blobs + beforeEach(() => { + const blob1Response = { + content: Buffer.from('site_name: Test\n').toString('base64'), + } as Partial; + + const blob3Response = { + content: Buffer.from('# Test\n').toString('base64'), + } as Partial; + + worker.use( + rest.get( + 'https://api.github.com/repos/backstage/mock/git/blobs/1', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(blob1Response), + ), + ), + rest.get( + 'https://api.github.com/repos/backstage/mock/git/blobs/3', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(blob3Response), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/git/blobs/1', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(blob1Response), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/git/blobs/3', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(blob3Response), + ), + ), + ); + }); + + async function runTests(reader: GithubUrlReader, baseUrl: string) { + const r1 = await reader.search( + `${baseUrl}/backstage/mock/tree/main/**/*`, + ); + expect(r1.etag).toBe('etag123abc'); + expect(r1.files.length).toBe(2); + + const r2 = await reader.search( + `${baseUrl}/backstage/mock/tree/main/**/*`, + { etag: 'somethingElse' }, + ); + expect(r2.etag).toBe('etag123abc'); + expect(r2.files.length).toBe(2); + + const r3 = await reader.search(`${baseUrl}/backstage/mock/tree/main/o`); + expect(r3.files.length).toBe(0); + + const r4 = await reader.search( + `${baseUrl}/backstage/mock/tree/main/*docs*`, + ); + expect(r4.files.length).toBe(1); + expect(r4.files[0].url).toBe( + `${baseUrl}/backstage/mock/tree/main/mkdocs.yml`, + ); + await expect(r4.files[0].content()).resolves.toEqual( + Buffer.from('site_name: Test\n'), + ); + + const r5 = await reader.search( + `${baseUrl}/backstage/mock/tree/main/*/index.*`, + ); + expect(r5.files.length).toBe(1); + expect(r5.files[0].url).toBe( + `${baseUrl}/backstage/mock/tree/main/docs/index.md`, + ); + await expect(r5.files[0].content()).resolves.toEqual( + Buffer.from('# Test\n'), + ); + } + + // eslint-disable-next-line jest/expect-expect + it('succeeds on github when going via repo listing', async () => { + worker.use( + rest.get( + 'https://api.github.com/repos/backstage/mock/git/trees/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + truncated: false, + tree: githubTreeContents, + } as Partial), + ), + ), + ); + await runTests(githubProcessor, 'https://github.com'); + }); + + // eslint-disable-next-line jest/expect-expect + it('succeeds on github when going via readTree', async () => { + worker.use( + rest.get( + 'https://api.github.com/repos/backstage/mock/git/trees/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + truncated: true, + tree: [], + } as Partial), + ), + ), + ); + await runTests(githubProcessor, 'https://github.com'); + }); + + // eslint-disable-next-line jest/expect-expect + it('succeeds on ghe when going via repo listing', async () => { + worker.use( + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/git/trees/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + truncated: false, + tree: gheTreeContents, + } as Partial), + ), + ), + ); + await runTests(gheProcessor, 'https://ghe.github.com'); + }); + + // eslint-disable-next-line jest/expect-expect + it('succeeds on ghe when going via readTree', async () => { + worker.use( + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/git/trees/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + truncated: true, + tree: [], + } as Partial), + ), + ), + ); + await runTests(gheProcessor, 'https://ghe.github.com'); + }); + + it('throws NotModifiedError when same etag', async () => { + await expect( + githubProcessor.search( + 'https://githib.com/backstage/mock/tree/main/**/*', + { etag: 'etag123abc' }, + ), + ).rejects.toThrow(NotModifiedError); + }); + + it('throws NotFoundError when missing branch', async () => { + await expect( + githubProcessor.search( + 'https://githib.com/backstage/mock/tree/branchDoesNotExist/**/*', + ), + ).rejects.toThrow(NotFoundError); + }); + }); }); diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 99d641f2dd..291740a9fc 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -23,6 +23,7 @@ import { import { RestEndpointMethodTypes } from '@octokit/rest'; import fetch from 'cross-fetch'; import parseGitUrl from 'git-url-parse'; +import { Minimatch } from 'minimatch'; import { Readable } from 'stream'; import { NotFoundError, NotModifiedError } from '../errors'; import { ReadTreeResponseFactory } from './tree'; @@ -30,9 +31,17 @@ import { ReaderFactory, ReadTreeOptions, ReadTreeResponse, + SearchOptions, + SearchResponse, + SearchResponseFile, UrlReader, } from './types'; +export type GhRepoResponse = RestEndpointMethodTypes['repos']['get']['response']['data']; +export type GhBranchResponse = RestEndpointMethodTypes['repos']['getBranch']['response']['data']; +export type GhTreeResponse = RestEndpointMethodTypes['git']['getTree']['response']['data']; +export type GhBlobResponse = RestEndpointMethodTypes['git']['getBlob']['response']['data']; + /** * A processor that adds the ability to read files from GitHub v3 APIs, such as * the one exposed by GitHub itself. @@ -106,27 +115,43 @@ export class GithubUrlReader implements UrlReader { throw new NotModifiedError(); } + const { filepath } = parseGitUrl(url); const { headers } = await this.deps.credentialsProvider.getCredentials({ url, }); - // archive_url looks like "https://api.github.com/repos/owner/repo/{archive_format}{/ref}" - const archive = await this.fetchResponse( - repoDetails.repo.archive_url - .replace('{archive_format}', 'tarball') - .replace('{/ref}', `/${commitSha}`), + return this.doReadTree( + repoDetails.repo.archive_url, + commitSha, + filepath, + { headers }, + options, + ); + } + + async search(url: string, options?: SearchOptions): Promise { + const repoDetails = await this.getRepoDetails(url); + const commitSha = repoDetails.branch.commit.sha!; + + if (options?.etag && options.etag === commitSha) { + throw new NotModifiedError(); + } + + const { filepath } = parseGitUrl(url); + const { headers } = await this.deps.credentialsProvider.getCredentials({ + url, + }); + + const files = await this.doSearch( + url, + repoDetails.repo.trees_url, + repoDetails.repo.archive_url, + commitSha, + filepath, { headers }, ); - const { filepath } = parseGitUrl(url); - return await this.deps.treeResponseFactory.fromTarArchive({ - // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want - // to stick to using that in exclusively backend code. - stream: (archive.body as unknown) as Readable, - subpath: filepath, - etag: commitSha, - filter: options?.filter, - }); + return { files, etag: commitSha }; } toString() { @@ -134,11 +159,92 @@ export class GithubUrlReader implements UrlReader { return `github{host=${host},authed=${Boolean(token)}}`; } + private async doReadTree( + archiveUrl: string, + sha: string, + subpath: string, + init: RequestInit, + options?: ReadTreeOptions, + ): Promise { + // archive_url looks like "https://api.github.com/repos/owner/repo/{archive_format}{/ref}" + const archive = await this.fetchResponse( + archiveUrl + .replace('{archive_format}', 'tarball') + .replace('{/ref}', `/${sha}`), + init, + ); + + return await this.deps.treeResponseFactory.fromTarArchive({ + // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want + // to stick to using that in exclusively backend code. + stream: (archive.body as unknown) as Readable, + subpath, + etag: sha, + filter: options?.filter, + }); + } + + private async doSearch( + url: string, + treesUrl: string, + archiveUrl: string, + sha: string, + query: string, + init: RequestInit, + ): Promise { + function pathToUrl(path: string): string { + // TODO(freben): Use the integration package facility for this instead + // pathname starts as /backstage/backstage/blob/master/ + const updated = new URL(url); + const base = updated.pathname.split('/').slice(1, 5).join('/'); + updated.pathname = `${base}/${path}`; + return updated.toString(); + } + + const matcher = new Minimatch(query.replace(/^\/+/, '')); + + // trees_url looks like "https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}" + const recursiveTree: GhTreeResponse = await this.fetchJson( + treesUrl.replace('{/sha}', `/${sha}?recursive=true`), + init, + ); + + // The simple case is that we got the entire tree in a single operation. + if (!recursiveTree.truncated) { + const matching = recursiveTree.tree.filter( + item => + item.type === 'blob' && + item.path && + item.url && + matcher.match(item.path), + ); + + return matching.map(item => ({ + url: pathToUrl(item.path!), + content: async () => { + const blob: GhBlobResponse = await this.fetchJson(item.url!, init); + return Buffer.from(blob.content, 'base64'); + }, + })); + } + + // For larger repos, we leverage readTree and filter through that instead + const tree = await this.doReadTree(archiveUrl, sha, '', init, { + filter: path => matcher.match(path), + }); + const files = await tree.files(); + + return files.map(file => ({ + url: pathToUrl(file.path), + content: file.content, + })); + } + private async getRepoDetails( url: string, ): Promise<{ - repo: RestEndpointMethodTypes['repos']['get']['response']['data']; - branch: RestEndpointMethodTypes['repos']['getBranch']['response']['data']; + repo: GhRepoResponse; + branch: GhBranchResponse; }> { const parsed = parseGitUrl(url); const { ref, full_name } = parsed; @@ -151,13 +257,13 @@ export class GithubUrlReader implements UrlReader { url, }); - const repo: RestEndpointMethodTypes['repos']['get']['response']['data'] = await this.fetchJson( + const repo: GhRepoResponse = await this.fetchJson( `${this.config.apiBaseUrl}/repos/${full_name}`, { headers }, ); // branches_url looks like "https://api.github.com/repos/owner/repo/branches{/branch}" - const branch: RestEndpointMethodTypes['repos']['getBranch']['response']['data'] = await this.fetchJson( + const branch: GhBranchResponse = await this.fetchJson( repo.branches_url.replace('{/branch}', `/${ref || repo.default_branch}`), { headers }, ); diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 8a763d1259..5e02a9b8e6 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -21,16 +21,17 @@ import { readGitLabIntegrationConfigs, } from '@backstage/integration'; import fetch from 'cross-fetch'; +import parseGitUrl from 'git-url-parse'; +import { Readable } from 'stream'; import { NotFoundError, NotModifiedError } from '../errors'; import { ReadTreeResponseFactory } from './tree'; import { ReaderFactory, ReadTreeOptions, ReadTreeResponse, + SearchResponse, UrlReader, } from './types'; -import parseGitUrl from 'git-url-parse'; -import { Readable } from 'stream'; export class GitlabUrlReader implements UrlReader { private readonly treeResponseFactory: ReadTreeResponseFactory; @@ -154,6 +155,10 @@ export class GitlabUrlReader implements UrlReader { }); } + async search(): Promise { + throw new Error('GitlabUrlReader does not implement search'); + } + toString() { const { host, token } = this.config; return `gitlab{host=${host},authed=${Boolean(token)}}`; diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index 3183aa0c28..91f4ae63ab 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -18,6 +18,8 @@ import { NotAllowedError } from '../errors'; import { ReadTreeOptions, ReadTreeResponse, + SearchOptions, + SearchResponse, UrlReader, UrlReaderPredicateTuple, } from './types'; @@ -60,6 +62,18 @@ export class UrlReaderPredicateMux implements UrlReader { throw new NotAllowedError(`Reading from '${url}' is not allowed`); } + async search(url: string, options?: SearchOptions): Promise { + const parsed = new URL(url); + + for (const { predicate, reader } of this.readers) { + if (predicate(parsed)) { + return await reader.search(url, options); + } + } + + throw new NotAllowedError(`Reading from '${url}' is not allowed`); + } + toString() { return `predicateMux{readers=${this.readers.map(t => t.reader).join(',')}`; } diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 8a6c2bf1a3..1266161ddf 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -export type { UrlReader, ReadTreeResponse } from './types'; +export type { UrlReader, ReadTreeResponse, SearchResponse } from './types'; export { UrlReaders } from './UrlReaders'; export { AzureUrlReader } from './AzureUrlReader'; export { BitbucketUrlReader } from './BitbucketUrlReader'; diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index be788fc6e2..4caa76841a 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -24,6 +24,7 @@ import { ReadTreeResponseFactory } from './tree'; export type UrlReader = { read(url: string): Promise; readTree(url: string, options?: ReadTreeOptions): Promise; + search(url: string, options?: SearchOptions): Promise; }; export type UrlReaderPredicateTuple = { @@ -103,3 +104,51 @@ export type ReadTreeResponseFile = { path: string; content(): Promise; }; + +/** + * An options object for search operations. + */ +export type SearchOptions = { + /** + * An etag can be provided to check whether the search response has changed from a previous execution. + * + * In the search() response, an etag is returned along with the files. The etag is a unique identifer + * of the current tree, usually the commit SHA or etag from the target. + * + * When an etag is given in SearchOptions, search will first compare the etag against the etag + * on the target branch. If they match, search will throw a NotModifiedError indicating that the search + * response will not differ from the previous response which included this particular etag. If they mismatch, + * search will return the rest of SearchResponse along with a new etag. + */ + etag?: string; +}; + +/** + * The output of a search operation. + */ +export type SearchResponse = { + /** + * The files that matched the search query. + */ + files: SearchResponseFile[]; + + /** + * A unique identifer of the current remote tree, usually the commit SHA or etag from the target. + */ + etag: string; +}; + +/** + * Represents a single file in a search response. + */ +export type SearchResponseFile = { + /** + * The full URL to the file. + */ + url: string; + + /** + * The binary contents of the file. + */ + content(): Promise; +}; diff --git a/packages/techdocs-common/src/helpers.test.ts b/packages/techdocs-common/src/helpers.test.ts index 740a08aaa2..eba1386b11 100644 --- a/packages/techdocs-common/src/helpers.test.ts +++ b/packages/techdocs-common/src/helpers.test.ts @@ -14,14 +14,18 @@ * limitations under the License. */ +import { + ReadTreeResponse, + SearchResponse, + UrlReader, +} from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import { Readable } from 'stream'; import { getDocFilesFromRepository, getLocationForEntity, parseReferenceAnnotation, } from './helpers'; -import { UrlReader, ReadTreeResponse } from '@backstage/backend-common'; -import { Entity } from '@backstage/catalog-model'; const entityBase: Entity = { metadata: { @@ -138,6 +142,13 @@ describe('getDocFilesFromRepository', () => { etag: '', }; } + + async search(): Promise { + return { + etag: '', + files: [], + }; + } } const output = await getDocFilesFromRepository( diff --git a/packages/techdocs-common/src/stages/prepare/dir.test.ts b/packages/techdocs-common/src/stages/prepare/dir.test.ts index 630608e817..cd8d358afe 100644 --- a/packages/techdocs-common/src/stages/prepare/dir.test.ts +++ b/packages/techdocs-common/src/stages/prepare/dir.test.ts @@ -49,6 +49,7 @@ const mockConfig = new ConfigReader({}); const mockUrlReader: jest.Mocked = { read: jest.fn(), readTree: jest.fn(), + search: jest.fn(), }; describe('directory preparer', () => { diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index a789e669d0..be88aa2810 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -139,6 +139,16 @@ export class LocationReaders implements LocationReader { if (emitResult.type === 'relation') { throw new Error('readLocation may not emit entity relations'); } + if ( + emitResult.type === 'location' && + emitResult.location.type === item.location.type && + emitResult.location.target === item.location.target + ) { + // Ignore self-referential locations silently (this can happen for + // example if you use a glob target like "**/*.yaml" in a Location + // entity) + return; + } emit(emitResult); }; diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts index be4f15e501..eb1187eda5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts @@ -160,7 +160,7 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockResolvedValue(mockReadResult({ data: ownersText })); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; const result = await findRawCodeOwners(mockLocation(), { reader, logger, @@ -170,7 +170,7 @@ describe('CodeOwnersProcessor', () => { it('should return undefined when no codeowner', async () => { const read = jest.fn().mockRejectedValue(mockReadResult()); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; await expect( findRawCodeOwners(mockLocation(), { reader, logger }), @@ -184,7 +184,7 @@ describe('CodeOwnersProcessor', () => { .mockImplementationOnce(() => mockReadResult({ error: 'foo' })) .mockImplementationOnce(() => mockReadResult({ error: 'bar' })) .mockResolvedValue(mockReadResult({ data: ownersText })); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; const result = await findRawCodeOwners(mockLocation(), { reader, @@ -206,7 +206,7 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() })); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; const owner = await resolveCodeOwner(mockLocation(), { reader, logger }); expect(owner).toBe('backstage-core'); @@ -216,7 +216,7 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockImplementation(() => mockReadResult({ error: 'error: foo' })); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; await expect( resolveCodeOwner(mockLocation(), { reader, logger }), @@ -230,7 +230,7 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() })); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; const processor = new CodeOwnersProcessor({ reader, logger }); return { entity, processor, read }; diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts index 8b2632ad5d..00b6af94e5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts @@ -27,7 +27,7 @@ import { describe('PlaceholderProcessor', () => { const read: jest.MockedFunction = jest.fn(); - const reader: UrlReader = { read, readTree: jest.fn() }; + const reader: UrlReader = { read, readTree: jest.fn(), search: jest.fn() }; beforeEach(() => { jest.resetAllMocks(); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts index 49b6fe71e0..db731b5b17 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -14,24 +14,29 @@ * limitations under the License. */ -import { UrlReaderProcessor } from './UrlReaderProcessor'; -import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; +import { + getVoidLogger, + UrlReader, + UrlReaders, +} from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; +import { msw } from '@backstage/test-utils'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import { msw } from '@backstage/test-utils'; import { CatalogProcessorEntityResult, CatalogProcessorErrorResult, CatalogProcessorResult, } from './types'; +import { UrlReaderProcessor } from './UrlReaderProcessor'; import { defaultEntityDataParser } from './util/parse'; describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost'; - const server = setupServer(); + const server = setupServer(); msw.setupDefaultHandlers(server); + it('should load from url', async () => { const logger = getVoidLogger(); const reader = UrlReaders.default({ @@ -57,7 +62,7 @@ describe('UrlReaderProcessor', () => { )) as CatalogProcessorEntityResult; expect(generated.type).toBe('entity'); - expect(generated.location).toBe(spec); + expect(generated.location).toEqual(spec); expect(generated.entity).toEqual({ mock: 'entity' }); }); @@ -92,4 +97,27 @@ describe('UrlReaderProcessor', () => { `Unable to read url, NotFoundError: could not read ${mockApiOrigin}/component-notfound.yaml, 404 Not Found`, ); }); + + it('uses search when there are globs', async () => { + const logger = getVoidLogger(); + + const reader: jest.Mocked = { + read: jest.fn(), + readTree: jest.fn(), + search: jest.fn().mockImplementation(async () => []), + }; + + const processor = new UrlReaderProcessor({ reader, logger }); + + const emit = jest.fn(); + + await processor.readLocation( + { type: 'url', target: 'https://github.com/a/b/blob/x/**/b.yaml' }, + false, + emit, + defaultEntityDataParser, + ); + + expect(reader.search).toBeCalledTimes(1); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index 9daae29d3b..b70ef2877d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -16,6 +16,8 @@ import { UrlReader } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; +import parseGitUrl from 'git-url-parse'; +import limiterFactory from 'p-limit'; import { Logger } from 'winston'; import * as result from './results'; import { @@ -59,10 +61,14 @@ export class UrlReaderProcessor implements CatalogProcessor { } try { - const data = await this.options.reader.read(location.target); - - for await (const parseResult of parser({ data, location })) { - emit(parseResult); + const output = await this.doRead(location.target); + for (const item of output) { + for await (const parseResult of parser({ + data: item.data, + location: { type: location.type, target: item.url }, + })) { + emit(parseResult); + } } } catch (error) { const message = `Unable to read ${location.type}, ${error}`; @@ -78,4 +84,25 @@ export class UrlReaderProcessor implements CatalogProcessor { return true; } + + private async doRead( + location: string, + ): Promise<{ data: Buffer; url: string }[]> { + // Does it contain globs? I.e. does it contain asterisks or question marks + // (no curly braces for now) + const { filepath } = parseGitUrl(location); + if (filepath?.match(/[*?]/)) { + const limiter = limiterFactory(5); + const response = await this.options.reader.search(location); + const output = response.files.map(async file => ({ + url: file.url, + data: await limiter(file.content), + })); + return Promise.all(output); + } + + // Otherwise do a plain read + const data = await this.options.reader.read(location); + return [{ url: location, data }]; + } } diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts index 695902fec9..bc58338fce 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts @@ -44,6 +44,7 @@ describe('CatalogBuilder', () => { const reader: jest.Mocked = { read: jest.fn(), readTree: jest.fn(), + search: jest.fn(), }; const env: CatalogEnvironment = { logger: getVoidLogger(), diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index d91e2f08a2..0fd23a4f7d 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -53,6 +53,7 @@ export async function startStandaloneServer( const mockUrlReader: jest.Mocked = { read: jest.fn(), readTree: jest.fn(), + search: jest.fn(), }; logger.debug('Creating application...');