diff --git a/.changeset/afraid-teachers-cross.md b/.changeset/afraid-teachers-cross.md new file mode 100644 index 0000000000..8fc376441f --- /dev/null +++ b/.changeset/afraid-teachers-cross.md @@ -0,0 +1,29 @@ +--- +'@backstage/create-app': patch +--- + +Pass on plugin database management instance that is now required by the scaffolder plugin. + +To apply this change to an existing application, add the following to `src/plugins/scaffolder.ts`: + +```diff +export default async function createPlugin({ + logger, + config, ++ database, +}: PluginEnvironment) { + +// ...omitted... + + return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, + entityClient, ++ database, + }); +} +``` diff --git a/.changeset/cost-insights-tidy-geese-play.md b/.changeset/cost-insights-tidy-geese-play.md new file mode 100644 index 0000000000..f8aa503ca6 --- /dev/null +++ b/.changeset/cost-insights-tidy-geese-play.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-cost-insights': patch +--- + +Fix snooze quarter option diff --git a/.changeset/curly-crabs-compete.md b/.changeset/curly-crabs-compete.md new file mode 100644 index 0000000000..552a41a9f1 --- /dev/null +++ b/.changeset/curly-crabs-compete.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-circleci': patch +--- + +Migrated to new composability API, exporting the plugin instance as `circleCIPlugin`, the entity page content as `EntityCircleCIContent`, and entity conditional as `isCircleCIAvailable`. diff --git a/.changeset/early-hotels-mate.md b/.changeset/early-hotels-mate.md new file mode 100644 index 0000000000..2cda0344bf --- /dev/null +++ b/.changeset/early-hotels-mate.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +use child logger, if provided, to log single location refresh diff --git a/.changeset/four-rings-push.md b/.changeset/four-rings-push.md new file mode 100644 index 0000000000..500dd0e812 --- /dev/null +++ b/.changeset/four-rings-push.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-cloudbuild': patch +--- + +Migrate to new composability API, exporting the plugin instance as `cloudbuildPlugin`, the entity content as `EntityCloudbuildContent`, the entity conditional as `isCloudbuildAvailable`, and entity cards as `EntityLatestCloudbuildRunCard` and `EntityLatestCloudbuildsForBranchCard`. diff --git a/.changeset/large-terms-invite.md b/.changeset/large-terms-invite.md new file mode 100644 index 0000000000..6653ac8e1e --- /dev/null +++ b/.changeset/large-terms-invite.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-scaffolder-backend': minor +--- + +Introduced `v2` Scaffolder REST API, which uses an implementation that is database backed, making the scaffolder instances stateless. The `createRouter` function now requires a `PluginDatabaseManager` instance to be passed in, commonly available as `database` in the plugin environment in the backend. + +This API should be considered unstable until used by the scaffolder frontend. diff --git a/.changeset/odd-buckets-compare.md b/.changeset/odd-buckets-compare.md new file mode 100644 index 0000000000..faf68fb4a0 --- /dev/null +++ b/.changeset/odd-buckets-compare.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Set explicit content-type in error handler responses. diff --git a/.changeset/old-tools-exist.md b/.changeset/old-tools-exist.md new file mode 100644 index 0000000000..a9365cb784 --- /dev/null +++ b/.changeset/old-tools-exist.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Slight refactoring in support of a future search implementation in `UrlReader`. Mostly moving code around. diff --git a/.changeset/popular-cars-eat.md b/.changeset/popular-cars-eat.md new file mode 100644 index 0000000000..6d3bcdfab2 --- /dev/null +++ b/.changeset/popular-cars-eat.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-jenkins': patch +--- + +Migrate to new composability API, exporting the plugin instance as `jenkinsPlugin`, the entity content as `EntityJenkinsContent`, the entity conditional as `isJenkinsAvailable`, and the entity card as `EntityLatestJenkinsRunCard`. diff --git a/.changeset/selfish-years-pump.md b/.changeset/selfish-years-pump.md new file mode 100644 index 0000000000..e5cc5f3bc3 --- /dev/null +++ b/.changeset/selfish-years-pump.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-github-actions': patch +--- + +Migrate to new composability API, exporting the plugin instance as `githubActionsPlugin`, the entity content as `EntityGithubActionsContent`, entity conditional as `isGithubActionsAvailable`, and entity cards as `EntityLatestGithubActionRunCard`, `EntityLatestGithubActionsForBranchCard`, and `EntityRecentGithubActionsRunsCard`. diff --git a/.changeset/shiny-shirts-worry.md b/.changeset/shiny-shirts-worry.md new file mode 100644 index 0000000000..b4797b61b4 --- /dev/null +++ b/.changeset/shiny-shirts-worry.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Implement `UrlReader.search` which implements glob matching. diff --git a/.changeset/strange-cobras-unite.md b/.changeset/strange-cobras-unite.md new file mode 100644 index 0000000000..d53a324dab --- /dev/null +++ b/.changeset/strange-cobras-unite.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Refactored route response handling to use more explicit types and throw errors. diff --git a/docs/features/search/README.md b/docs/features/search/README.md index cd971d9d41..a06b9aee2b 100644 --- a/docs/features/search/README.md +++ b/docs/features/search/README.md @@ -24,12 +24,18 @@ Backstage ecosystem. ## Project roadmap -| Version | Description | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Backstage Search V.0 ✅ | Search Frontend letting you search through the entities of the software catalog. [See V.0 Use Cases.](#backstage-search-v0) | -| Backstage Search V.1 ⌛ | Basic “out-of-the-box” in-memory indexing process of entities, and their metadata, registered to the Software Catalog. [See V.1 Use Cases.](#backstage-search-v1) | -| Backstage Search V.2 ⌛ | Search Backend responsible for the indexing process of entities, and their metadata, registered to the Software Catalog. [See V.2 Use Cases.](#backstage-search-v2) | -| Backstage Search V.3 ⌛ | Standardized Search API lets you index other plugins data to the search engine of choice. [See V.3 Use Cases.](#backstage-search-v3) | +| Version | Description | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Backstage Search v0 ✅ | Search Frontend letting you search through the entities of the software catalog. [See v0 Use Cases.](#backstage-search-v0) | +| [Backstage Search V0.5 ⌛][v0.5] | Foundations for the architecture. | +| [Backstage Search v1 ⌛][v1] | Basic “out-of-the-box” in-memory indexing process of entities, and their metadata, registered to the Software Catalog. [See v1 Use Cases.](#backstage-search-v1) | +| [Backstage Search v2 ⌛][v2] | Search Backend responsible for the indexing process of entities, and their metadata, registered to the Software Catalog. [See v2 Use Cases.](#backstage-search-v2) | +| [Backstage Search v3 ⌛][v3] | Standardized Search API lets you index other plugins data to the search engine of choice. [See v3 Use Cases.](#backstage-search-v3) | + +[v0.5]: https://github.com/backstage/backstage/milestone/25 +[v1]: https://github.com/backstage/backstage/milestone/26 +[v2]: https://github.com/backstage/backstage/milestone/27 +[v3]: https://github.com/backstage/backstage/milestone/28 ## Use Cases diff --git a/docs/features/techdocs/creating-and-publishing.md b/docs/features/techdocs/creating-and-publishing.md index c15dc1fe69..b9340330a8 100644 --- a/docs/features/techdocs/creating-and-publishing.md +++ b/docs/features/techdocs/creating-and-publishing.md @@ -80,6 +80,10 @@ Create a `/docs` folder in the root of the project with at least an `index.md` file. _(If you add more markdown files, make sure to update the nav in the mkdocs.yml file to get a proper navigation for your documentation.)_ +> Note - Although `docs` is a popular directory name for storing documentation, +> it can be renamed to something else and can be configured by `mkdocs.yml`. See +> https://www.mkdocs.org/user-guide/configuration/#docs_dir + The `docs/index.md` can for example have the following content: ```md diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 3053784dc3..895d241c6a 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -33,6 +33,7 @@ "@backstage/config": "^0.1.2", "@backstage/config-loader": "^0.5.1", "@backstage/integration": "^0.3.2", + "@octokit/rest": "^18.0.12", "@types/cors": "^2.8.6", "@types/express": "^4.17.6", "archiver": "^5.0.2", @@ -49,6 +50,7 @@ "knex": "^0.21.6", "lodash": "^4.17.15", "logform": "^2.1.1", + "minimatch": "^3.0.4", "minimist": "^1.2.5", "morgan": "^1.10.0", "selfsigned": "^1.10.7", diff --git a/packages/backend-common/src/middleware/errorHandler.ts b/packages/backend-common/src/middleware/errorHandler.ts index a08849813d..2f3b904d38 100644 --- a/packages/backend-common/src/middleware/errorHandler.ts +++ b/packages/backend-common/src/middleware/errorHandler.ts @@ -63,10 +63,10 @@ export function errorHandler( return ( error: Error, _request: Request, - response: Response, + res: Response, next: NextFunction, ) => { - if (response.headersSent) { + if (res.headersSent) { // If the headers have already been sent, do not send the response again // as this will throw an error in the backend. next(error); @@ -80,7 +80,9 @@ export function errorHandler( logger.error(error); } - response.status(status).send(message); + res.status(status); + res.setHeader('content-type', 'text/plain'); + res.send(message); }; } diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index 578db2ac92..bc1b61a041 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -29,6 +29,7 @@ import { ReaderFactory, ReadTreeOptions, ReadTreeResponse, + SearchResponse, UrlReader, } from './types'; import { ReadTreeResponseFactory } from './tree'; @@ -116,6 +117,10 @@ export class AzureUrlReader implements UrlReader { }); } + async search(): Promise { + throw new Error('AzureUrlReader does not implement search'); + } + toString() { const { host, token } = this.options; return `azure{host=${host},authed=${Boolean(token)}}`; diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 373c75588e..e51c03e562 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -31,6 +31,7 @@ import { ReaderFactory, ReadTreeOptions, ReadTreeResponse, + SearchResponse, UrlReader, } from './types'; @@ -129,6 +130,10 @@ export class BitbucketUrlReader implements UrlReader { }); } + async search(): Promise { + throw new Error('BitbucketUrlReader does not implement search'); + } + toString() { const { host, token, username, appPassword } = this.config; let authed = Boolean(token); diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts index 81f1dfa90e..0b26dfe911 100644 --- a/packages/backend-common/src/reading/FetchUrlReader.ts +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -16,7 +16,12 @@ import fetch from 'cross-fetch'; import { NotFoundError } from '../errors'; -import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; +import { + ReaderFactory, + ReadTreeResponse, + SearchResponse, + UrlReader, +} from './types'; /** * A UrlReader that does a plain fetch of the URL. @@ -68,10 +73,14 @@ export class FetchUrlReader implements UrlReader { throw new Error(message); } - readTree(): Promise { + async readTree(): Promise { throw new Error('FetchUrlReader does not implement readTree'); } + async search(): Promise { + throw new Error('FetchUrlReader does not implement search'); + } + toString() { return 'fetch{}'; } diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index fae3f3ba12..d4e579b143 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -23,7 +23,13 @@ import { rest } from 'msw'; import { setupServer } from 'msw/node'; import path from 'path'; import { NotFoundError, NotModifiedError } from '../errors'; -import { GithubUrlReader } from './GithubUrlReader'; +import { + GhBlobResponse, + GhBranchResponse, + GhRepoResponse, + GhTreeResponse, + GithubUrlReader, +} from './GithubUrlReader'; import { ReadTreeResponseFactory } from './tree'; const treeResponseFactory = ReadTreeResponseFactory.create({ @@ -69,6 +75,10 @@ describe('GithubUrlReader', () => { }); }); + /* + * read + */ + describe('read', () => { it('should use the headers from the credentials provider to the fetch request when doing read', async () => { expect.assertions(2); @@ -107,6 +117,10 @@ describe('GithubUrlReader', () => { }); }); + /* + * readTree + */ + describe('readTree', () => { beforeEach(() => { mockFs({ @@ -128,29 +142,31 @@ describe('GithubUrlReader', () => { ); const reposGithubApiResponse = { - id: '123', + id: 123, full_name: 'backstage/mock', default_branch: 'main', branches_url: 'https://api.github.com/repos/backstage/mock/branches{/branch}', archive_url: 'https://api.github.com/repos/backstage/mock/{archive_format}{/ref}', - }; + } as Partial; const reposGheApiResponse = { - ...reposGithubApiResponse, + id: 123, + full_name: 'backstage/mock', + default_branch: 'main', branches_url: 'https://ghe.github.com/api/v3/repos/backstage/mock/branches{/branch}', archive_url: 'https://ghe.github.com/api/v3/repos/backstage/mock/{archive_format}{/ref}', - }; + } as Partial; const branchesApiResponse = { name: 'main', commit: { sha: 'etag123abc', }, - }; + } as Partial; beforeEach(() => { worker.use( @@ -392,4 +408,359 @@ describe('GithubUrlReader', () => { }).toThrowError('must configure an explicit apiBaseUrl'); }); }); + + /* + * search + */ + + describe('search', () => { + beforeEach(() => { + mockFs({ '/tmp': mockFs.directory() }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + const repoBuffer = fs.readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'backstage-mock-etag123.tar.gz', + ), + ); + + const githubTreeContents: GhTreeResponse['tree'] = [ + { + path: 'mkdocs.yml', + type: 'blob', + url: 'https://api.github.com/repos/backstage/mock/git/blobs/1', + }, + { + path: 'docs', + type: 'tree', + url: 'https://api.github.com/repos/backstage/mock/git/trees/2', + }, + { + path: 'docs/index.md', + type: 'blob', + url: 'https://api.github.com/repos/backstage/mock/git/blobs/3', + }, + ]; + + const gheTreeContents: GhTreeResponse['tree'] = [ + { + path: 'mkdocs.yml', + type: 'blob', + url: 'https://ghe.github.com/api/v3/repos/backstage/mock/git/blobs/1', + }, + { + path: 'docs', + type: 'tree', + url: 'https://ghe.github.com/api/v3/repos/backstage/mock/git/trees/2', + }, + { + path: 'docs/index.md', + type: 'blob', + url: 'https://ghe.github.com/api/v3/repos/backstage/mock/git/blobs/3', + }, + ]; + + // Tarballs + beforeEach(() => { + worker.use( + rest.get( + 'https://api.github.com/repos/backstage/mock/tarball/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-etag123.tar.gz', + ), + ctx.body(repoBuffer), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/tarball/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-etag123.tar.gz', + ), + ctx.body(repoBuffer), + ), + ), + ); + }); + + // Repo details + beforeEach(() => { + const githubResponse = { + id: 123, + full_name: 'backstage/mock', + default_branch: 'main', + branches_url: + 'https://api.github.com/repos/backstage/mock/branches{/branch}', + archive_url: + 'https://api.github.com/repos/backstage/mock/{archive_format}{/ref}', + trees_url: + 'https://api.github.com/repos/backstage/mock/git/trees{/sha}', + } as Partial; + + const gheResponse = { + id: 123, + full_name: 'backstage/mock', + default_branch: 'main', + branches_url: + 'https://ghe.github.com/api/v3/repos/backstage/mock/branches{/branch}', + archive_url: + 'https://ghe.github.com/api/v3/repos/backstage/mock/{archive_format}{/ref}', + trees_url: + 'https://ghe.github.com/api/v3/repos/backstage/mock/git/trees{/sha}', + } as Partial; + + worker.use( + rest.get('https://api.github.com/repos/backstage/mock', (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(githubResponse), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(gheResponse), + ), + ), + ); + }); + + // Branch details + beforeEach(() => { + const response = { + name: 'main', + commit: { + sha: 'etag123abc', + }, + } as Partial; + + worker.use( + rest.get( + 'https://api.github.com/repos/backstage/mock/branches/main', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(response), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/branches/main', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(response), + ), + ), + rest.get( + 'https://api.github.com/repos/backstage/mock/branches/branchDoesNotExist', + (_, res, ctx) => res(ctx.status(404)), + ), + ); + }); + + // Blobs + beforeEach(() => { + const blob1Response = { + content: Buffer.from('site_name: Test\n').toString('base64'), + } as Partial; + + const blob3Response = { + content: Buffer.from('# Test\n').toString('base64'), + } as Partial; + + worker.use( + rest.get( + 'https://api.github.com/repos/backstage/mock/git/blobs/1', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(blob1Response), + ), + ), + rest.get( + 'https://api.github.com/repos/backstage/mock/git/blobs/3', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(blob3Response), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/git/blobs/1', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(blob1Response), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/git/blobs/3', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(blob3Response), + ), + ), + ); + }); + + async function runTests(reader: GithubUrlReader, baseUrl: string) { + const r1 = await reader.search( + `${baseUrl}/backstage/mock/tree/main/**/*`, + ); + expect(r1.etag).toBe('etag123abc'); + expect(r1.files.length).toBe(2); + + const r2 = await reader.search( + `${baseUrl}/backstage/mock/tree/main/**/*`, + { etag: 'somethingElse' }, + ); + expect(r2.etag).toBe('etag123abc'); + expect(r2.files.length).toBe(2); + + const r3 = await reader.search(`${baseUrl}/backstage/mock/tree/main/o`); + expect(r3.files.length).toBe(0); + + const r4 = await reader.search( + `${baseUrl}/backstage/mock/tree/main/*docs*`, + ); + expect(r4.files.length).toBe(1); + expect(r4.files[0].url).toBe( + `${baseUrl}/backstage/mock/tree/main/mkdocs.yml`, + ); + await expect(r4.files[0].content()).resolves.toEqual( + Buffer.from('site_name: Test\n'), + ); + + const r5 = await reader.search( + `${baseUrl}/backstage/mock/tree/main/*/index.*`, + ); + expect(r5.files.length).toBe(1); + expect(r5.files[0].url).toBe( + `${baseUrl}/backstage/mock/tree/main/docs/index.md`, + ); + await expect(r5.files[0].content()).resolves.toEqual( + Buffer.from('# Test\n'), + ); + } + + // eslint-disable-next-line jest/expect-expect + it('succeeds on github when going via repo listing', async () => { + worker.use( + rest.get( + 'https://api.github.com/repos/backstage/mock/git/trees/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + truncated: false, + tree: githubTreeContents, + } as Partial), + ), + ), + ); + await runTests(githubProcessor, 'https://github.com'); + }); + + // eslint-disable-next-line jest/expect-expect + it('succeeds on github when going via readTree', async () => { + worker.use( + rest.get( + 'https://api.github.com/repos/backstage/mock/git/trees/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + truncated: true, + tree: [], + } as Partial), + ), + ), + ); + await runTests(githubProcessor, 'https://github.com'); + }); + + // eslint-disable-next-line jest/expect-expect + it('succeeds on ghe when going via repo listing', async () => { + worker.use( + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/git/trees/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + truncated: false, + tree: gheTreeContents, + } as Partial), + ), + ), + ); + await runTests(gheProcessor, 'https://ghe.github.com'); + }); + + // eslint-disable-next-line jest/expect-expect + it('succeeds on ghe when going via readTree', async () => { + worker.use( + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/git/trees/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + truncated: true, + tree: [], + } as Partial), + ), + ), + ); + await runTests(gheProcessor, 'https://ghe.github.com'); + }); + + it('throws NotModifiedError when same etag', async () => { + await expect( + githubProcessor.search( + 'https://githib.com/backstage/mock/tree/main/**/*', + { etag: 'etag123abc' }, + ), + ).rejects.toThrow(NotModifiedError); + }); + + it('throws NotFoundError when missing branch', async () => { + await expect( + githubProcessor.search( + 'https://githib.com/backstage/mock/tree/branchDoesNotExist/**/*', + ), + ).rejects.toThrow(NotFoundError); + }); + }); }); diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 4f1aa7c07b..291740a9fc 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -15,13 +15,15 @@ */ import { - GitHubIntegrationConfig, - readGitHubIntegrationConfigs, getGitHubFileFetchUrl, GithubCredentialsProvider, + GitHubIntegrationConfig, + readGitHubIntegrationConfigs, } from '@backstage/integration'; +import { RestEndpointMethodTypes } from '@octokit/rest'; import fetch from 'cross-fetch'; import parseGitUrl from 'git-url-parse'; +import { Minimatch } from 'minimatch'; import { Readable } from 'stream'; import { NotFoundError, NotModifiedError } from '../errors'; import { ReadTreeResponseFactory } from './tree'; @@ -29,9 +31,17 @@ import { ReaderFactory, ReadTreeOptions, ReadTreeResponse, + SearchOptions, + SearchResponse, + SearchResponseFile, UrlReader, } from './types'; +export type GhRepoResponse = RestEndpointMethodTypes['repos']['get']['response']['data']; +export type GhBranchResponse = RestEndpointMethodTypes['repos']['getBranch']['response']['data']; +export type GhTreeResponse = RestEndpointMethodTypes['git']['getTree']['response']['data']; +export type GhBlobResponse = RestEndpointMethodTypes['git']['getBlob']['response']['data']; + /** * A processor that adds the ability to read files from GitHub v3 APIs, such as * the one exposed by GitHub itself. @@ -98,86 +108,190 @@ export class GithubUrlReader implements UrlReader { url: string, options?: ReadTreeOptions, ): Promise { - const { ref, filepath, full_name } = parseGitUrl(url); - // Caveat: The ref will totally be incorrect if the branch name includes a / - // Thus, readTree can not work on url containing branch name that has a / - - const { headers } = await this.deps.credentialsProvider.getCredentials({ - url, - }); - - // Get GitHub API urls for the repository - const repoGitHubResponse = await fetch( - new URL(`${this.config.apiBaseUrl}/repos/${full_name}`).toString(), - { - headers, - }, - ); - if (!repoGitHubResponse.ok) { - const message = `Failed to read tree (repository) from ${url}, ${repoGitHubResponse.status} ${repoGitHubResponse.statusText}`; - if (repoGitHubResponse.status === 404) { - throw new NotFoundError(message); - } - throw new Error(message); - } - - const repoResponseJson = await repoGitHubResponse.json(); - - // ref is an empty string if no branch is set in provided url to readTree. - // Use GitHub API to get the default branch of the repository. - const branch = ref || repoResponseJson.default_branch; - const branchesApiUrl = repoResponseJson.branches_url; - const archiveApiUrl = repoResponseJson.archive_url; - - // Fetch the latest commit in the provided or default branch to compare against - // the provided sha. - const branchGitHubResponse = await fetch( - // branchesApiUrl looks like "https://api.github.com/repos/owner/repo/branches{/branch}" - branchesApiUrl.replace('{/branch}', `/${branch}`), - { - headers, - }, - ); - if (!branchGitHubResponse.ok) { - const message = `Failed to read tree (branch) from ${url}, ${branchGitHubResponse.status} ${branchGitHubResponse.statusText}`; - if (branchGitHubResponse.status === 404) { - throw new NotFoundError(message); - } - throw new Error(message); - } - const commitSha = (await branchGitHubResponse.json()).commit.sha; + const repoDetails = await this.getRepoDetails(url); + const commitSha = repoDetails.branch.commit.sha!; if (options?.etag && options.etag === commitSha) { throw new NotModifiedError(); } - const archive = await fetch( - // archiveApiUrl looks like "https://api.github.com/repos/owner/repo/{archive_format}{/ref}" - archiveApiUrl - .replace('{archive_format}', 'tarball') - .replace('{/ref}', `/${commitSha}`), + const { filepath } = parseGitUrl(url); + const { headers } = await this.deps.credentialsProvider.getCredentials({ + url, + }); + + return this.doReadTree( + repoDetails.repo.archive_url, + commitSha, + filepath, { headers }, + options, ); - if (!archive.ok) { - const message = `Failed to read tree (archive) from ${url}, ${archive.status} ${archive.statusText}`; - if (archive.status === 404) { - throw new NotFoundError(message); - } - throw new Error(message); + } + + async search(url: string, options?: SearchOptions): Promise { + const repoDetails = await this.getRepoDetails(url); + const commitSha = repoDetails.branch.commit.sha!; + + if (options?.etag && options.etag === commitSha) { + throw new NotModifiedError(); } - return await this.deps.treeResponseFactory.fromTarArchive({ - // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want - // to stick to using that in exclusively backend code. - stream: (archive.body as unknown) as Readable, - subpath: filepath, - etag: commitSha, - filter: options?.filter, + const { filepath } = parseGitUrl(url); + const { headers } = await this.deps.credentialsProvider.getCredentials({ + url, }); + + const files = await this.doSearch( + url, + repoDetails.repo.trees_url, + repoDetails.repo.archive_url, + commitSha, + filepath, + { headers }, + ); + + return { files, etag: commitSha }; } toString() { const { host, token } = this.config; return `github{host=${host},authed=${Boolean(token)}}`; } + + private async doReadTree( + archiveUrl: string, + sha: string, + subpath: string, + init: RequestInit, + options?: ReadTreeOptions, + ): Promise { + // archive_url looks like "https://api.github.com/repos/owner/repo/{archive_format}{/ref}" + const archive = await this.fetchResponse( + archiveUrl + .replace('{archive_format}', 'tarball') + .replace('{/ref}', `/${sha}`), + init, + ); + + return await this.deps.treeResponseFactory.fromTarArchive({ + // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want + // to stick to using that in exclusively backend code. + stream: (archive.body as unknown) as Readable, + subpath, + etag: sha, + filter: options?.filter, + }); + } + + private async doSearch( + url: string, + treesUrl: string, + archiveUrl: string, + sha: string, + query: string, + init: RequestInit, + ): Promise { + function pathToUrl(path: string): string { + // TODO(freben): Use the integration package facility for this instead + // pathname starts as /backstage/backstage/blob/master/ + const updated = new URL(url); + const base = updated.pathname.split('/').slice(1, 5).join('/'); + updated.pathname = `${base}/${path}`; + return updated.toString(); + } + + const matcher = new Minimatch(query.replace(/^\/+/, '')); + + // trees_url looks like "https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}" + const recursiveTree: GhTreeResponse = await this.fetchJson( + treesUrl.replace('{/sha}', `/${sha}?recursive=true`), + init, + ); + + // The simple case is that we got the entire tree in a single operation. + if (!recursiveTree.truncated) { + const matching = recursiveTree.tree.filter( + item => + item.type === 'blob' && + item.path && + item.url && + matcher.match(item.path), + ); + + return matching.map(item => ({ + url: pathToUrl(item.path!), + content: async () => { + const blob: GhBlobResponse = await this.fetchJson(item.url!, init); + return Buffer.from(blob.content, 'base64'); + }, + })); + } + + // For larger repos, we leverage readTree and filter through that instead + const tree = await this.doReadTree(archiveUrl, sha, '', init, { + filter: path => matcher.match(path), + }); + const files = await tree.files(); + + return files.map(file => ({ + url: pathToUrl(file.path), + content: file.content, + })); + } + + private async getRepoDetails( + url: string, + ): Promise<{ + repo: GhRepoResponse; + branch: GhBranchResponse; + }> { + const parsed = parseGitUrl(url); + const { ref, full_name } = parsed; + + // Caveat: The ref will totally be incorrect if the branch name includes a + // slash. Thus, some operations can not work on URLs containing branch + // names that have a slash in them. + + const { headers } = await this.deps.credentialsProvider.getCredentials({ + url, + }); + + const repo: GhRepoResponse = await this.fetchJson( + `${this.config.apiBaseUrl}/repos/${full_name}`, + { headers }, + ); + + // branches_url looks like "https://api.github.com/repos/owner/repo/branches{/branch}" + const branch: GhBranchResponse = await this.fetchJson( + repo.branches_url.replace('{/branch}', `/${ref || repo.default_branch}`), + { headers }, + ); + + return { repo, branch }; + } + + private async fetchResponse( + url: string | URL, + init: RequestInit, + ): Promise { + const urlAsString = url.toString(); + + const response = await fetch(urlAsString, init); + + if (!response.ok) { + const message = `Request failed for ${urlAsString}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + return response; + } + + private async fetchJson(url: string | URL, init: RequestInit): Promise { + const response = await this.fetchResponse(url, init); + return await response.json(); + } } diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 8a763d1259..5e02a9b8e6 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -21,16 +21,17 @@ import { readGitLabIntegrationConfigs, } from '@backstage/integration'; import fetch from 'cross-fetch'; +import parseGitUrl from 'git-url-parse'; +import { Readable } from 'stream'; import { NotFoundError, NotModifiedError } from '../errors'; import { ReadTreeResponseFactory } from './tree'; import { ReaderFactory, ReadTreeOptions, ReadTreeResponse, + SearchResponse, UrlReader, } from './types'; -import parseGitUrl from 'git-url-parse'; -import { Readable } from 'stream'; export class GitlabUrlReader implements UrlReader { private readonly treeResponseFactory: ReadTreeResponseFactory; @@ -154,6 +155,10 @@ export class GitlabUrlReader implements UrlReader { }); } + async search(): Promise { + throw new Error('GitlabUrlReader does not implement search'); + } + toString() { const { host, token } = this.config; return `gitlab{host=${host},authed=${Boolean(token)}}`; diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index 3183aa0c28..91f4ae63ab 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -18,6 +18,8 @@ import { NotAllowedError } from '../errors'; import { ReadTreeOptions, ReadTreeResponse, + SearchOptions, + SearchResponse, UrlReader, UrlReaderPredicateTuple, } from './types'; @@ -60,6 +62,18 @@ export class UrlReaderPredicateMux implements UrlReader { throw new NotAllowedError(`Reading from '${url}' is not allowed`); } + async search(url: string, options?: SearchOptions): Promise { + const parsed = new URL(url); + + for (const { predicate, reader } of this.readers) { + if (predicate(parsed)) { + return await reader.search(url, options); + } + } + + throw new NotAllowedError(`Reading from '${url}' is not allowed`); + } + toString() { return `predicateMux{readers=${this.readers.map(t => t.reader).join(',')}`; } diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 8a6c2bf1a3..1266161ddf 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -export type { UrlReader, ReadTreeResponse } from './types'; +export type { UrlReader, ReadTreeResponse, SearchResponse } from './types'; export { UrlReaders } from './UrlReaders'; export { AzureUrlReader } from './AzureUrlReader'; export { BitbucketUrlReader } from './BitbucketUrlReader'; diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts index 24bfa3f1a8..d22529293f 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -14,16 +14,16 @@ * limitations under the License. */ -import tar, { Parse, ParseStream, ReadEntry } from 'tar'; -import platformPath from 'path'; -import fs from 'fs-extra'; -import { Readable, pipeline as pipelineCb } from 'stream'; -import { promisify } from 'util'; import concatStream from 'concat-stream'; +import fs from 'fs-extra'; +import platformPath from 'path'; +import { pipeline as pipelineCb, Readable } from 'stream'; +import tar, { Parse, ParseStream, ReadEntry } from 'tar'; +import { promisify } from 'util'; import { ReadTreeResponse, - ReadTreeResponseFile, ReadTreeResponseDirOptions, + ReadTreeResponseFile, } from '../types'; // Tar types for `Parse` is not a proper constructor, but it should be diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 323f7b4778..2547c7b0e4 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -14,15 +14,15 @@ * limitations under the License. */ -import platformPath from 'path'; -import fs from 'fs-extra'; -import unzipper, { Entry } from 'unzipper'; import archiver from 'archiver'; +import fs from 'fs-extra'; +import platformPath from 'path'; import { Readable } from 'stream'; +import unzipper, { Entry } from 'unzipper'; import { ReadTreeResponse, - ReadTreeResponseFile, ReadTreeResponseDirOptions, + ReadTreeResponseFile, } from '../types'; // Matches a directory name + one `/` at the start of any string, diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index f9dfbe9aff..4caa76841a 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -18,6 +18,33 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; import { ReadTreeResponseFactory } from './tree'; +/** + * A generic interface for fetching plain data from URLs. + */ +export type UrlReader = { + read(url: string): Promise; + readTree(url: string, options?: ReadTreeOptions): Promise; + search(url: string, options?: SearchOptions): Promise; +}; + +export type UrlReaderPredicateTuple = { + predicate: (url: URL) => boolean; + reader: UrlReader; +}; + +/** + * A factory function that can read config to construct zero or more + * UrlReaders along with a predicate for when it should be used. + */ +export type ReaderFactory = (options: { + config: Config; + logger: Logger; + treeResponseFactory: ReadTreeResponseFactory; +}) => UrlReaderPredicateTuple[]; + +/** + * An options object for readTree operations. + */ export type ReadTreeOptions = { /** * A filter that can be used to select which files should be included. @@ -47,39 +74,6 @@ export type ReadTreeOptions = { etag?: string; }; -/** - * A generic interface for fetching plain data from URLs. - */ -export type UrlReader = { - read(url: string): Promise; - readTree(url: string, options?: ReadTreeOptions): Promise; -}; - -export type UrlReaderPredicateTuple = { - predicate: (url: URL) => boolean; - reader: UrlReader; -}; - -/** - * A factory function that can read config to construct zero or more - * UrlReaders along with a predicate for when it should be used. - */ -export type ReaderFactory = (options: { - config: Config; - logger: Logger; - treeResponseFactory: ReadTreeResponseFactory; -}) => UrlReaderPredicateTuple[]; - -export type ReadTreeResponseFile = { - path: string; - content(): Promise; -}; - -export type ReadTreeResponseDirOptions = { - /** The directory to write files to. Defaults to the OS tmpdir or `backend.workingDirectory` if set in config */ - targetDir?: string; -}; - export type ReadTreeResponse = { /** * files() returns an array of all the files inside the tree and corresponding functions to read their content. @@ -97,3 +91,64 @@ export type ReadTreeResponse = { */ etag: string; }; + +export type ReadTreeResponseDirOptions = { + /** The directory to write files to. Defaults to the OS tmpdir or `backend.workingDirectory` if set in config */ + targetDir?: string; +}; + +/** + * Represents a single file in a readTree response. + */ +export type ReadTreeResponseFile = { + path: string; + content(): Promise; +}; + +/** + * An options object for search operations. + */ +export type SearchOptions = { + /** + * An etag can be provided to check whether the search response has changed from a previous execution. + * + * In the search() response, an etag is returned along with the files. The etag is a unique identifer + * of the current tree, usually the commit SHA or etag from the target. + * + * When an etag is given in SearchOptions, search will first compare the etag against the etag + * on the target branch. If they match, search will throw a NotModifiedError indicating that the search + * response will not differ from the previous response which included this particular etag. If they mismatch, + * search will return the rest of SearchResponse along with a new etag. + */ + etag?: string; +}; + +/** + * The output of a search operation. + */ +export type SearchResponse = { + /** + * The files that matched the search query. + */ + files: SearchResponseFile[]; + + /** + * A unique identifer of the current remote tree, usually the commit SHA or etag from the target. + */ + etag: string; +}; + +/** + * Represents a single file in a search response. + */ +export type SearchResponseFile = { + /** + * The full URL to the file. + */ + url: string; + + /** + * The binary contents of the file. + */ + content(): Promise; +}; diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 4e2257a46c..723165ff82 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -30,6 +30,7 @@ import Docker from 'dockerode'; export default async function createPlugin({ logger, config, + database, }: PluginEnvironment) { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); @@ -54,5 +55,6 @@ export default async function createPlugin({ config, dockerClient, entityClient, + database, }); } diff --git a/packages/cli/package.json b/packages/cli/package.json index fb33a154a7..9aaeb00459 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -93,7 +93,7 @@ "rollup-plugin-esbuild": "2.6.x", "rollup-plugin-peer-deps-external": "^2.2.2", "rollup-plugin-postcss": "^3.1.1", - "rollup-plugin-typescript2": "^0.27.3", + "rollup-plugin-typescript2": "^0.29.0", "rollup-pluginutils": "^2.8.2", "semver": "^7.3.2", "start-server-webpack-plugin": "^2.2.5", diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts index c8bd3e5012..d68f90ce08 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts @@ -14,6 +14,7 @@ import Docker from 'dockerode'; export default async function createPlugin({ logger, config, + database, }: PluginEnvironment) { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); @@ -38,5 +39,6 @@ export default async function createPlugin({ config, dockerClient, entityClient, + database, }); } diff --git a/packages/integration/src/github/GithubCredentialsProvider.ts b/packages/integration/src/github/GithubCredentialsProvider.ts index a22c69909b..0bf217c824 100644 --- a/packages/integration/src/github/GithubCredentialsProvider.ts +++ b/packages/integration/src/github/GithubCredentialsProvider.ts @@ -53,7 +53,7 @@ class Cache { /** * This accept header is required when calling App APIs in GitHub Enterprise. - * It has no effect on calls to github.com and can probably be removed entierly + * It has no effect on calls to github.com and can probably be removed entirely * once GitHub Apps is out of preview. */ const HEADERS = { diff --git a/packages/techdocs-common/src/helpers.test.ts b/packages/techdocs-common/src/helpers.test.ts index 740a08aaa2..eba1386b11 100644 --- a/packages/techdocs-common/src/helpers.test.ts +++ b/packages/techdocs-common/src/helpers.test.ts @@ -14,14 +14,18 @@ * limitations under the License. */ +import { + ReadTreeResponse, + SearchResponse, + UrlReader, +} from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import { Readable } from 'stream'; import { getDocFilesFromRepository, getLocationForEntity, parseReferenceAnnotation, } from './helpers'; -import { UrlReader, ReadTreeResponse } from '@backstage/backend-common'; -import { Entity } from '@backstage/catalog-model'; const entityBase: Entity = { metadata: { @@ -138,6 +142,13 @@ describe('getDocFilesFromRepository', () => { etag: '', }; } + + async search(): Promise { + return { + etag: '', + files: [], + }; + } } const output = await getDocFilesFromRepository( diff --git a/packages/techdocs-common/src/stages/prepare/dir.test.ts b/packages/techdocs-common/src/stages/prepare/dir.test.ts index 630608e817..cd8d358afe 100644 --- a/packages/techdocs-common/src/stages/prepare/dir.test.ts +++ b/packages/techdocs-common/src/stages/prepare/dir.test.ts @@ -49,6 +49,7 @@ const mockConfig = new ConfigReader({}); const mockUrlReader: jest.Mocked = { read: jest.fn(), readTree: jest.fn(), + search: jest.fn(), }; describe('directory preparer', () => { diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts index ba97232810..467c0eb2e8 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts @@ -130,7 +130,7 @@ export class HigherOrderOperations implements HigherOrderOperation { `Locations Refresh: Refreshing location ${location.type}:${location.target}`, ); try { - await this.refreshSingleLocation(location); + await this.refreshSingleLocation(location, logger); await this.locationsCatalog.logUpdateSuccess(location.id, undefined); } catch (e) { logger.warn( @@ -148,8 +148,12 @@ export class HigherOrderOperations implements HigherOrderOperation { } // Performs a full refresh of a single location - private async refreshSingleLocation(location: Location) { + private async refreshSingleLocation( + location: Location, + optionalLogger?: Logger, + ) { let startTimestamp = process.hrtime(); + const logger = optionalLogger || this.logger; const readerOutput = await this.locationReader.read({ type: location.type, @@ -157,12 +161,12 @@ export class HigherOrderOperations implements HigherOrderOperation { }); for (const item of readerOutput.errors) { - this.logger.warn( + logger.warn( `Failed item in location ${item.location.type}:${item.location.target}, ${item.error.stack}`, ); } - this.logger.info( + logger.info( `Read ${readerOutput.entities.length} entities from location ${ location.type }:${location.target} in ${durationText(startTimestamp)}`, @@ -186,14 +190,14 @@ export class HigherOrderOperations implements HigherOrderOperation { throw e; } - this.logger.debug(`Posting update success markers`); + logger.debug(`Posting update success markers`); await this.locationsCatalog.logUpdateSuccess( location.id, readerOutput.entities.map(e => e.entity.metadata.name), ); - this.logger.info( + logger.info( `Wrote ${readerOutput.entities.length} entities from location ${ location.type }:${location.target} in ${durationText(startTimestamp)}`, diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index a789e669d0..be88aa2810 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -139,6 +139,16 @@ export class LocationReaders implements LocationReader { if (emitResult.type === 'relation') { throw new Error('readLocation may not emit entity relations'); } + if ( + emitResult.type === 'location' && + emitResult.location.type === item.location.type && + emitResult.location.target === item.location.target + ) { + // Ignore self-referential locations silently (this can happen for + // example if you use a glob target like "**/*.yaml" in a Location + // entity) + return; + } emit(emitResult); }; diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts index be4f15e501..eb1187eda5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts @@ -160,7 +160,7 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockResolvedValue(mockReadResult({ data: ownersText })); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; const result = await findRawCodeOwners(mockLocation(), { reader, logger, @@ -170,7 +170,7 @@ describe('CodeOwnersProcessor', () => { it('should return undefined when no codeowner', async () => { const read = jest.fn().mockRejectedValue(mockReadResult()); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; await expect( findRawCodeOwners(mockLocation(), { reader, logger }), @@ -184,7 +184,7 @@ describe('CodeOwnersProcessor', () => { .mockImplementationOnce(() => mockReadResult({ error: 'foo' })) .mockImplementationOnce(() => mockReadResult({ error: 'bar' })) .mockResolvedValue(mockReadResult({ data: ownersText })); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; const result = await findRawCodeOwners(mockLocation(), { reader, @@ -206,7 +206,7 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() })); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; const owner = await resolveCodeOwner(mockLocation(), { reader, logger }); expect(owner).toBe('backstage-core'); @@ -216,7 +216,7 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockImplementation(() => mockReadResult({ error: 'error: foo' })); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; await expect( resolveCodeOwner(mockLocation(), { reader, logger }), @@ -230,7 +230,7 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() })); - const reader = { read, readTree: jest.fn() }; + const reader = { read, readTree: jest.fn(), search: jest.fn() }; const processor = new CodeOwnersProcessor({ reader, logger }); return { entity, processor, read }; diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts index 8b2632ad5d..00b6af94e5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts @@ -27,7 +27,7 @@ import { describe('PlaceholderProcessor', () => { const read: jest.MockedFunction = jest.fn(); - const reader: UrlReader = { read, readTree: jest.fn() }; + const reader: UrlReader = { read, readTree: jest.fn(), search: jest.fn() }; beforeEach(() => { jest.resetAllMocks(); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts index 49b6fe71e0..db731b5b17 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -14,24 +14,29 @@ * limitations under the License. */ -import { UrlReaderProcessor } from './UrlReaderProcessor'; -import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; +import { + getVoidLogger, + UrlReader, + UrlReaders, +} from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; +import { msw } from '@backstage/test-utils'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import { msw } from '@backstage/test-utils'; import { CatalogProcessorEntityResult, CatalogProcessorErrorResult, CatalogProcessorResult, } from './types'; +import { UrlReaderProcessor } from './UrlReaderProcessor'; import { defaultEntityDataParser } from './util/parse'; describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost'; - const server = setupServer(); + const server = setupServer(); msw.setupDefaultHandlers(server); + it('should load from url', async () => { const logger = getVoidLogger(); const reader = UrlReaders.default({ @@ -57,7 +62,7 @@ describe('UrlReaderProcessor', () => { )) as CatalogProcessorEntityResult; expect(generated.type).toBe('entity'); - expect(generated.location).toBe(spec); + expect(generated.location).toEqual(spec); expect(generated.entity).toEqual({ mock: 'entity' }); }); @@ -92,4 +97,27 @@ describe('UrlReaderProcessor', () => { `Unable to read url, NotFoundError: could not read ${mockApiOrigin}/component-notfound.yaml, 404 Not Found`, ); }); + + it('uses search when there are globs', async () => { + const logger = getVoidLogger(); + + const reader: jest.Mocked = { + read: jest.fn(), + readTree: jest.fn(), + search: jest.fn().mockImplementation(async () => []), + }; + + const processor = new UrlReaderProcessor({ reader, logger }); + + const emit = jest.fn(); + + await processor.readLocation( + { type: 'url', target: 'https://github.com/a/b/blob/x/**/b.yaml' }, + false, + emit, + defaultEntityDataParser, + ); + + expect(reader.search).toBeCalledTimes(1); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index 9daae29d3b..b70ef2877d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -16,6 +16,8 @@ import { UrlReader } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; +import parseGitUrl from 'git-url-parse'; +import limiterFactory from 'p-limit'; import { Logger } from 'winston'; import * as result from './results'; import { @@ -59,10 +61,14 @@ export class UrlReaderProcessor implements CatalogProcessor { } try { - const data = await this.options.reader.read(location.target); - - for await (const parseResult of parser({ data, location })) { - emit(parseResult); + const output = await this.doRead(location.target); + for (const item of output) { + for await (const parseResult of parser({ + data: item.data, + location: { type: location.type, target: item.url }, + })) { + emit(parseResult); + } } } catch (error) { const message = `Unable to read ${location.type}, ${error}`; @@ -78,4 +84,25 @@ export class UrlReaderProcessor implements CatalogProcessor { return true; } + + private async doRead( + location: string, + ): Promise<{ data: Buffer; url: string }[]> { + // Does it contain globs? I.e. does it contain asterisks or question marks + // (no curly braces for now) + const { filepath } = parseGitUrl(location); + if (filepath?.match(/[*?]/)) { + const limiter = limiterFactory(5); + const response = await this.options.reader.search(location); + const output = response.files.map(async file => ({ + url: file.url, + data: await limiter(file.content), + })); + return Promise.all(output); + } + + // Otherwise do a plain read + const data = await this.options.reader.read(location); + return [{ url: location, data }]; + } } diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts index 695902fec9..bc58338fce 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts @@ -44,6 +44,7 @@ describe('CatalogBuilder', () => { const reader: jest.Mocked = { read: jest.fn(), readTree: jest.fn(), + search: jest.fn(), }; const env: CatalogEnvironment = { logger: getVoidLogger(), diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index 6064164e31..17cfc77d8b 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { errorHandler } from '@backstage/backend-common'; +import { errorHandler, NotFoundError } from '@backstage/backend-common'; import { locationSpecSchema, analyzeLocationSchema, @@ -57,7 +57,7 @@ export async function createRouter( const filter = EntityFilters.ofQuery(req.query); const fieldMapper = translateQueryToFieldMapper(req.query); const entities = await entitiesCatalog.entities(filter); - res.status(200).send(entities.map(fieldMapper)); + res.status(200).json(entities.map(fieldMapper)); }) .post('/entities', async (req, res) => { const body = await requireRequestBody(req); @@ -67,7 +67,7 @@ export async function createRouter( const [entity] = await entitiesCatalog.entities( EntityFilters.ofMatchers({ 'metadata.uid': result.entityId }), ); - res.status(200).send(entity); + res.status(200).json(entity); }) .get('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; @@ -75,15 +75,14 @@ export async function createRouter( EntityFilters.ofMatchers({ 'metadata.uid': uid }), ); if (!entities.length) { - res.status(404).send(`No entity with uid ${uid}`); - } else { - res.status(200).send(entities[0]); + throw new NotFoundError(`No entity with uid ${uid}`); } + res.status(200).json(entities[0]); }) .delete('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; await entitiesCatalog.removeEntityByUid(uid); - res.status(204).send(); + res.status(204).end(); }) .get('/entities/by-name/:kind/:namespace/:name', async (req, res) => { const { kind, namespace, name } = req.params; @@ -95,14 +94,11 @@ export async function createRouter( }), ); if (!entities.length) { - res - .status(404) - .send( - `No entity with kind ${kind} namespace ${namespace} name ${name}`, - ); - } else { - res.status(200).send(entities[0]); + throw new NotFoundError( + `No entity with kind ${kind} namespace ${namespace} name ${name}`, + ); } + res.status(200).json(entities[0]); }); } @@ -111,7 +107,7 @@ export async function createRouter( const input = await validateRequestBody(req, locationSpecSchema); const dryRun = yn(req.query.dryRun, { default: false }); const output = await higherOrderOperation.addLocation(input, { dryRun }); - res.status(201).send(output); + res.status(201).json(output); }); } @@ -119,22 +115,22 @@ export async function createRouter( router .get('/locations', async (_req, res) => { const output = await locationsCatalog.locations(); - res.status(200).send(output); + res.status(200).json(output); }) .get('/locations/:id/history', async (req, res) => { const { id } = req.params; const output = await locationsCatalog.locationHistory(id); - res.status(200).send(output); + res.status(200).json(output); }) .get('/locations/:id', async (req, res) => { const { id } = req.params; const output = await locationsCatalog.location(id); - res.status(200).send(output); + res.status(200).json(output); }) .delete('/locations/:id', async (req, res) => { const { id } = req.params; await locationsCatalog.removeLocation(id); - res.status(204).send(); + res.status(204).end(); }); } @@ -142,7 +138,7 @@ export async function createRouter( router.post('/analyze-location', async (req, res) => { const input = await validateRequestBody(req, analyzeLocationSchema); const output = await locationAnalyzer.analyzeLocation(input); - res.status(200).send(output); + res.status(200).json(output); }); } diff --git a/plugins/circleci/dev/index.tsx b/plugins/circleci/dev/index.tsx index 812a5585d4..19a32c0d04 100644 --- a/plugins/circleci/dev/index.tsx +++ b/plugins/circleci/dev/index.tsx @@ -15,6 +15,6 @@ */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { circleCIPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(circleCIPlugin).render(); diff --git a/plugins/circleci/src/components/Router.tsx b/plugins/circleci/src/components/Router.tsx index 5fe9a20eff..d18b6c1e61 100644 --- a/plugins/circleci/src/components/Router.tsx +++ b/plugins/circleci/src/components/Router.tsx @@ -21,15 +21,25 @@ import { BuildWithStepsPage } from './BuildWithStepsPage/'; import { BuildsPage } from './BuildsPage'; import { CIRCLECI_ANNOTATION } from '../constants'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { MissingAnnotationEmptyState } from '@backstage/core'; -export const isPluginApplicableToEntity = (entity: Entity) => +export const isCircleCIAvailable = (entity: Entity) => Boolean(entity.metadata.annotations?.[CIRCLECI_ANNOTATION]); -export const Router = ({ entity }: { entity: Entity }) => - !isPluginApplicableToEntity(entity) ? ( - - ) : ( +type Props = { + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}; + +export const Router = (_props: Props) => { + const { entity } = useEntity(); + + if (!isCircleCIAvailable(entity)) { + return ; + } + + return ( } /> /> ); +}; diff --git a/plugins/circleci/src/index.ts b/plugins/circleci/src/index.ts index c6cb140208..2a37cbdee0 100644 --- a/plugins/circleci/src/index.ts +++ b/plugins/circleci/src/index.ts @@ -14,8 +14,16 @@ * limitations under the License. */ -export { plugin } from './plugin'; +export { + circleCIPlugin, + circleCIPlugin as plugin, + EntityCircleCIContent, +} from './plugin'; export * from './api'; export * from './route-refs'; -export { Router, isPluginApplicableToEntity } from './components/Router'; +export { + Router, + isCircleCIAvailable, + isCircleCIAvailable as isPluginApplicableToEntity, +} from './components/Router'; export { CIRCLECI_ANNOTATION } from './constants'; diff --git a/plugins/circleci/src/plugin.test.ts b/plugins/circleci/src/plugin.test.ts index 821a503257..bdd637027c 100644 --- a/plugins/circleci/src/plugin.test.ts +++ b/plugins/circleci/src/plugin.test.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import { plugin } from './plugin'; +import { circleCIPlugin } from './plugin'; describe('circleci', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(circleCIPlugin).toBeDefined(); }); }); diff --git a/plugins/circleci/src/plugin.ts b/plugins/circleci/src/plugin.ts index c5bb122ed3..4e0d485726 100644 --- a/plugins/circleci/src/plugin.ts +++ b/plugins/circleci/src/plugin.ts @@ -18,10 +18,12 @@ import { createPlugin, createApiFactory, discoveryApiRef, + createRoutableExtension, } from '@backstage/core'; import { circleCIApiRef, CircleCIApi } from './api'; +import { circleCIRouteRef } from './route-refs'; -export const plugin = createPlugin({ +export const circleCIPlugin = createPlugin({ id: 'circleci', apis: [ createApiFactory({ @@ -31,3 +33,10 @@ export const plugin = createPlugin({ }), ], }); + +export const EntityCircleCIContent = circleCIPlugin.provide( + createRoutableExtension({ + component: () => import('./components/Router').then(m => m.Router), + mountPoint: circleCIRouteRef, + }), +); diff --git a/plugins/cloudbuild/dev/index.tsx b/plugins/cloudbuild/dev/index.tsx index 264d6f801f..26fb65d151 100644 --- a/plugins/cloudbuild/dev/index.tsx +++ b/plugins/cloudbuild/dev/index.tsx @@ -14,6 +14,6 @@ * limitations under the License. */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { cloudbuildPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(cloudbuildPlugin).render(); diff --git a/plugins/cloudbuild/package.json b/plugins/cloudbuild/package.json index 41a5285d7d..cbad4928b7 100644 --- a/plugins/cloudbuild/package.json +++ b/plugins/cloudbuild/package.json @@ -31,6 +31,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.1", + "@backstage/plugin-catalog-react": "^0.0.2", "@backstage/core": "^0.6.0", "@backstage/theme": "^0.2.3", "@material-ui/core": "^4.11.0", diff --git a/plugins/cloudbuild/src/components/Cards/Cards.tsx b/plugins/cloudbuild/src/components/Cards/Cards.tsx index bbfb0b59c3..75ae33027e 100644 --- a/plugins/cloudbuild/src/components/Cards/Cards.tsx +++ b/plugins/cloudbuild/src/components/Cards/Cards.tsx @@ -17,6 +17,7 @@ import React, { useEffect } from 'react'; import { useWorkflowRuns } from '../useWorkflowRuns'; import { WorkflowRun, WorkflowRunsTable } from '../WorkflowRunsTable'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { WorkflowRunStatus } from '../WorkflowRunStatus'; import { Link, Theme, makeStyles, LinearProgress } from '@material-ui/core'; import { @@ -72,12 +73,13 @@ const WidgetContent = ({ }; export const LatestWorkflowRunCard = ({ - entity, branch = 'master', }: { - entity: Entity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; branch: string; }) => { + const { entity } = useEntity(); const errorApi = useApi(errorApiRef); const projectId = entity?.metadata.annotations?.[CLOUDBUILD_ANNOTATION] || ''; @@ -104,13 +106,17 @@ export const LatestWorkflowRunCard = ({ }; export const LatestWorkflowsForBranchCard = ({ - entity, branch = 'master', }: { - entity: Entity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; branch: string; -}) => ( - - - -); +}) => { + const { entity } = useEntity(); + + return ( + + + + ); +}; diff --git a/plugins/cloudbuild/src/components/Router.tsx b/plugins/cloudbuild/src/components/Router.tsx index af4959f4be..0f71efcf86 100644 --- a/plugins/cloudbuild/src/components/Router.tsx +++ b/plugins/cloudbuild/src/components/Router.tsx @@ -15,6 +15,7 @@ */ import React from 'react'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { Routes, Route } from 'react-router'; import { rootRouteRef, buildRouteRef } from '../plugin'; import { WorkflowRunDetails } from './WorkflowRunDetails'; @@ -22,14 +23,22 @@ import { WorkflowRunsTable } from './WorkflowRunsTable'; import { CLOUDBUILD_ANNOTATION } from './useProjectName'; import { MissingAnnotationEmptyState } from '@backstage/core'; -export const isPluginApplicableToEntity = (entity: Entity) => +export const isCloudbuildAvailable = (entity: Entity) => Boolean(entity.metadata.annotations?.[CLOUDBUILD_ANNOTATION]); -export const Router = ({ entity }: { entity: Entity }) => - // TODO(shmidt-i): move warning to a separate standardized component - !isPluginApplicableToEntity(entity) ? ( - - ) : ( +type Props = { + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}; + +export const Router = (_props: Props) => { + const { entity } = useEntity(); + + if (!isCloudbuildAvailable(entity)) { + // TODO(shmidt-i): move warning to a separate standardized component + return ; + } + return ( ) ); +}; diff --git a/plugins/cloudbuild/src/index.ts b/plugins/cloudbuild/src/index.ts index 616bb6b4e7..b7cd6c29d5 100644 --- a/plugins/cloudbuild/src/index.ts +++ b/plugins/cloudbuild/src/index.ts @@ -13,8 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { plugin } from './plugin'; +export { + cloudbuildPlugin, + cloudbuildPlugin as plugin, + EntityCloudbuildContent, + EntityLatestCloudbuildRunCard, + EntityLatestCloudbuildsForBranchCard, +} from './plugin'; export * from './api'; -export { Router, isPluginApplicableToEntity } from './components/Router'; +export { + Router, + isCloudbuildAvailable, + isCloudbuildAvailable as isPluginApplicableToEntity, +} from './components/Router'; export * from './components/Cards'; export { CLOUDBUILD_ANNOTATION } from './components/useProjectName'; diff --git a/plugins/cloudbuild/src/plugin.test.ts b/plugins/cloudbuild/src/plugin.test.ts index dae16f1d1b..ea042a901a 100644 --- a/plugins/cloudbuild/src/plugin.test.ts +++ b/plugins/cloudbuild/src/plugin.test.ts @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { plugin } from './plugin'; +import { cloudbuildPlugin } from './plugin'; describe('cloudbuild', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(cloudbuildPlugin).toBeDefined(); }); }); diff --git a/plugins/cloudbuild/src/plugin.ts b/plugins/cloudbuild/src/plugin.ts index ee997cc7ec..f5aaf4696a 100644 --- a/plugins/cloudbuild/src/plugin.ts +++ b/plugins/cloudbuild/src/plugin.ts @@ -18,6 +18,8 @@ import { createRouteRef, createApiFactory, googleAuthApiRef, + createRoutableExtension, + createComponentExtension, } from '@backstage/core'; import { cloudbuildApiRef, CloudbuildClient } from './api'; @@ -31,7 +33,7 @@ export const buildRouteRef = createRouteRef({ title: 'Cloudbuild Run', }); -export const plugin = createPlugin({ +export const cloudbuildPlugin = createPlugin({ id: 'cloudbuild', apis: [ createApiFactory({ @@ -42,4 +44,32 @@ export const plugin = createPlugin({ }, }), ], + routes: { + entityContent: rootRouteRef, + }, }); + +export const EntityCloudbuildContent = cloudbuildPlugin.provide( + createRoutableExtension({ + component: () => import('./components/Router').then(m => m.Router), + mountPoint: rootRouteRef, + }), +); + +export const EntityLatestCloudbuildRunCard = cloudbuildPlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./components/Cards').then(m => m.LatestWorkflowRunCard), + }, + }), +); + +export const EntityLatestCloudbuildsForBranchCard = cloudbuildPlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./components/Cards').then(m => m.LatestWorkflowsForBranchCard), + }, + }), +); diff --git a/plugins/cost-insights/src/types/Alert.ts b/plugins/cost-insights/src/types/Alert.ts index eb8919b71a..84c37a7c15 100644 --- a/plugins/cost-insights/src/types/Alert.ts +++ b/plugins/cost-insights/src/types/Alert.ts @@ -149,7 +149,7 @@ export const AlertSnoozeOptions: AlertSnoozeOption[] = [ label: '1 Month', }, { - duration: Duration.P3M, + duration: Duration.P90D, label: '1 Quarter', }, ]; diff --git a/plugins/github-actions/dev/index.tsx b/plugins/github-actions/dev/index.tsx index 812a5585d4..9156224202 100644 --- a/plugins/github-actions/dev/index.tsx +++ b/plugins/github-actions/dev/index.tsx @@ -15,6 +15,6 @@ */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { githubActionsPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(githubActionsPlugin).render(); diff --git a/plugins/github-actions/package.json b/plugins/github-actions/package.json index 49f1b0007a..7d05395b42 100644 --- a/plugins/github-actions/package.json +++ b/plugins/github-actions/package.json @@ -33,6 +33,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.1", + "@backstage/plugin-catalog-react": "^0.0.2", "@backstage/core": "^0.6.0", "@backstage/integration": "^0.3.2", "@backstage/theme": "^0.2.3", diff --git a/plugins/github-actions/src/components/Cards/Cards.tsx b/plugins/github-actions/src/components/Cards/Cards.tsx index f1c27cb666..202bf5aa92 100644 --- a/plugins/github-actions/src/components/Cards/Cards.tsx +++ b/plugins/github-actions/src/components/Cards/Cards.tsx @@ -17,6 +17,7 @@ import React, { useEffect } from 'react'; import { useWorkflowRuns } from '../useWorkflowRuns'; import { WorkflowRun, WorkflowRunsTable } from '../WorkflowRunsTable'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { readGitHubIntegrationConfigs } from '@backstage/integration'; import { WorkflowRunStatus } from '../WorkflowRunStatus'; import { @@ -81,11 +82,11 @@ const WidgetContent = ({ }; export const LatestWorkflowRunCard = ({ - entity, branch = 'master', // Display the card full height suitable for variant, }: Props) => { + const { entity } = useEntity(); const config = useApi(configApiRef); const errorApi = useApi(errorApiRef); // TODO: Get github hostname from metadata annotation @@ -121,17 +122,21 @@ export const LatestWorkflowRunCard = ({ }; type Props = { - entity: Entity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; branch: string; variant?: string; }; export const LatestWorkflowsForBranchCard = ({ - entity, branch = 'master', variant, -}: Props) => ( - - - -); +}: Props) => { + const { entity } = useEntity(); + + return ( + + + + ); +}; diff --git a/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.test.tsx b/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.test.tsx index 0deb4ae39d..18cc59439b 100644 --- a/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.test.tsx +++ b/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.test.tsx @@ -22,6 +22,7 @@ import { ConfigApi, ConfigReader, } from '@backstage/core'; +import { EntityProvider } from '@backstage/plugin-catalog-react'; import { lightTheme } from '@backstage/theme'; import { ThemeProvider } from '@material-ui/core'; import { render } from '@testing-library/react'; @@ -84,7 +85,9 @@ describe('', () => { configApi, )} > - + + + , diff --git a/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx b/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx index aa30423c3e..4425bc8364 100644 --- a/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx +++ b/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx @@ -23,6 +23,7 @@ import { useApi, } from '@backstage/core'; import { readGitHubIntegrationConfigs } from '@backstage/integration'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { Button, Link } from '@material-ui/core'; import React, { useEffect } from 'react'; import { generatePath, Link as RouterLink } from 'react-router-dom'; @@ -33,7 +34,8 @@ import { WorkflowRunStatus } from '../WorkflowRunStatus'; const firstLine = (message: string): string => message.split('\n')[0]; export type Props = { - entity: Entity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; branch?: string; dense?: boolean; limit?: number; @@ -41,12 +43,12 @@ export type Props = { }; export const RecentWorkflowRunsCard = ({ - entity, branch, dense = false, limit = 5, variant, }: Props) => { + const { entity } = useEntity(); const config = useApi(configApiRef); const errorApi = useApi(errorApiRef); // TODO: Get github hostname from metadata annotation diff --git a/plugins/github-actions/src/components/Router.tsx b/plugins/github-actions/src/components/Router.tsx index e7430d40ed..834fcae43c 100644 --- a/plugins/github-actions/src/components/Router.tsx +++ b/plugins/github-actions/src/components/Router.tsx @@ -15,6 +15,7 @@ */ import React from 'react'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { Routes, Route } from 'react-router'; import { rootRouteRef, buildRouteRef } from '../plugin'; import { WorkflowRunDetails } from './WorkflowRunDetails'; @@ -22,13 +23,23 @@ import { WorkflowRunsTable } from './WorkflowRunsTable'; import { GITHUB_ACTIONS_ANNOTATION } from './useProjectName'; import { MissingAnnotationEmptyState } from '@backstage/core'; -export const isPluginApplicableToEntity = (entity: Entity) => +export const isGithubActionsAvailable = (entity: Entity) => Boolean(entity.metadata.annotations?.[GITHUB_ACTIONS_ANNOTATION]); -export const Router = ({ entity }: { entity: Entity }) => - !isPluginApplicableToEntity(entity) ? ( - - ) : ( +type Props = { + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}; + +export const Router = (_props: Props) => { + const { entity } = useEntity(); + + if (!isGithubActionsAvailable(entity)) { + return ( + + ); + } + return ( ) ); +}; diff --git a/plugins/github-actions/src/index.ts b/plugins/github-actions/src/index.ts index 24fe6fc90d..b4179a9d92 100644 --- a/plugins/github-actions/src/index.ts +++ b/plugins/github-actions/src/index.ts @@ -14,8 +14,19 @@ * limitations under the License. */ -export { plugin } from './plugin'; +export { + githubActionsPlugin, + githubActionsPlugin as plugin, + EntityGithubActionsContent, + EntityLatestGithubActionRunCard, + EntityLatestGithubActionsForBranchCard, + EntityRecentGithubActionsRunsCard, +} from './plugin'; export * from './api'; -export { Router, isPluginApplicableToEntity } from './components/Router'; +export { + Router, + isGithubActionsAvailable, + isGithubActionsAvailable as isPluginApplicableToEntity, +} from './components/Router'; export * from './components/Cards'; export { GITHUB_ACTIONS_ANNOTATION } from './components/useProjectName'; diff --git a/plugins/github-actions/src/plugin.test.ts b/plugins/github-actions/src/plugin.test.ts index fa1075cbc8..53bc96c77a 100644 --- a/plugins/github-actions/src/plugin.test.ts +++ b/plugins/github-actions/src/plugin.test.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import { plugin } from './plugin'; +import { githubActionsPlugin } from './plugin'; describe('github-actions', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(githubActionsPlugin).toBeDefined(); }); }); diff --git a/plugins/github-actions/src/plugin.ts b/plugins/github-actions/src/plugin.ts index 39ffc468b1..d296b4ad30 100644 --- a/plugins/github-actions/src/plugin.ts +++ b/plugins/github-actions/src/plugin.ts @@ -20,6 +20,8 @@ import { createRouteRef, createApiFactory, githubAuthApiRef, + createRoutableExtension, + createComponentExtension, } from '@backstage/core'; import { githubActionsApiRef, GithubActionsClient } from './api'; @@ -34,7 +36,7 @@ export const buildRouteRef = createRouteRef({ title: 'GitHub Actions Workflow Run', }); -export const plugin = createPlugin({ +export const githubActionsPlugin = createPlugin({ id: 'github-actions', apis: [ createApiFactory({ @@ -44,4 +46,41 @@ export const plugin = createPlugin({ new GithubActionsClient({ configApi, githubAuthApi }), }), ], + routes: { + entityContent: rootRouteRef, + }, }); + +export const EntityGithubActionsContent = githubActionsPlugin.provide( + createRoutableExtension({ + component: () => import('./components/Router').then(m => m.Router), + mountPoint: rootRouteRef, + }), +); + +export const EntityLatestGithubActionRunCard = githubActionsPlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./components/Cards').then(m => m.LatestWorkflowRunCard), + }, + }), +); + +export const EntityLatestGithubActionsForBranchCard = githubActionsPlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./components/Cards').then(m => m.LatestWorkflowsForBranchCard), + }, + }), +); + +export const EntityRecentGithubActionsRunsCard = githubActionsPlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./components/Cards').then(m => m.RecentWorkflowRunsCard), + }, + }), +); diff --git a/plugins/jenkins/dev/index.tsx b/plugins/jenkins/dev/index.tsx index 812a5585d4..8beb6eabc1 100644 --- a/plugins/jenkins/dev/index.tsx +++ b/plugins/jenkins/dev/index.tsx @@ -15,6 +15,6 @@ */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { jenkinsPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(jenkinsPlugin).render(); diff --git a/plugins/jenkins/src/components/Router.tsx b/plugins/jenkins/src/components/Router.tsx index 91bcde2a8b..85f06bb0e1 100644 --- a/plugins/jenkins/src/components/Router.tsx +++ b/plugins/jenkins/src/components/Router.tsx @@ -15,6 +15,7 @@ */ import React from 'react'; import { Route, Routes } from 'react-router'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { buildRouteRef, rootRouteRef } from '../plugin'; import { DetailedViewPage } from './BuildWithStepsPage/'; import { JENKINS_ANNOTATION } from '../constants'; @@ -22,13 +23,22 @@ import { Entity } from '@backstage/catalog-model'; import { MissingAnnotationEmptyState } from '@backstage/core'; import { CITable } from './BuildsPage/lib/CITable'; -export const isPluginApplicableToEntity = (entity: Entity) => +export const isJenkinsAvailable = (entity: Entity) => Boolean(entity.metadata.annotations?.[JENKINS_ANNOTATION]); -export const Router = ({ entity }: { entity: Entity }) => { - return !isPluginApplicableToEntity(entity) ? ( - - ) : ( +type Props = { + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}; + +export const Router = (_props: Props) => { + const { entity } = useEntity(); + + if (!isJenkinsAvailable(entity)) { + return ; + } + + return ( } /> } /> diff --git a/plugins/jenkins/src/index.ts b/plugins/jenkins/src/index.ts index 30fa0c70a5..e434e959e8 100644 --- a/plugins/jenkins/src/index.ts +++ b/plugins/jenkins/src/index.ts @@ -14,8 +14,17 @@ * limitations under the License. */ -export { plugin } from './plugin'; +export { + jenkinsPlugin, + jenkinsPlugin as plugin, + EntityJenkinsContent, + EntityLatestJenkinsRunCard, +} from './plugin'; export { LatestRunCard } from './components/Cards'; -export { Router, isPluginApplicableToEntity } from './components/Router'; +export { + Router, + isJenkinsAvailable, + isJenkinsAvailable as isPluginApplicableToEntity, +} from './components/Router'; export { JENKINS_ANNOTATION } from './constants'; export * from './api'; diff --git a/plugins/jenkins/src/plugin.test.ts b/plugins/jenkins/src/plugin.test.ts index 8c3f057ecb..0498b80b90 100644 --- a/plugins/jenkins/src/plugin.test.ts +++ b/plugins/jenkins/src/plugin.test.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import { plugin } from './plugin'; +import { jenkinsPlugin } from './plugin'; describe('jenkins', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(jenkinsPlugin).toBeDefined(); }); }); diff --git a/plugins/jenkins/src/plugin.ts b/plugins/jenkins/src/plugin.ts index 23c54743a3..85001d9bbc 100644 --- a/plugins/jenkins/src/plugin.ts +++ b/plugins/jenkins/src/plugin.ts @@ -19,6 +19,8 @@ import { createRouteRef, createApiFactory, discoveryApiRef, + createRoutableExtension, + createComponentExtension, } from '@backstage/core'; import { jenkinsApiRef, JenkinsApi } from './api'; @@ -32,7 +34,7 @@ export const buildRouteRef = createRouteRef({ title: 'Jenkins run', }); -export const plugin = createPlugin({ +export const jenkinsPlugin = createPlugin({ id: 'jenkins', apis: [ createApiFactory({ @@ -41,4 +43,21 @@ export const plugin = createPlugin({ factory: ({ discoveryApi }) => new JenkinsApi({ discoveryApi }), }), ], + routes: { + entityContent: rootRouteRef, + }, }); + +export const EntityJenkinsContent = jenkinsPlugin.provide( + createRoutableExtension({ + component: () => import('./components/Router').then(m => m.Router), + mountPoint: rootRouteRef, + }), +); +export const EntityLatestJenkinsRunCard = jenkinsPlugin.provide( + createComponentExtension({ + component: { + lazy: () => import('./components/Cards').then(m => m.LatestRunCard), + }, + }), +); diff --git a/plugins/lighthouse/src/Router.tsx b/plugins/lighthouse/src/Router.tsx index 0634a0aaa0..643d30cdc2 100644 --- a/plugins/lighthouse/src/Router.tsx +++ b/plugins/lighthouse/src/Router.tsx @@ -16,6 +16,7 @@ import React from 'react'; import { Route, Routes } from 'react-router-dom'; +import { useEntity } from '@backstage/plugin-catalog-react'; import AuditList from './components/AuditList'; import AuditView, { AuditViewContent } from './components/AuditView'; import CreateAudit, { CreateAuditContent } from './components/CreateAudit'; @@ -35,15 +36,27 @@ export const Router = () => ( ); -export const EmbeddedRouter = ({ entity }: { entity: Entity }) => - !isLighthouseAvailable(entity) ? ( - - ) : ( +type Props = { + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}; + +export const EmbeddedRouter = (_props: Props) => { + const { entity } = useEntity(); + + if (!isLighthouseAvailable(entity)) { + return ( + + ); + } + + return ( } /> } /> } /> ); +}; diff --git a/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx b/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx index 506a98d012..2f0f1e2e16 100644 --- a/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx +++ b/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx @@ -21,7 +21,7 @@ import { RELATION_PARENT_OF, } from '@backstage/catalog-model'; import { Avatar, InfoCard } from '@backstage/core'; -import { entityRouteParams } from '@backstage/plugin-catalog-react'; +import { useEntity, entityRouteParams } from '@backstage/plugin-catalog-react'; import { Box, Grid, Link, Tooltip, Typography } from '@material-ui/core'; import AccountTreeIcon from '@material-ui/icons/AccountTree'; import EmailIcon from '@material-ui/icons/Email'; @@ -33,26 +33,29 @@ import { generatePath, Link as RouterLink } from 'react-router-dom'; const GroupLink = ({ groupName, index = 0, - entity, }: { groupName: string; index?: number; - entity: Entity; -}) => ( - <> - {index >= 1 ? ', ' : ''} - - [{groupName}] - - -); + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}) => { + const { entity } = useEntity(); + return ( + <> + {index >= 1 ? ', ' : ''} + + [{groupName}] + + + ); +}; const CardTitle = ({ title }: { title: string }) => ( @@ -61,12 +64,13 @@ const CardTitle = ({ title }: { title: string }) => ( ); export const GroupProfileCard = ({ - entity: group, variant, }: { - entity: GroupEntity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: GroupEntity; variant: string; }) => { + const group = useEntity().entity as GroupEntity; const { metadata: { name, description }, spec: { profile }, diff --git a/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.test.tsx b/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.test.tsx index 6815fd0e55..28c55b376b 100644 --- a/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.test.tsx +++ b/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.test.tsx @@ -16,7 +16,11 @@ import { Entity, GroupEntity } from '@backstage/catalog-model'; import { ApiProvider, ApiRegistry } from '@backstage/core'; -import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog-react'; +import { + CatalogApi, + catalogApiRef, + EntityProvider, +} from '@backstage/plugin-catalog-react'; import { renderWithEffects, wrapInTestApp } from '@backstage/test-utils'; import React from 'react'; import { MembersListCard } from './MembersListCard'; @@ -78,7 +82,10 @@ describe('MemberTab Test', () => { const rendered = await renderWithEffects( wrapInTestApp( - + + + + , , ), ); diff --git a/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx b/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx index f644e7db7e..7f8ede2eb7 100644 --- a/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx +++ b/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx @@ -21,6 +21,7 @@ import { } from '@backstage/catalog-model'; import { Avatar, InfoCard, Progress, useApi } from '@backstage/core'; import { + useEntity, catalogApiRef, entityRouteParams, } from '@backstage/plugin-catalog-react'; @@ -105,11 +106,11 @@ const MemberComponent = ({ ); }; -export const MembersListCard = ({ - entity: groupEntity, -}: { - entity: GroupEntity; +export const MembersListCard = (_props: { + /** @deprecated The entity is now grabbed from context instead */ + entity?: GroupEntity; }) => { + const groupEntity = useEntity().entity as GroupEntity; const { metadata: { name: groupName }, spec: { profile }, diff --git a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx index 60f06bd53a..80e80bddcc 100644 --- a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx +++ b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx @@ -16,7 +16,11 @@ import { Entity } from '@backstage/catalog-model'; import { InfoCard, Progress, useApi } from '@backstage/core'; -import { catalogApiRef, isOwnerOf } from '@backstage/plugin-catalog-react'; +import { + catalogApiRef, + isOwnerOf, + useEntity, +} from '@backstage/plugin-catalog-react'; import { pageTheme } from '@backstage/theme'; import { Box, @@ -113,12 +117,13 @@ const EntityCountTile = ({ }; export const OwnershipCard = ({ - entity, variant, }: { - entity: Entity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; variant: string; }) => { + const { entity } = useEntity(); const catalogApi = useApi(catalogApiRef); const { loading, diff --git a/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.test.tsx b/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.test.tsx index 77708e6f27..7c7d26c228 100644 --- a/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.test.tsx +++ b/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.test.tsx @@ -15,6 +15,7 @@ */ import { UserEntity } from '@backstage/catalog-model'; +import { EntityProvider } from '@backstage/plugin-catalog-react'; import { renderWithEffects, wrapInTestApp } from '@backstage/test-utils'; import React from 'react'; import { UserProfileCard } from './UserProfileCard'; @@ -48,7 +49,11 @@ describe('UserSummary Test', () => { it('Display Profile Card', async () => { const rendered = await renderWithEffects( - wrapInTestApp(), + wrapInTestApp( + + + , + ), ); expect(rendered.getByText('calum-leavy@example.com')).toBeInTheDocument(); diff --git a/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx b/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx index bdd0cd15f6..1cf7c968b3 100644 --- a/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx +++ b/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx @@ -19,7 +19,7 @@ import { UserEntity, } from '@backstage/catalog-model'; import { Avatar, InfoCard } from '@backstage/core'; -import { entityRouteParams } from '@backstage/plugin-catalog-react'; +import { useEntity, entityRouteParams } from '@backstage/plugin-catalog-react'; import { Box, Grid, Link, Tooltip, Typography } from '@material-ui/core'; import EmailIcon from '@material-ui/icons/Email'; import GroupIcon from '@material-ui/icons/Group'; @@ -60,12 +60,13 @@ const CardTitle = ({ title }: { title?: string }) => ) : null; export const UserProfileCard = ({ - entity: user, variant, }: { - entity: UserEntity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: UserEntity; variant: string; }) => { + const user = useEntity().entity as UserEntity; const { metadata: { name: metaName }, spec: { profile }, diff --git a/plugins/scaffolder-backend/migrations/20210120143715_init.js b/plugins/scaffolder-backend/migrations/20210120143715_init.js new file mode 100644 index 0000000000..256ec7d423 --- /dev/null +++ b/plugins/scaffolder-backend/migrations/20210120143715_init.js @@ -0,0 +1,85 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('tasks', table => { + table.comment('The table of scaffolder tasks'); + table.uuid('id').primary().notNullable().comment('The ID of the task'); + table + .text('spec') + .notNullable() + .comment('A JSON encoded task specification'); + table + .text('status') + .notNullable() + .comment('The current status of the task'); + table + .dateTime('created_at') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The timestamp when this task was created'); + table + .dateTime('last_heartbeat_at') + .nullable() + .comment('The last timestamp when a heartbeat was received'); + }); + + await knex.schema.createTable('task_events', table => { + table.comment('The event stream a given task'); + table + .bigIncrements('id') + .primary() + .notNullable() + .comment('The ID of the event'); + table + .uuid('task_id') + .references('id') + .inTable('tasks') + .notNullable() + .onDelete('CASCADE') + .comment('The task that generated the event'); + table + .text('body') + .notNullable() + .comment('The JSON encoded body of the event'); + table.text('event_type').notNullable().comment('The type of event'); + table + .timestamp('created_at') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The timestamp when this event was generated'); + + table.index(['task_id'], 'task_events_task_id_idx'); + }); +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('task_events', table => { + table.dropIndex([], 'ctask_events_task_id_idx'); + }); + } + await knex.schema.dropTable('task_events'); + await knex.schema.dropTable('tasks'); +}; diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 95d15c45a8..ecce161c8e 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -53,6 +53,7 @@ "helmet": "^4.0.0", "isomorphic-git": "^1.8.0", "jsonschema": "^1.2.6", + "knex": "^0.21.6", "morgan": "^1.10.0", "uuid": "^8.2.0", "winston": "^3.2.1", @@ -71,6 +72,7 @@ }, "files": [ "dist", + "migrations", "config.d.ts" ], "configSchema": "config.d.ts" diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts new file mode 100644 index 0000000000..f151780ba9 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -0,0 +1,110 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TemplateActionRegistry } from '../tasks/TemplateConverter'; +import { FilePreparer, PreparerBuilder } from './prepare'; +import Docker from 'dockerode'; +import { TemplaterBuilder, TemplaterValues } from './templater'; +import { PublisherBuilder } from './publish'; + +type Options = { + dockerClient: Docker; + preparers: PreparerBuilder; + templaters: TemplaterBuilder; + publishers: PublisherBuilder; +}; + +export function registerLegacyActions( + registry: TemplateActionRegistry, + options: Options, +) { + const { dockerClient, preparers, templaters, publishers } = options; + + registry.register({ + id: 'legacy:prepare', + async handler(ctx) { + const { protocol, url } = ctx.parameters; + const preparer = + protocol === 'file' ? new FilePreparer() : preparers.get(url as string); + + ctx.logger.info('Prepare the skeleton'); + await preparer.prepare({ + url: url as string, + logger: ctx.logger, + workspacePath: ctx.workspacePath, + }); + }, + }); + + registry.register({ + id: 'legacy:template', + async handler(ctx) { + const { logger } = ctx; + + const templater = templaters.get(ctx.parameters.templater as string); + + logger.info('Run the templater'); + await templater.run({ + workspacePath: ctx.workspacePath, + dockerClient, + logStream: ctx.logStream, + values: ctx.parameters.values as TemplaterValues, + }); + }, + }); + + registry.register({ + id: 'legacy:publish', + async handler(ctx) { + const { values } = ctx.parameters; + if ( + typeof values !== 'object' || + values === null || + Array.isArray(values) + ) { + throw new Error( + `Invalid values passed to publish, got ${typeof values}`, + ); + } + const storePath = values.storePath as unknown; + if (typeof storePath !== 'string') { + throw new Error( + `Invalid store path passed to publish, got ${typeof storePath}`, + ); + } + const owner = values.owner as unknown; + if (typeof owner !== 'string') { + throw new Error(`Invalid owner passed to publish, got ${typeof owner}`); + } + + const publisher = publishers.get(storePath); + ctx.logger.info('Will now store the template'); + const { remoteUrl, catalogInfoUrl } = await publisher.publish({ + values: { + ...values, + owner, + storePath, + }, + workspacePath: ctx.workspacePath, + logger: ctx.logger, + }); + ctx.output('remoteUrl', remoteUrl); + if (catalogInfoUrl) { + ctx.output('catalogInfoUrl', catalogInfoUrl); + } + }, + }); +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts new file mode 100644 index 0000000000..5717f3c8c1 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -0,0 +1,272 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject } from '@backstage/config'; +import { + ConflictError, + NotFoundError, + resolvePackagePath, +} from '@backstage/backend-common'; +import Knex from 'knex'; +import { v4 as uuid } from 'uuid'; +import { + DbTaskEventRow, + DbTaskRow, + Status, + TaskEventType, + TaskSpec, + TaskStore, + TaskStoreEmitOptions, + TaskStoreGetEventsOptions, +} from './types'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-scaffolder-backend', + 'migrations', +); + +export type RawDbTaskRow = { + id: string; + spec: string; + status: Status; + last_heartbeat_at?: string; + created_at: string; +}; + +export type RawDbTaskEventRow = { + id: number; + task_id: string; + body: string; + event_type: TaskEventType; + created_at: string; +}; + +export class DatabaseTaskStore implements TaskStore { + static async create(knex: Knex): Promise { + await knex.migrate.latest({ + directory: migrationsDir, + }); + return new DatabaseTaskStore(knex); + } + + constructor(private readonly db: Knex) {} + + async get(taskId: string): Promise { + const [result] = await this.db('tasks') + .where({ id: taskId }) + .select(); + if (!result) { + throw new NotFoundError(`No task with id '${taskId}' found`); + } + try { + const spec = JSON.parse(result.spec); + return { + id: result.id, + spec, + status: result.status, + lastHeartbeatAt: result.last_heartbeat_at, + createdAt: result.created_at, + }; + } catch (error) { + throw new Error(`Failed to parse spec of task '${taskId}', ${error}`); + } + } + + async createTask(spec: TaskSpec): Promise<{ taskId: string }> { + const taskId = uuid(); + await this.db('tasks').insert({ + id: taskId, + spec: JSON.stringify(spec), + status: 'open', + }); + return { taskId }; + } + + async claimTask(): Promise { + return this.db.transaction(async tx => { + const [task] = await tx('tasks') + .where({ + status: 'open', + }) + .limit(1) + .select(); + + if (!task) { + return undefined; + } + + const updateCount = await tx('tasks') + .where({ id: task.id, status: 'open' }) + .update({ + status: 'processing', + last_heartbeat_at: this.db.fn.now(), + }); + + if (updateCount < 1) { + return undefined; + } + + try { + const spec = JSON.parse(task.spec); + return { + id: task.id, + spec, + status: 'processing', + lastHeartbeatAt: task.last_heartbeat_at, + createdAt: task.created_at, + }; + } catch (error) { + throw new Error(`Failed to parse spec of task '${task.id}', ${error}`); + } + }); + } + + async heartbeatTask(taskId: string): Promise { + const updateCount = await this.db('tasks') + .where({ id: taskId, status: 'processing' }) + .update({ + last_heartbeat_at: this.db.fn.now(), + }); + if (updateCount === 0) { + throw new ConflictError(`No running task with taskId ${taskId} found`); + } + } + + async listStaleTasks({ + timeoutS, + }: { + timeoutS: number; + }): Promise<{ + tasks: { taskId: string }[]; + }> { + const rawRows = await this.db('tasks') + .where('status', 'processing') + .andWhere( + 'last_heartbeat_at', + '<=', + this.db.client.config.client === 'sqlite3' + ? this.db.raw(`datetime('now', ?)`, [`-${timeoutS} seconds`]) + : this.db.raw(`dateadd('second', ?, ?)`, [ + `-${timeoutS}`, + this.db.fn.now(), + ]), + ); + const tasks = rawRows.map(row => ({ + taskId: row.id, + })); + return { tasks }; + } + + async completeTask({ + taskId, + status, + eventBody, + }: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise { + let oldStatus: string; + if (status === 'failed' || status === 'completed') { + oldStatus = 'processing'; + } else { + throw new Error( + `Invalid status update of run '${taskId}' to status '${status}'`, + ); + } + await this.db.transaction(async tx => { + const [task] = await tx('tasks') + .where({ + id: taskId, + }) + .limit(1) + .select(); + + if (!task) { + throw new Error(`No task with taskId ${taskId} found`); + } + if (task.status !== oldStatus) { + throw new ConflictError( + `Refusing to update status of run '${taskId}' to status '${status}' ` + + `as it is currently '${task.status}', expected '${oldStatus}'`, + ); + } + const updateCount = await tx('tasks') + .where({ + id: taskId, + status: oldStatus, + }) + .update({ + status, + }); + if (updateCount !== 1) { + throw new ConflictError( + `Failed to update status to '${status}' for taskId ${taskId}`, + ); + } + + await tx('task_events').insert({ + task_id: taskId, + event_type: 'completion', + body: JSON.stringify(eventBody), + }); + }); + } + + async emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise { + const serliazedBody = JSON.stringify(body); + await this.db('task_events').insert({ + task_id: taskId, + event_type: 'log', + body: serliazedBody, + }); + } + + async listEvents({ + taskId, + after, + }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }> { + const rawEvents = await this.db('task_events') + .where({ + task_id: taskId, + }) + .andWhere(builder => { + if (typeof after === 'number') { + builder.where('id', '>', after).orWhere('event_type', 'completion'); + } + }) + .orderBy('id') + .select(); + + const events = rawEvents.map(event => { + try { + const body = JSON.parse(event.body) as JsonObject; + return { + id: event.id, + taskId, + body, + type: event.event_type, + createdAt: event.created_at, + }; + } catch (error) { + throw new Error( + `Failed to parse event body from event taskId=${taskId} id=${event.id}, ${error}`, + ); + } + }); + return { events }; + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts new file mode 100644 index 0000000000..c45fa46335 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -0,0 +1,183 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + getVoidLogger, + SingleConnectionDatabaseManager, +} from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { DatabaseTaskStore } from './DatabaseTaskStore'; +import { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; +import { TaskSpec, DbTaskEventRow } from './types'; + +async function createStore(): Promise { + const manager = SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'sqlite3', + connection: ':memory:', + }, + }, + }), + ).forPlugin('scaffolder'); + return await DatabaseTaskStore.create(await manager.getClient()); +} + +describe('StorageTaskBroker', () => { + let storage: DatabaseTaskStore; + + beforeAll(async () => { + storage = await createStore(); + }); + + const logger = getVoidLogger(); + it('should claim a dispatched work item', async () => { + const broker = new StorageTaskBroker(storage, logger); + await broker.dispatch({ steps: [] }); + await expect(broker.claim()).resolves.toEqual(expect.any(TaskAgent)); + }); + + it('should wait for a dispatched work item', async () => { + const broker = new StorageTaskBroker(storage, logger); + const promise = broker.claim(); + + await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting'); + + await broker.dispatch({ steps: [] }); + await expect(promise).resolves.toEqual(expect.any(TaskAgent)); + }); + + it('should dispatch multiple items and claim them in order', async () => { + const broker = new StorageTaskBroker(storage, logger); + await broker.dispatch({ steps: [{ id: 'a' }] } as TaskSpec); + await broker.dispatch({ steps: [{ id: 'b' }] } as TaskSpec); + await broker.dispatch({ steps: [{ id: 'c' }] } as TaskSpec); + + const taskA = await broker.claim(); + const taskB = await broker.claim(); + const taskC = await broker.claim(); + await expect(taskA).toEqual(expect.any(TaskAgent)); + await expect(taskB).toEqual(expect.any(TaskAgent)); + await expect(taskC).toEqual(expect.any(TaskAgent)); + await expect(taskA.spec.steps[0].id).toBe('a'); + await expect(taskB.spec.steps[0].id).toBe('b'); + await expect(taskC.spec.steps[0].id).toBe('c'); + }); + + it('should complete a task', async () => { + const broker = new StorageTaskBroker(storage, logger); + const dispatchResult = await broker.dispatch({ steps: [] }); + const task = await broker.claim(); + await task.complete('completed'); + const taskRow = await storage.get(dispatchResult.taskId); + expect(taskRow.status).toBe('completed'); + }, 10000); + + it('should fail a task', async () => { + const broker = new StorageTaskBroker(storage, logger); + const dispatchResult = await broker.dispatch({ steps: [] }); + const task = await broker.claim(); + await task.complete('failed'); + const taskRow = await storage.get(dispatchResult.taskId); + expect(taskRow.status).toBe('failed'); + }); + + it('multiple brokers should be able to observe a single task', async () => { + const broker1 = new StorageTaskBroker(storage, logger); + const broker2 = new StorageTaskBroker(storage, logger); + + const { taskId } = await broker1.dispatch({ steps: [] }); + + const logPromise = new Promise(resolve => { + const observedEvents = new Array(); + + broker2.observe({ taskId, after: undefined }, (_err, { events }) => { + observedEvents.push(...events); + if (events.some(e => e.type === 'completion')) { + resolve(observedEvents); + } + }); + }); + const task = await broker1.claim(); + await task.emitLog('log 1'); + await task.emitLog('log 2'); + await task.emitLog('log 3'); + await task.complete('completed'); + + const logs = await logPromise; + expect(logs.map(l => l.body.message, logger)).toEqual([ + 'log 1', + 'log 2', + 'log 3', + 'Run completed with status: completed', + ]); + + const afterLogs = await new Promise(resolve => { + broker2.observe({ taskId, after: logs[1].id }, (_err, { events }) => + resolve(events.map(e => e.body.message as string)), + ); + }); + expect(afterLogs).toEqual([ + 'log 3', + 'Run completed with status: completed', + ]); + }); + + it('should heartbeat', async () => { + const broker = new StorageTaskBroker(storage, logger); + const { taskId } = await broker.dispatch({ steps: [] }); + const task = await broker.claim(); + + const initialTask = await storage.get(taskId); + + for (;;) { + const maybeTask = await storage.get(taskId); + if (maybeTask.lastHeartbeatAt !== initialTask.lastHeartbeatAt) { + break; + } + await new Promise(resolve => setTimeout(resolve, 50)); + } + await task.complete('completed'); + expect.assertions(0); + }); + + it('should be update the status to failed if heartbeat fails', async () => { + const broker = new StorageTaskBroker(storage, logger); + const { taskId } = await broker.dispatch({ steps: [] }); + const task = await broker.claim(); + + jest + .spyOn((task as any).storage, 'heartbeatTask') + .mockRejectedValue(new Error('nah m8')); + + const intervalId = setInterval(() => { + broker.vacuumTasks({ timeoutS: 2 }).catch(fail); + }, 500); + + for (;;) { + const maybeTask = await storage.get(taskId); + if (maybeTask.status === 'failed') { + break; + } + await new Promise(resolve => setTimeout(resolve, 50)); + } + + clearInterval(intervalId); + + expect(task.done).toBe(true); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts new file mode 100644 index 0000000000..48c0d5e5ba --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -0,0 +1,205 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Logger } from 'winston'; +import { + CompletedTaskState, + Task, + TaskSpec, + TaskStore, + TaskBroker, + DispatchResult, + DbTaskEventRow, +} from './types'; + +export class TaskAgent implements Task { + private isDone = false; + + private heartbeatTimeoutId?: ReturnType; + + static create(state: TaskState, storage: TaskStore, logger: Logger) { + const agent = new TaskAgent(state, storage, logger); + agent.startTimeout(); + return agent; + } + + // Runs heartbeat internally + private constructor( + private readonly state: TaskState, + private readonly storage: TaskStore, + private readonly logger: Logger, + ) {} + + get spec() { + return this.state.spec; + } + + async getWorkspaceName() { + return this.state.taskId; + } + + get done() { + return this.isDone; + } + + async emitLog(message: string): Promise { + await this.storage.emitLogEvent({ + taskId: this.state.taskId, + body: { message }, + }); + } + + async complete(result: CompletedTaskState): Promise { + await this.storage.completeTask({ + taskId: this.state.taskId, + status: result === 'failed' ? 'failed' : 'completed', + eventBody: { message: `Run completed with status: ${result}` }, + }); + this.isDone = true; + if (this.heartbeatTimeoutId) { + clearTimeout(this.heartbeatTimeoutId); + } + } + + private startTimeout() { + this.heartbeatTimeoutId = setTimeout(async () => { + try { + await this.storage.heartbeatTask(this.state.taskId); + this.startTimeout(); + } catch (error) { + this.isDone = true; + + this.logger.error( + `Heartbeat for task ${this.state.taskId} failed`, + error, + ); + } + }, 1000); + } +} + +interface TaskState { + spec: TaskSpec; + taskId: string; +} + +function defer() { + let resolve = () => {}; + const promise = new Promise(_resolve => { + resolve = _resolve; + }); + return { promise, resolve }; +} + +export class StorageTaskBroker implements TaskBroker { + constructor( + private readonly storage: TaskStore, + private readonly logger: Logger, + ) {} + private deferredDispatch = defer(); + + async claim(): Promise { + for (;;) { + const pendingTask = await this.storage.claimTask(); + if (pendingTask) { + return TaskAgent.create( + { + taskId: pendingTask.id, + spec: pendingTask.spec, + }, + this.storage, + this.logger, + ); + } + + await this.waitForDispatch(); + } + } + + async dispatch(spec: TaskSpec): Promise { + const taskRow = await this.storage.createTask(spec); + this.signalDispatch(); + return { + taskId: taskRow.taskId, + }; + } + + observe( + options: { + taskId: string; + after: number | undefined; + }, + callback: ( + error: Error | undefined, + result: { events: DbTaskEventRow[] }, + ) => void, + ): () => void { + const { taskId } = options; + + let cancelled = false; + const unsubscribe = () => { + cancelled = true; + }; + + (async () => { + let after = options.after; + while (!cancelled) { + const result = await this.storage.listEvents({ taskId, after: after }); + const { events } = result; + if (events.length) { + after = events[events.length - 1].id; + try { + callback(undefined, result); + } catch (error) { + callback(error, { events: [] }); + } + } + + await new Promise(resolve => setTimeout(resolve, 1000)); + } + })(); + + return unsubscribe; + } + + async vacuumTasks(timeoutS: { timeoutS: number }): Promise { + const { tasks } = await this.storage.listStaleTasks(timeoutS); + await Promise.all( + tasks.map(async task => { + try { + await this.storage.completeTask({ + taskId: task.taskId, + status: 'failed', + eventBody: { + message: + 'The task was cancelled because the task worker lost connection to the task broker', + }, + }); + } catch (error) { + this.logger.warn(`Failed to cancel task '${task.taskId}', ${error}`); + } + }), + ); + } + + private waitForDispatch() { + return this.deferredDispatch.promise; + } + + private signalDispatch() { + this.deferredDispatch.resolve(); + this.deferredDispatch = defer(); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts new file mode 100644 index 0000000000..c62495ca4c --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -0,0 +1,110 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PassThrough } from 'stream'; +import { Logger } from 'winston'; +import * as winston from 'winston'; +import { JsonValue } from '@backstage/config'; +import { TaskBroker, Task } from './types'; +import fs from 'fs-extra'; +import path from 'path'; +import { TemplateActionRegistry } from './TemplateConverter'; + +type Options = { + logger: Logger; + taskBroker: TaskBroker; + workingDirectory: string; + actionRegistry: TemplateActionRegistry; +}; + +export class TaskWorker { + constructor(private readonly options: Options) {} + + start() { + (async () => { + for (;;) { + const task = await this.options.taskBroker.claim(); + await this.runOneTask(task); + } + })(); + } + + async runOneTask(task: Task) { + try { + const { actionRegistry, logger } = this.options; + + const workspacePath = path.join( + this.options.workingDirectory, + await task.getWorkspaceName(), + ); + await fs.ensureDir(workspacePath); + + const taskLogger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: {}, + }); + + const stream = new PassThrough(); + stream.on('data', data => { + const message = data.toString().trim(); + if (message?.length > 1) task.emitLog(message); + }); + + taskLogger.add(new winston.transports.Stream({ stream })); + + // Give us some time to curl observe + task.emitLog('Task claimed, waiting ...'); + await new Promise(resolve => setTimeout(resolve, 5000)); + task.emitLog(`Starting up work with ${task.spec.steps.length} steps`); + + const outputs: { [name: string]: JsonValue } = {}; + + for (const step of task.spec.steps) { + task.emitLog(`Beginning step ${step.name}`); + + const action = actionRegistry.get(step.action); + if (!action) { + throw new Error(`Action '${step.action}' does not exist`); + } + + // TODO: substitute any placeholders with output from previous steps + const parameters = step.parameters!; + + await action.handler({ + logger, + logStream: stream, + parameters, + workspacePath, + output(name: string, value: JsonValue) { + outputs[name] = value; + }, + }); + + task.emitLog(`Finished step ${step.name}`); + } + + await task.complete('completed'); + } catch (error) { + task.emitLog(String(error.stack)); + await task.complete('failed'); + } + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts new file mode 100644 index 0000000000..69788238cc --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -0,0 +1,117 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolve as resolvePath } from 'path'; +import { JsonValue } from '@backstage/config'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { Logger } from 'winston'; +import type { Writable } from 'stream'; + +import { TaskSpec } from './types'; +import { ConflictError, NotFoundError } from '@backstage/backend-common'; +import { + getTemplaterKey, + joinGitUrlPath, + parseLocationAnnotation, + TemplaterValues, +} from '../stages'; + +export function templateEntityToSpec( + template: TemplateEntityV1alpha1, + values: TemplaterValues, +): TaskSpec { + const steps: TaskSpec['steps'] = []; + + const { protocol, location } = parseLocationAnnotation(template); + + let url: string; + if (protocol === 'file') { + const path = resolvePath(location, template.spec.path || '.'); + + url = `file://${path}`; + } else { + url = joinGitUrlPath(location, template.spec.path); + } + const templater = getTemplaterKey(template); + + steps.push({ + id: 'prepare', + name: 'Prepare', + action: 'legacy:prepare', + parameters: { + protocol, + url, + }, + }); + + steps.push({ + id: 'template', + name: 'Template', + action: 'legacy:template', + parameters: { + templater, + values, + }, + }); + + steps.push({ + id: 'publish', + name: 'Publishing', + action: 'legacy:publish', + parameters: { + values, + }, + }); + + return { steps }; +} + +type ActionContext = { + logger: Logger; + logStream: Writable; + + workspacePath: string; + parameters: { [name: string]: JsonValue }; + output(name: string, value: JsonValue): void; +}; + +type TemplateAction = { + id: string; + handler: (ctx: ActionContext) => Promise; +}; + +export class TemplateActionRegistry { + private readonly actions = new Map(); + + register(action: TemplateAction) { + if (this.actions.has(action.id)) { + throw new ConflictError( + `Template action with ID '${action.id}' has already been registered`, + ); + } + this.actions.set(action.id, action); + } + + get(actionId: string): TemplateAction { + const action = this.actions.get(actionId); + if (!action) { + throw new NotFoundError( + `Template action with ID '${actionId}' is not registered.`, + ); + } + return action; + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts new file mode 100644 index 0000000000..c85135426e --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { DatabaseTaskStore } from './DatabaseTaskStore'; +export { StorageTaskBroker } from './StorageTaskBroker'; +export { TaskWorker } from './TaskWorker'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts new file mode 100644 index 0000000000..0c2592109b --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonValue, JsonObject } from '@backstage/config'; + +export type Status = + | 'open' + | 'processing' + | 'failed' + | 'cancelled' + | 'completed'; + +export type CompletedTaskState = 'failed' | 'completed'; + +export type DbTaskRow = { + id: string; + spec: TaskSpec; + status: Status; + createdAt: string; + lastHeartbeatAt?: string; +}; + +export type TaskEventType = 'completion' | 'log'; +export type DbTaskEventRow = { + id: number; + taskId: string; + body: JsonObject; + type: TaskEventType; + createdAt: string; +}; + +export type TaskSpec = { + steps: Array<{ + id: string; + name: string; + action: string; + parameters?: { [name: string]: JsonValue }; + }>; +}; + +export type DispatchResult = { + taskId: string; +}; + +export interface Task { + spec: TaskSpec; + done: boolean; + emitLog(message: string): Promise; + complete(result: CompletedTaskState): Promise; + getWorkspaceName(): Promise; +} + +export interface TaskBroker { + claim(): Promise; + dispatch(spec: TaskSpec): Promise; + vacuumTasks(timeoutS: { timeoutS: number }): Promise; + observe( + options: { + taskId: string; + after: number | undefined; + }, + callback: ( + error: Error | undefined, + result: { events: DbTaskEventRow[] }, + ) => void, + ): () => void; +} + +export type TaskStoreEmitOptions = { + taskId: string; + body: JsonObject; +}; + +export type TaskStoreGetEventsOptions = { + taskId: string; + after?: number | undefined; +}; +export interface TaskStore { + createTask(task: TaskSpec): Promise<{ taskId: string }>; + claimTask(): Promise; + completeTask(options: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise; + heartbeatTask(taskId: string): Promise; + listStaleTasks(options: { + timeoutS: number; + }): Promise<{ + tasks: { taskId: string }[]; + }>; + + emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; + listEvents({ + taskId, + after, + }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; +} diff --git a/plugins/scaffolder-backend/src/service/helpers.ts b/plugins/scaffolder-backend/src/service/helpers.ts new file mode 100644 index 0000000000..dd3d43c6d1 --- /dev/null +++ b/plugins/scaffolder-backend/src/service/helpers.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import fs from 'fs-extra'; +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; + +export async function getWorkingDirectory( + config: Config, + logger: Logger, +): Promise { + if (!config.has('backend.workingDirectory')) { + return os.tmpdir(); + } + + const workingDirectory = config.getString('backend.workingDirectory'); + try { + // Check if working directory exists and is writable + await fs.access(workingDirectory, fs.constants.F_OK | fs.constants.W_OK); + logger.info(`using working directory: ${workingDirectory}`); + } catch (err) { + logger.error( + `working directory ${workingDirectory} ${ + err.code === 'ENOENT' ? 'does not exist' : 'is not writable' + }`, + ); + throw err; + } + return workingDirectory; +} diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 31f62febd2..8a0fa6cba6 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -16,6 +16,7 @@ const mockAccess = jest.fn(); jest.doMock('fs-extra', () => ({ + access: mockAccess, promises: { access: mockAccess, }, @@ -27,7 +28,11 @@ jest.doMock('fs-extra', () => ({ remove: jest.fn(), })); -import { getVoidLogger } from '@backstage/backend-common'; +import { + SingleConnectionDatabaseManager, + PluginDatabaseManager, + getVoidLogger, +} from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; @@ -41,6 +46,19 @@ const generateEntityClient: any = (template: any) => ({ findTemplate: () => Promise.resolve(template), }); +function createDatabase(): PluginDatabaseManager { + return SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'sqlite3', + connection: ':memory:', + }, + }, + }), + ).forPlugin('scaffolder'); +} + describe('createRouter - working directory', () => { const mockPrepare = jest.fn(); const mockPreparers = new Preparers(); @@ -78,7 +96,6 @@ describe('createRouter - working directory', () => { }; const mockedEntityClient = generateEntityClient(template); - it('should throw an error when working directory does not exist or is not writable', async () => { mockAccess.mockImplementation(() => { throw new Error('access error'); @@ -93,6 +110,7 @@ describe('createRouter - working directory', () => { config: new ConfigReader(workDirConfig('/path')), dockerClient: new Docker(), entityClient: mockedEntityClient, + database: createDatabase(), }), ).rejects.toThrow('access error'); }); @@ -106,6 +124,7 @@ describe('createRouter - working directory', () => { config: new ConfigReader(workDirConfig('/path')), dockerClient: new Docker(), entityClient: mockedEntityClient, + database: createDatabase(), }); const app = express().use(router); @@ -134,6 +153,7 @@ describe('createRouter - working directory', () => { config: new ConfigReader({}), dockerClient: new Docker(), entityClient: mockedEntityClient, + database: createDatabase(), }); const app = express().use(router); @@ -203,6 +223,7 @@ describe('createRouter', () => { config: new ConfigReader({}), dockerClient: new Docker(), entityClient: generateEntityClient(template), + database: createDatabase(), }); app = express().use(router); }); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 9098e5d05e..5821edfe8c 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -33,6 +33,18 @@ import { import { CatalogEntityClient } from '../lib/catalog'; import { validate, ValidatorResult } from 'jsonschema'; import parseGitUrl from 'git-url-parse'; +import { + DatabaseTaskStore, + StorageTaskBroker, + TaskWorker, +} from '../scaffolder/tasks'; +import { + TemplateActionRegistry, + templateEntityToSpec, +} from '../scaffolder/tasks/TemplateConverter'; +import { registerLegacyActions } from '../scaffolder/stages/legacy'; +import { getWorkingDirectory } from './helpers'; +import { PluginDatabaseManager } from '@backstage/backend-common'; export interface RouterOptions { preparers: PreparerBuilder; @@ -43,6 +55,7 @@ export interface RouterOptions { config: Config; dockerClient: Docker; entityClient: CatalogEntityClient; + database: PluginDatabaseManager; } export async function createRouter( @@ -59,11 +72,34 @@ export async function createRouter( config, dockerClient, entityClient, + database, } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); + const workingDirectory = await getWorkingDirectory(config, logger); const jobProcessor = await JobProcessor.fromConfig({ config, logger }); + const databaseTaskStore = await DatabaseTaskStore.create( + await database.getClient(), + ); + const taskBroker = new StorageTaskBroker(databaseTaskStore, logger); + const actionRegistry = new TemplateActionRegistry(); + const worker = new TaskWorker({ + logger, + taskBroker, + actionRegistry, + workingDirectory, + }); + + registerLegacyActions(actionRegistry, { + dockerClient, + preparers, + publishers, + templaters, + }); + + worker.start(); + router .get('/v1/job/:jobId', ({ params }, res) => { const job = jobProcessor.get(params.jobId); @@ -184,6 +220,75 @@ export async function createRouter( res.status(201).json({ id: job.id }); }); + // NOTE: The v2 API is unstable + router + .post('/v2/tasks', async (req, res) => { + const templateName: string = req.body.templateName; + const values: TemplaterValues = { + ...req.body.values, + destination: { + git: parseGitUrl(req.body.values.storePath), + }, + }; + const template = await entityClient.findTemplate(templateName); + + const validationResult: ValidatorResult = validate( + values, + template.spec.schema, + ); + + if (!validationResult.valid) { + res.status(400).json({ errors: validationResult.errors }); + return; + } + const taskSpec = templateEntityToSpec(template, values); + const result = await taskBroker.dispatch(taskSpec); + + res.status(201).json({ id: result.taskId }); + }) + .get('/v2/tasks/:taskId/eventstream', async (req, res) => { + const { taskId } = req.params; + const after = Number(req.query.after) || undefined; + logger.debug(`Event stream observing taskId '${taskId}' opened`); + + // Mandatory headers and http status to keep connection open + res.writeHead(200, { + Connection: 'keep-alive', + 'Cache-Control': 'no-cache', + 'Content-Type': 'text/event-stream', + }); + + // After client opens connection send all events as string + const unsubscribe = taskBroker.observe( + { taskId, after }, + (error, { events }) => { + if (error) { + logger.error( + `Received error from event stream when observing taskId '${taskId}', ${error}`, + ); + } + + for (const event of events) { + res.write( + `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ); + if (event.type === 'completion') { + unsubscribe(); + // Closing the event stream here would cause the frontend + // to automatically reconnect because it lost connection. + } + } + res.flush(); + }, + ); + // When client closes connection we update the clients list + // avoiding the disconnected one + req.on('close', () => { + unsubscribe(); + logger.debug(`Event stream observing taskId '${taskId}' closed`); + }); + }); + const app = express(); app.set('logger', logger); app.use('/', router); diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index d91e2f08a2..0fd23a4f7d 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -53,6 +53,7 @@ export async function startStandaloneServer( const mockUrlReader: jest.Mocked = { read: jest.fn(), readTree: jest.fn(), + search: jest.fn(), }; logger.debug('Creating application...'); diff --git a/yarn.lock b/yarn.lock index 45f8a8b4fb..05282e5285 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2645,7 +2645,31 @@ remark-gfm "^1.0.0" zen-observable "^0.8.15" -"@backstage/plugin-catalog@^0.2.0", "@backstage/plugin-catalog@^0.2.1": +"@backstage/plugin-catalog@^0.2.0": + version "0.3.0" + dependencies: + "@backstage/catalog-client" "^0.3.5" + "@backstage/catalog-model" "^0.7.1" + "@backstage/core" "^0.6.0" + "@backstage/plugin-catalog-react" "^0.0.2" + "@backstage/plugin-scaffolder" "^0.4.2" + "@backstage/theme" "^0.2.3" + "@material-ui/core" "^4.11.0" + "@material-ui/icons" "^4.9.1" + "@material-ui/lab" "4.0.0-alpha.45" + "@types/react" "^16.9" + classnames "^2.2.6" + git-url-parse "^11.4.4" + moment "^2.26.0" + react "^16.13.1" + react-dom "^16.13.1" + react-helmet "6.1.0" + react-router "6.0.0-beta.0" + react-router-dom "6.0.0-beta.0" + react-use "^15.3.3" + swr "^0.3.0" + +"@backstage/plugin-catalog@^0.2.1": version "0.3.0" dependencies: "@backstage/catalog-client" "^0.3.5" @@ -13795,12 +13819,12 @@ find-up@^5.0.0: locate-path "^6.0.0" path-exists "^4.0.0" -find-versions@^3.2.0: - version "3.2.0" - resolved "https://registry.npmjs.org/find-versions/-/find-versions-3.2.0.tgz#10297f98030a786829681690545ef659ed1d254e" - integrity sha512-P8WRou2S+oe222TOCHitLy8zj+SIsVJh52VP4lvXkaFVnOFFdoWv1H1Jjvel1aI6NCFOAaeAVm8qrI0odiLcww== +find-versions@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/find-versions/-/find-versions-4.0.0.tgz#3c57e573bf97769b8cb8df16934b627915da4965" + integrity sha512-wgpWy002tA+wgmO27buH/9KzyEOQnKsG/R0yrcjPT9BOFm0zRBVQbZ95nRGXWMywS8YR5knRbpohio0bcJABxQ== dependencies: - semver-regex "^2.0.0" + semver-regex "^3.1.2" find-yarn-workspace-root2@1.2.16: version "1.2.16" @@ -15406,17 +15430,17 @@ humanize-ms@^1.2.1: ms "^2.0.0" husky@^4.2.3: - version "4.3.6" - resolved "https://registry.npmjs.org/husky/-/husky-4.3.6.tgz#ebd9dd8b9324aa851f1587318db4cccb7665a13c" - integrity sha512-o6UjVI8xtlWRL5395iWq9LKDyp/9TE7XMOTvIpEVzW638UcGxTmV5cfel6fsk/jbZSTlvfGVJf2svFtybcIZag== + version "4.3.8" + resolved "https://registry.npmjs.org/husky/-/husky-4.3.8.tgz#31144060be963fd6850e5cc8f019a1dfe194296d" + integrity sha512-LCqqsB0PzJQ/AlCgfrfzRe3e3+NvmefAdKQhRYpxS4u6clblBoDdzzvHi8fmxKRzvMxPY/1WZWzomPZww0Anow== dependencies: chalk "^4.0.0" ci-info "^2.0.0" compare-versions "^3.6.0" cosmiconfig "^7.0.0" - find-versions "^3.2.0" + find-versions "^4.0.0" opencollective-postinstall "^2.0.2" - pkg-dir "^4.2.0" + pkg-dir "^5.0.0" please-upgrade-node "^3.2.0" slash "^3.0.0" which-pm-runs "^1.0.0" @@ -20738,6 +20762,13 @@ pkg-dir@^4.1.0, pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +pkg-dir@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-5.0.0.tgz#a02d6aebe6ba133a928f74aec20bafdfe6b8e760" + integrity sha512-NPE8TDbzl/3YQYY7CSS228s3g2ollTFnc+Qi3tqmqJp9Vg2ovUpixcJEo2HJScN2Ez+kEaal6y70c0ehqJBJeA== + dependencies: + find-up "^5.0.0" + pkg-up@3.1.0, pkg-up@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/pkg-up/-/pkg-up-3.1.0.tgz#100ec235cc150e4fd42519412596a28512a0def5" @@ -23147,10 +23178,10 @@ rollup-plugin-postcss@^3.1.1: safe-identifier "^0.4.1" style-inject "^0.3.0" -rollup-plugin-typescript2@^0.27.3: - version "0.27.3" - resolved "https://registry.npmjs.org/rollup-plugin-typescript2/-/rollup-plugin-typescript2-0.27.3.tgz#cd9455ac026d325b20c5728d2cc54a08a771b68b" - integrity sha512-gmYPIFmALj9D3Ga1ZbTZAKTXq1JKlTQBtj299DXhqYz9cL3g/AQfUvbb2UhH+Nf++cCq941W2Mv7UcrcgLzJJg== +rollup-plugin-typescript2@^0.29.0: + version "0.29.0" + resolved "https://registry.npmjs.org/rollup-plugin-typescript2/-/rollup-plugin-typescript2-0.29.0.tgz#b7ad83f5241dbc5bdf1e98d9c3fca005ffe39e1a" + integrity sha512-YytahBSZCIjn/elFugEGQR5qTsVhxhUwGZIsA9TmrSsC88qroGo65O5HZP/TTArH2dm0vUmYWhKchhwi2wL9bw== dependencies: "@rollup/pluginutils" "^3.1.0" find-cache-dir "^3.3.1" @@ -23359,10 +23390,10 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -semver-regex@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/semver-regex/-/semver-regex-2.0.0.tgz#a93c2c5844539a770233379107b38c7b4ac9d338" - integrity sha512-mUdIBBvdn0PLOeP3TEkMH7HHeUP3GjsXCwKarjv/kGmUFOYg1VqEemKhoQpWMu6X2I8kHeuVdGibLGkVK+/5Qw== +semver-regex@^3.1.2: + version "3.1.2" + resolved "https://registry.npmjs.org/semver-regex/-/semver-regex-3.1.2.tgz#34b4c0d361eef262e07199dbef316d0f2ab11807" + integrity sha512-bXWyL6EAKOJa81XG1OZ/Yyuq+oT0b2YLlxx7c+mrdYPaPbnj6WgVULXhinMIeZGufuUBu/eVRqXEhiv4imfwxA== "semver@2 || 3 || 4 || 5", "semver@2.x || 3.x || 4 || 5", semver@^5.3.0, semver@^5.4.1, semver@^5.5.0, semver@^5.5.1, semver@^5.6.0, semver@^5.7.0, semver@^5.7.1: version "5.7.1"