From 1136ac88b604946f522a6e08333f5e271580384b Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 15:03:51 +0200 Subject: [PATCH 01/33] First part of GitLabDiscoveryProcessor (no target parsing yet) Signed-off-by: Roy Jacobs --- .../GitLabDiscoveryProcessor.test.ts | 252 ++++++++++++++++++ .../processors/GitLabDiscoveryProcessor.ts | 129 +++++++++ .../src/ingestion/processors/gitlab/client.ts | 94 +++++++ .../src/ingestion/processors/gitlab/index.ts | 18 ++ .../src/ingestion/processors/gitlab/types.ts | 23 ++ .../src/ingestion/processors/index.ts | 1 + .../src/next/NextCatalogBuilder.ts | 2 + .../src/service/CatalogBuilder.ts | 2 + 8 files changed, 521 insertions(+) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..5eb5c51869 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts @@ -0,0 +1,252 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { + GitLabDiscoveryProcessor, +} from './GitLabDiscoveryProcessor'; +import { setupServer } from 'msw/node'; +import { rest } from 'msw'; +import { GitLabProject } from "./gitlab"; + +const server = setupServer(); + +function setupFakeGitLab( + callback: (request: { + page: number; + }) => { data: GitLabProject[]; nextPage?: number }, +) { + server.use( + rest.get('https://gitlab.fake/api/v4/projects', (req, res, ctx) => { + if (req.headers.get('private-token') !== 'test-token') { + return res(ctx.status(401), ctx.json({})); + } + const page = req.url.searchParams.get('page'); + const response = callback({ + page: parseInt(page!, 10), + }); + + // Filter the fake results based on the `last_activity_after` parameter + const last_activity_after = req.url.searchParams.get( + 'last_activity_after', + ); + const filteredData = response.data.filter( + v => + !last_activity_after || + Date.parse(v.last_activity_at) >= Date.parse(last_activity_after), + ); + + return res( + ctx.set('x-next-page', response.nextPage?.toString() ?? ''), + ctx.json(filteredData), + ); + }), + ); +} + +function getConfig(): any { + return { + backend: { + cache: { store: 'memory' }, + }, + integrations: { + gitlab: [ + { + host: 'gitlab.fake', + apiBaseUrl: 'https://gitlab.fake/api/v4', + token: 'test-token', + }, + ], + }, + }; +} + +function getProcessor(config?: any): GitLabDiscoveryProcessor { + return GitLabDiscoveryProcessor.fromConfig( + new ConfigReader(config || getConfig()), + { + logger: getVoidLogger(), + }, + ); +} + +describe('GitlabDiscoveryProcessor', () => { + beforeAll(() => { + server.listen(); + jest.useFakeTimers('modern'); + jest.setSystemTime(new Date('2001-01-01T12:34:56Z')); + }); + afterEach(() => server.resetHandlers()); + afterAll(() => { + server.close(); + jest.useRealTimers(); + }); + + const location: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/north-star', + }; + + describe('handles repositories', () => { + it('pages through all repositories', async () => { + const processor = getProcessor(); + setupFakeGitLab(request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + nextPage: 2, + }; + case 2: + return { + data: [ + { + id: 2, + archived: false, + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/2', + }, + { + id: 3, + archived: true, // ARCHIVED + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/3', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(location, false, e => { + result.push(e); + }); + expect(result).toEqual([ + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/1/-/blob/main/catalog-info.yaml', + }, + optional: true, + }, + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/2/-/blob/master/catalog-info.yaml', + }, + optional: true, + }, + ]); + }); + + it('uses the previous scan timestamp to filter', async () => { + const processor = getProcessor(); + setupFakeGitLab(request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2000-01-01T00:00:00Z', + web_url: 'https://gitlab.fake/1', + }, + { + id: 2, + archived: false, + default_branch: 'main', + last_activity_at: '2002-01-01T00:00:00Z', + web_url: 'https://gitlab.fake/2', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + + // First scan should find all repos, since no last activity was cached + await processor.readLocation(location, false, e => { + result.push(e); + }); + expect(result).toHaveLength(2); + + // Second scan should have used the mocked Date to set the last scanned time to 2001 + // This should result in only the second repo being scanned, since that has a timestamp of 2002 + const result2: any[] = []; + await processor.readLocation(location, false, e => { + result2.push(e); + }); + expect(result2).toHaveLength(1); + }); + }); + + describe('handles failure', () => { + it('invalid token', async () => { + // Setup an empty fake gitlab, since we don't care about actual results + setupFakeGitLab(_ => { + return { + data: [], + }; + }); + + const config = getConfig(); + config.integrations.gitlab[0].token = 'invalid'; + await expect( + getProcessor(config).readLocation(location, false, _ => {}), + ).rejects.toThrow(/Unauthorized/); + }); + + it('missing integration', async () => { + const config = getConfig(); + delete config.integrations; + await expect( + getProcessor(config).readLocation(location, false, _ => {}), + ).rejects.toThrow(/no GitLab integration/); + }); + + it('location type', async () => { + const incorrectLocation: LocationSpec = { + type: 'something-that-is-not-gitlab-discovery', + target: 'https://gitlab.fake/north-star', + }; + + await expect( + getProcessor().readLocation(incorrectLocation, false, _ => {}), + ).resolves.toBeFalsy(); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts new file mode 100644 index 0000000000..b3cce6e711 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + ScmIntegrations, +} from '@backstage/integration'; +import { Logger } from 'winston'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { GitLabClient, GitLabProject, paginated } from "./gitlab"; +import { CacheClient, CacheManager, PluginCacheManager } from "@backstage/backend-common"; + +/** + * Extracts repositories out of a GitLab org. + */ +export class GitLabDiscoveryProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly logger: Logger; + private readonly cache: CacheClient; + + static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + const pluginCache = CacheManager.fromConfig(config).forPlugin( + 'gitlab-discovery', + ); + + return new GitLabDiscoveryProcessor({ + ...options, + integrations, + pluginCache, + }); + } + + private constructor(options: { + integrations: ScmIntegrations; + pluginCache: PluginCacheManager; + logger: Logger; + }) { + this.integrations = options.integrations; + this.cache = options.pluginCache.getClient(); + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'gitlab-discovery') { + return false; + } + + const integration = this.integrations.gitlab.byUrl(location.target); + if (!integration) { + throw new Error( + `There is no GitLab integration that matches ${location.target}. Please add a configuration entry for it under integrations.gitlab`, + ); + } + + const client = new GitLabClient({ + config: integration.config, + logger: this.logger, + }); + const startTimestamp = Date.now(); + this.logger.info(`Reading GitLab projects from ${location.target}`); + + const projects = paginated(options => client.listProjects(options), { + last_activity_after: await this.updateLastActivity(), + page: 1, + }); + + const result: Result = { + scanned: 0, + matches: [], + }; + for await (const project of projects) { + result.scanned++; + if (!project.archived) { + result.matches.push(project); + } + } + + for (const project of result.matches) { + emit( + results.location( + { + type: 'url', + // The format expected by the GitLabUrlReader: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + target: `${project.web_url}/-/blob/${project.default_branch}/catalog-info.yaml`, + }, + true, + ), + ); + } + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${result.scanned} GitLab repositories in ${duration} seconds`, + ); + + return true; + } + + async updateLastActivity(): Promise { + const lastActivity = await this.cache.get('last-activity'); + await this.cache.set('last-activity', new Date().toISOString()); + return lastActivity as string | undefined; + } +} + +type Result = { + scanned: number; + matches: GitLabProject[]; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts new file mode 100644 index 0000000000..fb7763dd1c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'cross-fetch'; +import { + getGitLabRequestOptions, + GitLabIntegrationConfig, +} from '@backstage/integration'; +import { Logger } from 'winston'; + +export class GitLabClient { + private readonly config: GitLabIntegrationConfig; + private readonly logger: Logger; + + constructor(options: { config: GitLabIntegrationConfig; logger: Logger }) { + this.config = options.config; + this.logger = options.logger; + } + + async listProjects(options?: ListOptions): Promise> { + return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); + } + + private async pagedRequest( + endpoint: string, + options?: ListOptions, + ): Promise> { + const request = new URL(endpoint); + for (const key in options) { + if (options[key]) { + request.searchParams.append(key, options[key]!.toString()); + } + } + + this.logger.debug(`Fetching: ${request.toString()}`); + const response = await fetch( + request.toString(), + getGitLabRequestOptions(this.config), + ); + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + return response.json().then(items => { + const nextPage = response.headers.get('x-next-page'); + + return { + items, + nextPage: nextPage ? Number(nextPage) : null, + } as PagedResponse; + }); + } +} + +export type ListOptions = { + [key: string]: string | number | undefined; + per_page?: number | undefined; + page?: number | undefined; +}; + +export type PagedResponse = { + items: T[]; + nextPage?: number; +}; + +export async function* paginated( + request: (options: ListOptions) => Promise>, + options: ListOptions, +) { + let res; + do { + res = await request(options); + options.page = res.nextPage; + for (const item of res.items) { + yield item; + } + } while (res.nextPage); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts new file mode 100644 index 0000000000..a6c6419d21 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitLabClient, paginated } from './client'; +export type { GitLabProject } from "./types"; diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts new file mode 100644 index 0000000000..a66411ddc8 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type GitLabProject = { + id: number; + default_branch: string; + archived: boolean; + last_activity_at: string; + web_url: string; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 6a44ea0a8f..fe4d374829 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -26,6 +26,7 @@ export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; +export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; export { LocationEntityProcessor } from './LocationEntityProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; export type { PlaceholderResolver } from './PlaceholderProcessor'; diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index fb12077508..88fcc27055 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -50,6 +50,7 @@ import { FileReaderProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, + GitLabDiscoveryProcessor, PlaceholderProcessor, PlaceholderResolver, UrlReaderProcessor, @@ -372,6 +373,7 @@ export class NextCatalogBuilder { BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), + GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new AnnotateLocationEntityProcessor({ integrations }), diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 76b4af7450..70c1890fbf 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -46,6 +46,7 @@ import { FileReaderProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, + GitLabDiscoveryProcessor, HigherOrderOperation, HigherOrderOperations, LocationEntityProcessor, @@ -316,6 +317,7 @@ export class CatalogBuilder { BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), + GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new LocationEntityProcessor({ integrations }), From a487f61564c3ccdc6938f8990fb1313b367dd827 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:24:07 +0200 Subject: [PATCH 02/33] Parse URLs and allow filtering based on group/subgroup Signed-off-by: Roy Jacobs --- .../GitLabDiscoveryProcessor.test.ts | 160 +++++++++++++++--- .../processors/GitLabDiscoveryProcessor.ts | 71 ++++++-- .../src/ingestion/processors/gitlab/client.ts | 15 +- 3 files changed, 210 insertions(+), 36 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts index 5eb5c51869..a0e44d7648 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts @@ -17,28 +17,47 @@ import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; -import { - GitLabDiscoveryProcessor, -} from './GitLabDiscoveryProcessor'; +import { GitLabDiscoveryProcessor, parseUrl } from './GitLabDiscoveryProcessor'; import { setupServer } from 'msw/node'; import { rest } from 'msw'; -import { GitLabProject } from "./gitlab"; +import { GitLabProject } from './gitlab'; const server = setupServer(); -function setupFakeGitLab( - callback: (request: { - page: number; - }) => { data: GitLabProject[]; nextPage?: number }, +const PROJECTS_URL = 'https://gitlab.fake/api/v4/projects'; +const GROUP_PROJECTS_URL = + 'https://gitlab.fake/api/v4/groups/group%2Fsubgroup/projects'; + +const PROJECT_LOCATION: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/blob/*/catalog-info.yaml', +}; +const PROJECT_LOCATION_MASTER_BRANCH: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/blob/master/catalog-info.yaml', +}; +const GROUP_LOCATION: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/group/subgroup/blob/*/catalog-info.yaml', +}; + +function setupFakeServer( + url: string, + callback: (request: { page: number; include_subgroups: boolean }) => { + data: GitLabProject[]; + nextPage?: number; + }, ) { server.use( - rest.get('https://gitlab.fake/api/v4/projects', (req, res, ctx) => { + rest.get(url, (req, res, ctx) => { if (req.headers.get('private-token') !== 'test-token') { return res(ctx.status(401), ctx.json({})); } const page = req.url.searchParams.get('page'); + const include_subgroups = req.url.searchParams.get('include_subgroups'); const response = callback({ page: parseInt(page!, 10), + include_subgroups: include_subgroups === 'true', }); // Filter the fake results based on the `last_activity_after` parameter @@ -97,15 +116,43 @@ describe('GitlabDiscoveryProcessor', () => { jest.useRealTimers(); }); - const location: LocationSpec = { - type: 'gitlab-discovery', - target: 'https://gitlab.fake/north-star', - }; + describe('parseUrl', () => { + it('parses well formed URLs', () => { + expect( + parseUrl('https://gitlab.com/group/subgroup/blob/master/catalog.yaml'), + ).toEqual({ + group: 'group/subgroup', + host: 'gitlab.com', + branch: 'master', + catalogPath: 'catalog.yaml', + }); + expect( + parseUrl('https://gitlab.com/blob/*/subfolder/catalog.yaml'), + ).toEqual({ + group: undefined, + host: 'gitlab.com', + branch: '*', + catalogPath: 'subfolder/catalog.yaml', + }); + }); + + it('throws on incorrectly formed URLs', () => { + expect(() => parseUrl('https://gitlab.com')).toThrow(); + expect(() => parseUrl('https://gitlab.com//')).toThrow(); + expect(() => parseUrl('https://gitlab.com/foo')).toThrow(); + expect(() => parseUrl('https://gitlab.com//foo')).toThrow(); + expect(() => parseUrl('https://gitlab.com/org/teams')).toThrow(); + expect(() => parseUrl('https://gitlab.com/org//teams')).toThrow(); + expect(() => + parseUrl('https://gitlab.com/org//teams/blob/catalog.yaml'), + ).toThrow(); + }); + }); describe('handles repositories', () => { it('pages through all repositories', async () => { const processor = getProcessor(); - setupFakeGitLab(request => { + setupFakeServer(PROJECTS_URL, request => { switch (request.page) { case 1: return { @@ -145,7 +192,7 @@ describe('GitlabDiscoveryProcessor', () => { }); const result: any[] = []; - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result.push(e); }); expect(result).toEqual([ @@ -168,9 +215,78 @@ describe('GitlabDiscoveryProcessor', () => { ]); }); + it('can force a branch name', async () => { + const processor = getProcessor(); + setupFakeServer(PROJECTS_URL, request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(PROJECT_LOCATION_MASTER_BRANCH, false, e => { + result.push(e); + }); + expect(result).toEqual([ + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/1/-/blob/master/catalog-info.yaml', + }, + optional: true, + }, + ]); + }); + + it('can filter based on group', async () => { + const processor = getProcessor(); + setupFakeServer(GROUP_PROJECTS_URL, request => { + if (!request.include_subgroups) { + throw new Error('include_subgroups should be set'); + } + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(GROUP_LOCATION, false, e => { + result.push(e); + }); + // If everything was set up correctly, we should have received the fake repo specified above + expect(result).toHaveLength(1); + }); + it('uses the previous scan timestamp to filter', async () => { const processor = getProcessor(); - setupFakeGitLab(request => { + setupFakeServer(PROJECTS_URL, request => { switch (request.page) { case 1: return { @@ -199,7 +315,7 @@ describe('GitlabDiscoveryProcessor', () => { const result: any[] = []; // First scan should find all repos, since no last activity was cached - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result.push(e); }); expect(result).toHaveLength(2); @@ -207,7 +323,7 @@ describe('GitlabDiscoveryProcessor', () => { // Second scan should have used the mocked Date to set the last scanned time to 2001 // This should result in only the second repo being scanned, since that has a timestamp of 2002 const result2: any[] = []; - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result2.push(e); }); expect(result2).toHaveLength(1); @@ -217,7 +333,7 @@ describe('GitlabDiscoveryProcessor', () => { describe('handles failure', () => { it('invalid token', async () => { // Setup an empty fake gitlab, since we don't care about actual results - setupFakeGitLab(_ => { + setupFakeServer(PROJECTS_URL, _ => { return { data: [], }; @@ -226,7 +342,7 @@ describe('GitlabDiscoveryProcessor', () => { const config = getConfig(); config.integrations.gitlab[0].token = 'invalid'; await expect( - getProcessor(config).readLocation(location, false, _ => {}), + getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}), ).rejects.toThrow(/Unauthorized/); }); @@ -234,14 +350,14 @@ describe('GitlabDiscoveryProcessor', () => { const config = getConfig(); delete config.integrations; await expect( - getProcessor(config).readLocation(location, false, _ => {}), + getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}), ).rejects.toThrow(/no GitLab integration/); }); it('location type', async () => { const incorrectLocation: LocationSpec = { type: 'something-that-is-not-gitlab-discovery', - target: 'https://gitlab.fake/north-star', + target: 'https://gitlab.fake/oh-dear', }; await expect( diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts index b3cce6e711..5603a1aa7f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -16,17 +16,19 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { - ScmIntegrations, -} from '@backstage/integration'; +import { ScmIntegrations } from '@backstage/integration'; import { Logger } from 'winston'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import { GitLabClient, GitLabProject, paginated } from "./gitlab"; -import { CacheClient, CacheManager, PluginCacheManager } from "@backstage/backend-common"; +import { GitLabClient, GitLabProject, paginated } from './gitlab'; +import { + CacheClient, + CacheManager, + PluginCacheManager, +} from '@backstage/backend-common'; /** - * Extracts repositories out of a GitLab org. + * Extracts repositories out of an GitLab instance. */ export class GitLabDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; @@ -35,9 +37,8 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { static fromConfig(config: Config, options: { logger: Logger }) { const integrations = ScmIntegrations.fromConfig(config); - const pluginCache = CacheManager.fromConfig(config).forPlugin( - 'gitlab-discovery', - ); + const pluginCache = + CacheManager.fromConfig(config).forPlugin('gitlab-discovery'); return new GitLabDiscoveryProcessor({ ...options, @@ -65,10 +66,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { return false; } - const integration = this.integrations.gitlab.byUrl(location.target); + const { group, host, branch, catalogPath } = parseUrl(location.target); + + const integration = this.integrations.gitlab.byUrl(`https://${host}`); if (!integration) { throw new Error( - `There is no GitLab integration that matches ${location.target}. Please add a configuration entry for it under integrations.gitlab`, + `There is no GitLab integration that matches ${host}. Please add a configuration entry for it under integrations.gitlab`, ); } @@ -80,6 +83,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { this.logger.info(`Reading GitLab projects from ${location.target}`); const projects = paginated(options => client.listProjects(options), { + group, last_activity_after: await this.updateLastActivity(), page: 1, }); @@ -96,12 +100,19 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { } for (const project of result.matches) { + const project_branch = branch === '*' ? project.default_branch : branch; + emit( results.location( { type: 'url', - // The format expected by the GitLabUrlReader: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath - target: `${project.web_url}/-/blob/${project.default_branch}/catalog-info.yaml`, + // The format expected by the GitLabUrlReader: + // https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + // + // This unfortunately will trigger another API call in `getGitLabFileFetchUrl` to get the project ID. + // The alternative is using the `buildRawUrl` function, which does not support subgroups, so providing a raw + // URL here won't work either. + target: `${project.web_url}/-/blob/${project_branch}/${catalogPath}`, }, true, ), @@ -127,3 +138,37 @@ type Result = { scanned: number; matches: GitLabProject[]; }; + +/* + * Helpers + */ + +export function parseUrl(urlString: string): { + group?: string; + host: string; + branch: string; + catalogPath: string; +} { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // (/group/subgroup)/blob/branch|*/filepath + const blobIndex = path.findIndex(p => p === 'blob'); + if (blobIndex !== -1 && path.length > blobIndex + 2) { + const group = + blobIndex > 0 ? path.slice(0, blobIndex).join('/') : undefined; + + return { + group, + host: url.host, + branch: decodeURIComponent(path[blobIndex + 1]), + catalogPath: decodeURIComponent(path.slice(blobIndex + 2).join('/')), + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +export function escapeRegExp(str: string): RegExp { + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts index fb7763dd1c..8781e071f7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -31,6 +31,18 @@ export class GitLabClient { } async listProjects(options?: ListOptions): Promise> { + if (options?.group) { + return this.pagedRequest( + `${this.config.apiBaseUrl}/groups/${encodeURIComponent( + options?.group, + )}/projects`, + { + ...options, + include_subgroups: true, + }, + ); + } + return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); } @@ -69,7 +81,8 @@ export class GitLabClient { } export type ListOptions = { - [key: string]: string | number | undefined; + [key: string]: string | number | boolean | undefined; + group?: string; per_page?: number | undefined; page?: number | undefined; }; From 29f6c7541a4f10dd3844924c7ed97a586be05359 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:28:36 +0200 Subject: [PATCH 03/33] Add documentation for GitLab discovery Signed-off-by: Roy Jacobs --- docs/integrations/gitlab/discovery.md | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs/integrations/gitlab/discovery.md diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md new file mode 100644 index 0000000000..ade575b74d --- /dev/null +++ b/docs/integrations/gitlab/discovery.md @@ -0,0 +1,36 @@ +--- +id: discovery +title: GitLab Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from repositories in GitLab +--- + +The GitLab integration has a special discovery processor for discovering catalog +entities from GitLab. The processor will crawl the GitLab instance and register +entities matching the configured path. This can be useful as an alternative to +static locations or manually adding things to the catalog. + +To use the discovery processor, you'll need a GitLab integration +[set up](locations.md) with a `token`. Then you can add a location target to the +catalog configuration: + +```yaml +catalog: + locations: + - type: gitlab-discovery + target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml +``` + +Note the `gitlab-discovery` type, as this is not a regular `url` processor. + +The target is composed of three parts: + +- The base URL, `https://gitlab.com` in this case +- The group path, `group/subgroup` in this case. This is optional: If you omit + this path the processor will scan the entire GitLab instance instead. +- The path within each repository to find the catalog YAML file. This will + usually be `/blob/main/catalog-info.yaml`, `/blob/master/catalog-info.yaml` or + a similar variation for catalog files stored in the root directory of each + repository. If you want to use the repository's default branch use the `*` + wildcard, e.g.: `/blob/*/catalog-info.yaml` From 96785dce38ba99b317677decb24585a9e5c22bef Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:31:57 +0200 Subject: [PATCH 04/33] Add changeset for GitLab discovery Signed-off-by: Roy Jacobs --- .changeset/lucky-gifts-help.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lucky-gifts-help.md diff --git a/.changeset/lucky-gifts-help.md b/.changeset/lucky-gifts-help.md new file mode 100644 index 0000000000..1bc100d824 --- /dev/null +++ b/.changeset/lucky-gifts-help.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added GitLabDiscoveryProcessor, which allows catalog discovery from a GitLab instance From 017f2f6c7e2a2bbf37f48776da976b53e9cca1e3 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:44:33 +0200 Subject: [PATCH 05/33] Forgot to run 'Prettier' Signed-off-by: Roy Jacobs --- .../catalog-backend/src/ingestion/processors/gitlab/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts index a6c6419d21..1df3d2cb84 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts @@ -15,4 +15,4 @@ */ export { GitLabClient, paginated } from './client'; -export type { GitLabProject } from "./types"; +export type { GitLabProject } from './types'; From a6ae6115d3b23e6688fcc7081d25a7ce8b8a2ae3 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 17:08:21 +0200 Subject: [PATCH 06/33] Update api-report.md Signed-off-by: Roy Jacobs --- plugins/catalog-backend/api-report.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 7286855d4e..0c7b08d94c 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -771,6 +771,27 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { ): Promise; } +// Warning: (ae-missing-release-tag) "GitLabDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export class GitLabDiscoveryProcessor implements CatalogProcessor { + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger_2; + }, + ): GitLabDiscoveryProcessor; + // (undocumented) + readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise; + // (undocumented) + updateLastActivity(): Promise; +} + // Warning: (ae-missing-release-tag) "HigherOrderOperation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) From 22fc579fe5cfd34c0d87c5b1bcb2eb88fae0f89d Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 19 Aug 2021 17:14:21 +0100 Subject: [PATCH 07/33] Retrieve the externalId from the config My previous change to enable the external id, had missed the part that actually reads the externalId from the config. This change is the final piece, all going well. Signed-off-by: Brian Fletcher --- .changeset/rich-mayflies-do.md | 5 +++++ .../src/cluster-locator/ConfigClusterLocator.ts | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changeset/rich-mayflies-do.md diff --git a/.changeset/rich-mayflies-do.md b/.changeset/rich-mayflies-do.md new file mode 100644 index 0000000000..d68e00c0aa --- /dev/null +++ b/.changeset/rich-mayflies-do.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-kubernetes-backend': patch +--- + +Fixes bug reading ExternalId from k8s backend config diff --git a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts index 2899445b2a..8352db6754 100644 --- a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts +++ b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts @@ -44,7 +44,9 @@ export class ConfigClusterLocator implements KubernetesClustersSupplier { } case 'aws': { const assumeRole = c.getOptionalString('assumeRole'); - return { assumeRole, ...clusterDetails }; + const externalId = c.getOptionalString('externalId'); + + return { assumeRole, externalId, ...clusterDetails }; } case 'serviceAccount': { return clusterDetails; From 80c5620397ff708d8ff0aeb938d81ae9689e91bd Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 19 Aug 2021 18:58:51 +0200 Subject: [PATCH 08/33] Sanitize special characters before building search query for postgres Signed-off-by: Oliver Sand --- .changeset/eleven-snakes-give.md | 5 +++++ .../src/PgSearchEngine/PgSearchEngine.test.ts | 11 +++++++++++ .../src/PgSearchEngine/PgSearchEngine.ts | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 .changeset/eleven-snakes-give.md diff --git a/.changeset/eleven-snakes-give.md b/.changeset/eleven-snakes-give.md new file mode 100644 index 0000000000..7c7a5724e6 --- /dev/null +++ b/.changeset/eleven-snakes-give.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-pg': patch +--- + +Sanitize special characters before building search query for postgres diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 49fb169bf8..7bedee6dea 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -65,6 +65,17 @@ describe('PgSearchEngine', () => { }); }); + it('should sanitize query term', async () => { + const actualTranslatedQuery = searchEngine.translator({ + term: 'H&e|l!l*o W\0o(r)l:d', + pageCursor: '', + }) as PgSearchQuery; + + expect(actualTranslatedQuery).toMatchObject({ + pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', + }); + }); + it('should return translated query with filters', async () => { const actualTranslatedQuery = searchEngine.translator({ term: 'testTerm', diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index 0b052a499d..74a1e09010 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -50,7 +50,7 @@ export class PgSearchEngine implements SearchEngine { return { pgTerm: query.term .split(/\s/) - .map(p => p.trim()) + .map(p => p.replace(/[\0()|&:*!]/g, '').trim()) .filter(p => p !== '') .map(p => `(${JSON.stringify(p)} | ${JSON.stringify(p)}:*)`) .join('&'), From d0276817123ba131c9211de30d229839f13d7775 Mon Sep 17 00:00:00 2001 From: Tim Hansen Date: Thu, 19 Aug 2021 22:21:21 -0600 Subject: [PATCH 09/33] Add warnings to all entity pages, links to APIs Signed-off-by: Tim Hansen --- .changeset/strange-ducks-rhyme.md | 5 +++ .../app/src/components/catalog/EntityPage.tsx | 38 ++++++++++++------- .../app/src/components/catalog/EntityPage.tsx | 17 ++++++++- 3 files changed, 44 insertions(+), 16 deletions(-) create mode 100644 .changeset/strange-ducks-rhyme.md diff --git a/.changeset/strange-ducks-rhyme.md b/.changeset/strange-ducks-rhyme.md new file mode 100644 index 0000000000..47e0ee0d09 --- /dev/null +++ b/.changeset/strange-ducks-rhyme.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': minor +--- + +Updated the default create-app `EntityPage` to include orphan and processing error alerts for all entity types. Previously these were only shown for entities with the `Component` kind. The `EntityPage` in Backstage applications should be updated following [1d517af](https://github.com/backstage/backstage/commit/1d517af7ab1c84dc7d45f6a3a4747d1a44e3ab6c). This also adds the `EntityLinkCard` for API entities. diff --git a/packages/app/src/components/catalog/EntityPage.tsx b/packages/app/src/components/catalog/EntityPage.tsx index d84a7341bc..4a392e0f05 100644 --- a/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/app/src/components/catalog/EntityPage.tsx @@ -211,20 +211,8 @@ const cicdCard = ( ); -const errorsContent = ( - - - - - - - - - -); - -const overviewContent = ( - +const entityWarningContent = ( + <> @@ -240,7 +228,24 @@ const overviewContent = ( + +); +const errorsContent = ( + + + + + + + + + +); + +const overviewContent = ( + + {entityWarningContent} @@ -448,6 +453,7 @@ const apiPage = ( + {entityWarningContent} @@ -478,6 +484,7 @@ const userPage = ( + {entityWarningContent} @@ -493,6 +500,7 @@ const groupPage = ( + {entityWarningContent} @@ -511,6 +519,7 @@ const systemPage = ( + {entityWarningContent} @@ -535,6 +544,7 @@ const domainPage = ( + {entityWarningContent} diff --git a/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx b/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx index a78d1a8f01..4bd5f7fe61 100644 --- a/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx @@ -82,8 +82,8 @@ const cicdContent = ( ); -const overviewContent = ( - +const entityWarningContent = ( + <> @@ -99,7 +99,12 @@ const overviewContent = ( + +); +const overviewContent = ( + + {entityWarningContent} @@ -214,9 +219,13 @@ const apiPage = ( + {entityWarningContent} + + + @@ -242,6 +251,7 @@ const userPage = ( + {entityWarningContent} @@ -257,6 +267,7 @@ const groupPage = ( + {entityWarningContent} @@ -275,6 +286,7 @@ const systemPage = ( + {entityWarningContent} @@ -299,6 +311,7 @@ const domainPage = ( + {entityWarningContent} From aa8e59f6f9da68a6e19f98f64470000f00745683 Mon Sep 17 00:00:00 2001 From: Tim Hansen Date: Thu, 19 Aug 2021 22:31:17 -0600 Subject: [PATCH 10/33] Fix commit reference Signed-off-by: Tim Hansen --- .changeset/strange-ducks-rhyme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/strange-ducks-rhyme.md b/.changeset/strange-ducks-rhyme.md index 47e0ee0d09..1f75593d04 100644 --- a/.changeset/strange-ducks-rhyme.md +++ b/.changeset/strange-ducks-rhyme.md @@ -2,4 +2,4 @@ '@backstage/create-app': minor --- -Updated the default create-app `EntityPage` to include orphan and processing error alerts for all entity types. Previously these were only shown for entities with the `Component` kind. The `EntityPage` in Backstage applications should be updated following [1d517af](https://github.com/backstage/backstage/commit/1d517af7ab1c84dc7d45f6a3a4747d1a44e3ab6c). This also adds the `EntityLinkCard` for API entities. +Updated the default create-app `EntityPage` to include orphan and processing error alerts for all entity types. Previously these were only shown for entities with the `Component` kind. The `EntityPage` in Backstage applications should be updated following [d027681](https://github.com/backstage/backstage/pull/6899/commits/d0276817123ba131c9211de30d229839f13d7775). This also adds the `EntityLinkCard` for API entities. From 32df8a75f382679cca3c0ea6bccc0947259c40d6 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Fri, 20 Aug 2021 09:39:06 +0200 Subject: [PATCH 11/33] Remove unused function Signed-off-by: Roy Jacobs --- .../src/ingestion/processors/GitLabDiscoveryProcessor.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts index 5603a1aa7f..ccd9f17050 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -168,7 +168,3 @@ export function parseUrl(urlString: string): { throw new Error(`Failed to parse ${urlString}`); } - -export function escapeRegExp(str: string): RegExp { - return new RegExp(`^${str.replace(/\*/g, '.*')}$`); -} From 72505a907ff014dd19c9c819ebf7fe5ed094bb1d Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Fri, 20 Aug 2021 09:46:18 +0200 Subject: [PATCH 12/33] Reduce log level to avoid spamming the logs Signed-off-by: Roy Jacobs --- .../src/ingestion/processors/GitLabDiscoveryProcessor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts index ccd9f17050..445f02829e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -80,7 +80,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { logger: this.logger, }); const startTimestamp = Date.now(); - this.logger.info(`Reading GitLab projects from ${location.target}`); + this.logger.debug(`Reading GitLab projects from ${location.target}`); const projects = paginated(options => client.listProjects(options), { group, From a8b41f32a3abbd65f5d98fd378374d3f9cb39d13 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Fri, 20 Aug 2021 11:12:02 +0100 Subject: [PATCH 13/33] fixes existing changes and adds 1 new test for k8 Signed-off-by: Brian Fletcher --- .../ConfigClusterLocator.test.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts index 1642ca5580..dac96e4cef 100644 --- a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts +++ b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts @@ -115,6 +115,14 @@ describe('ConfigClusterLocator', () => { authProvider: 'aws', skipTLSVerify: true, }, + { + assumeRole: 'SomeRole', + name: 'cluster2', + externalId: 'SomeExternalId', + url: 'http://localhost:8081', + authProvider: 'aws', + skipTLSVerify: true, + }, ], }); @@ -127,6 +135,7 @@ describe('ConfigClusterLocator', () => { assumeRole: undefined, name: 'cluster1', serviceAccountToken: 'token', + externalId: undefined, url: 'http://localhost:8080', authProvider: 'aws', skipTLSVerify: false, @@ -134,11 +143,21 @@ describe('ConfigClusterLocator', () => { { assumeRole: 'SomeRole', name: 'cluster2', + externalId: undefined, serviceAccountToken: undefined, url: 'http://localhost:8081', authProvider: 'aws', skipTLSVerify: true, }, + { + assumeRole: 'SomeRole', + name: 'cluster2', + externalId: 'SomeExternalId', + url: 'http://localhost:8081', + serviceAccountToken: undefined, + authProvider: 'aws', + skipTLSVerify: true, + }, ]); }); }); From 0f92df640aa77ddfd081e7d34196d4dcda48d32f Mon Sep 17 00:00:00 2001 From: Tim Hansen Date: Fri, 20 Aug 2021 10:25:36 -0600 Subject: [PATCH 14/33] Prettier Signed-off-by: Tim Hansen --- .../packages/app/src/components/catalog/EntityPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx b/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx index 4bd5f7fe61..d3b4b786a0 100644 --- a/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx @@ -103,7 +103,7 @@ const entityWarningContent = ( ); const overviewContent = ( - + {entityWarningContent} From 0898076bf093a73e83eff44f9563e63d5ce4b926 Mon Sep 17 00:00:00 2001 From: Erik Engervall Date: Mon, 23 Aug 2021 15:02:50 +0200 Subject: [PATCH 15/33] Expose GRM constants, helpers and components via an 'internal' variable Signed-off-by: Erik Engervall --- .../src/components/index.tsx | 24 +++++++++++++++++++ .../src/helpers/getBumpedTag.ts | 12 ++++++++++ .../git-release-manager/src/helpers/index.tsx | 24 +++++++++++++++++++ .../src/helpers/tagParts/getTagParts.ts | 4 ++++ plugins/git-release-manager/src/index.ts | 1 + plugins/git-release-manager/src/plugin.ts | 16 +++++++++---- 6 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 plugins/git-release-manager/src/components/index.tsx create mode 100644 plugins/git-release-manager/src/helpers/index.tsx diff --git a/plugins/git-release-manager/src/components/index.tsx b/plugins/git-release-manager/src/components/index.tsx new file mode 100644 index 0000000000..2ce1edc6d2 --- /dev/null +++ b/plugins/git-release-manager/src/components/index.tsx @@ -0,0 +1,24 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { Differ } from './Differ'; +export { Divider } from './Divider'; +export { InfoCardPlus } from './InfoCardPlus'; +export { LinearProgressWithLabel } from './ResponseStepDialog/LinearProgressWithLabel'; +export { NoLatestRelease } from './NoLatestRelease'; +export { ResponseStepDialog } from './ResponseStepDialog/ResponseStepDialog'; +export { ResponseStepList } from './ResponseStepDialog/ResponseStepList'; +export { ResponseStepListItem } from './ResponseStepDialog/ResponseStepListItem'; diff --git a/plugins/git-release-manager/src/helpers/getBumpedTag.ts b/plugins/git-release-manager/src/helpers/getBumpedTag.ts index c368047635..dbfb00818e 100644 --- a/plugins/git-release-manager/src/helpers/getBumpedTag.ts +++ b/plugins/git-release-manager/src/helpers/getBumpedTag.ts @@ -21,6 +21,14 @@ import { Project } from '../contexts/ProjectContext'; import { SEMVER_PARTS } from '../constants/constants'; import { SemverTagParts } from './tagParts/getSemverTagParts'; +/** + * Calculates the next version for the project + * + * For calendar versioning this means a bump in patch + * + * For semantic versioning this means either a minor or a patch bump + * depending on the value of `bumpLevel` + */ export function getBumpedTag({ project, tag, @@ -74,6 +82,10 @@ function getBumpedSemverTag( }; } +/** + * Calculates the next semantic version, taking into account + * whether or not it's a minor or patch + */ export function getBumpedSemverTagParts( tagParts: SemverTagParts, semverBumpLevel: keyof typeof SEMVER_PARTS, diff --git a/plugins/git-release-manager/src/helpers/index.tsx b/plugins/git-release-manager/src/helpers/index.tsx new file mode 100644 index 0000000000..1516e192a1 --- /dev/null +++ b/plugins/git-release-manager/src/helpers/index.tsx @@ -0,0 +1,24 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { calverRegexp, getCalverTagParts } from './tagParts/getCalverTagParts'; +export { getBumpedSemverTagParts, getBumpedTag } from './getBumpedTag'; +export { getSemverTagParts, semverRegexp } from './tagParts/getSemverTagParts'; +export { getShortCommitHash } from './getShortCommitHash'; +export { getTagParts } from './tagParts/getTagParts'; +export { isCalverTagParts } from './isCalverTagParts'; +export { isProjectValid } from './isProjectValid'; +export { validateTagName } from './tagParts/validateTagName'; diff --git a/plugins/git-release-manager/src/helpers/tagParts/getTagParts.ts b/plugins/git-release-manager/src/helpers/tagParts/getTagParts.ts index acedf2e8d1..750e4e23c5 100644 --- a/plugins/git-release-manager/src/helpers/tagParts/getTagParts.ts +++ b/plugins/git-release-manager/src/helpers/tagParts/getTagParts.ts @@ -18,6 +18,10 @@ import { getCalverTagParts } from './getCalverTagParts'; import { getSemverTagParts } from './getSemverTagParts'; import { Project } from '../../contexts/ProjectContext'; +/** + * Tag parts are the individual parts of a version, e.g. .. + * are the parts of a semantic version + */ export function getTagParts({ project, tag, diff --git a/plugins/git-release-manager/src/index.ts b/plugins/git-release-manager/src/index.ts index 767dc5c6e1..5646a2f61a 100644 --- a/plugins/git-release-manager/src/index.ts +++ b/plugins/git-release-manager/src/index.ts @@ -18,4 +18,5 @@ export { gitReleaseManagerPlugin, GitReleaseManagerPage, gitReleaseManagerApiRef, + internals, } from './plugin'; diff --git a/plugins/git-release-manager/src/plugin.ts b/plugins/git-release-manager/src/plugin.ts index 511fd4b92e..943102f188 100644 --- a/plugins/git-release-manager/src/plugin.ts +++ b/plugins/git-release-manager/src/plugin.ts @@ -14,10 +14,6 @@ * limitations under the License. */ -import { gitReleaseManagerApiRef } from './api/serviceApiRef'; - -import { GitReleaseClient } from './api/GitReleaseClient'; -import { rootRouteRef } from './routes'; import { configApiRef, createPlugin, @@ -26,7 +22,19 @@ import { createRoutableExtension, } from '@backstage/core-plugin-api'; +import { GitReleaseClient } from './api/GitReleaseClient'; +import { gitReleaseManagerApiRef } from './api/serviceApiRef'; +import { rootRouteRef } from './routes'; +import * as constants from './constants/constants'; +import * as helpers from './helpers'; +import * as components from './components'; + export { gitReleaseManagerApiRef }; +export const internals = { + constants, + helpers, + components, +}; export const gitReleaseManagerPlugin = createPlugin({ id: 'git-release-manager', From c9e61d909d20478513bb0340d9b5d812aa912dc5 Mon Sep 17 00:00:00 2001 From: Erik Engervall Date: Mon, 23 Aug 2021 15:06:29 +0200 Subject: [PATCH 16/33] Add changeset Signed-off-by: Erik Engervall --- .changeset/forty-peaches-cry.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/forty-peaches-cry.md diff --git a/.changeset/forty-peaches-cry.md b/.changeset/forty-peaches-cry.md new file mode 100644 index 0000000000..f388c8a0cd --- /dev/null +++ b/.changeset/forty-peaches-cry.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-git-release-manager': patch +--- + +Expose internal constants, helpers and components to make it easier for users to build custom features for GRM. From 70256a29a6cde652cef418ad4c7a520032d03e7d Mon Sep 17 00:00:00 2001 From: Erik Engervall Date: Mon, 23 Aug 2021 15:16:17 +0200 Subject: [PATCH 17/33] Fix ae-forgotten-export error Signed-off-by: Erik Engervall --- plugins/git-release-manager/src/index.ts | 9 ++++++++- plugins/git-release-manager/src/plugin.ts | 11 +++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/plugins/git-release-manager/src/index.ts b/plugins/git-release-manager/src/index.ts index 5646a2f61a..cb903044f4 100644 --- a/plugins/git-release-manager/src/index.ts +++ b/plugins/git-release-manager/src/index.ts @@ -18,5 +18,12 @@ export { gitReleaseManagerPlugin, GitReleaseManagerPage, gitReleaseManagerApiRef, - internals, } from './plugin'; + +import { components, constants, helpers } from './plugin'; + +export const internals = { + components, + constants, + helpers, +}; diff --git a/plugins/git-release-manager/src/plugin.ts b/plugins/git-release-manager/src/plugin.ts index 943102f188..a2d9623118 100644 --- a/plugins/git-release-manager/src/plugin.ts +++ b/plugins/git-release-manager/src/plugin.ts @@ -25,16 +25,11 @@ import { import { GitReleaseClient } from './api/GitReleaseClient'; import { gitReleaseManagerApiRef } from './api/serviceApiRef'; import { rootRouteRef } from './routes'; -import * as constants from './constants/constants'; -import * as helpers from './helpers'; -import * as components from './components'; +export * as constants from './constants/constants'; +export * as helpers from './helpers'; +export * as components from './components'; export { gitReleaseManagerApiRef }; -export const internals = { - constants, - helpers, - components, -}; export const gitReleaseManagerPlugin = createPlugin({ id: 'git-release-manager', From d1be40f0cc70f6abc58dde8f821a1a69c0815133 Mon Sep 17 00:00:00 2001 From: Erik Engervall Date: Mon, 23 Aug 2021 15:24:28 +0200 Subject: [PATCH 18/33] Fix linting errors Signed-off-by: Erik Engervall --- plugins/git-release-manager/src/plugin.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/git-release-manager/src/plugin.ts b/plugins/git-release-manager/src/plugin.ts index a2d9623118..03027a5369 100644 --- a/plugins/git-release-manager/src/plugin.ts +++ b/plugins/git-release-manager/src/plugin.ts @@ -25,11 +25,11 @@ import { import { GitReleaseClient } from './api/GitReleaseClient'; import { gitReleaseManagerApiRef } from './api/serviceApiRef'; import { rootRouteRef } from './routes'; +import * as constants from './constants/constants'; +import * as helpers from './helpers'; +import * as components from './components'; -export * as constants from './constants/constants'; -export * as helpers from './helpers'; -export * as components from './components'; -export { gitReleaseManagerApiRef }; +export { gitReleaseManagerApiRef, constants, helpers, components }; export const gitReleaseManagerPlugin = createPlugin({ id: 'git-release-manager', From 73ccf148c20936336f6b746cfba883103f1a6c0d Mon Sep 17 00:00:00 2001 From: Erik Engervall Date: Mon, 23 Aug 2021 15:38:09 +0200 Subject: [PATCH 19/33] Update api-report.md for GRM Signed-off-by: Erik Engervall --- plugins/git-release-manager/api-report.md | 252 ++++++++++++++++++++++ 1 file changed, 252 insertions(+) diff --git a/plugins/git-release-manager/api-report.md b/plugins/git-release-manager/api-report.md index 018a0f4be8..44fafd7387 100644 --- a/plugins/git-release-manager/api-report.md +++ b/plugins/git-release-manager/api-report.md @@ -7,9 +7,131 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { PropsWithChildren } from 'react'; import { default as React_2 } from 'react'; +import { ReactNode } from 'react'; import { RouteRef } from '@backstage/core-plugin-api'; +// Warning: (ae-missing-release-tag) "calverRegexp" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const calverRegexp: RegExp; + +// Warning: (ae-forgotten-export) The symbol "DifferProps" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "Differ" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const Differ: ({ current, next, icon }: DifferProps) => JSX.Element; + +// Warning: (ae-missing-release-tag) "DISABLE_CACHE" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const DISABLE_CACHE: { + readonly headers: { + readonly 'If-None-Match': ''; + }; +}; + +// Warning: (ae-missing-release-tag) "Divider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const Divider: () => JSX.Element; + +// Warning: (ae-forgotten-export) The symbol "SemverTagParts" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "getBumpedSemverTagParts" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +function getBumpedSemverTagParts( + tagParts: SemverTagParts, + semverBumpLevel: keyof typeof SEMVER_PARTS, +): { + bumpedTagParts: { + prefix: string; + major: number; + minor: number; + patch: number; + }; +}; + +// Warning: (ae-missing-release-tag) "getBumpedTag" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +function getBumpedTag({ + project, + tag, + bumpLevel, +}: { + project: Project; + tag: string; + bumpLevel: keyof typeof SEMVER_PARTS; +}): + | { + bumpedTag: string; + tagParts: CalverTagParts; + error: undefined; + } + | { + bumpedTag: string; + tagParts: { + prefix: string; + major: number; + minor: number; + patch: number; + }; + error: undefined; + } + | { + error: AlertError; + }; + +// Warning: (ae-missing-release-tag) "getCalverTagParts" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +function getCalverTagParts(tag: string): + | { + error: AlertError; + tagParts?: undefined; + } + | { + tagParts: CalverTagParts; + error?: undefined; + }; + +// Warning: (ae-missing-release-tag) "getSemverTagParts" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +function getSemverTagParts(tag: string): + | { + error: AlertError; + tagParts?: undefined; + } + | { + tagParts: SemverTagParts; + error?: undefined; + }; + +// Warning: (ae-missing-release-tag) "getShortCommitHash" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +function getShortCommitHash(hash: string): string; + +// Warning: (ae-missing-release-tag) "getTagParts" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +function getTagParts({ project, tag }: { project: Project; tag: string }): + | { + error: AlertError; + tagParts?: undefined; + } + | { + tagParts: CalverTagParts; + error?: undefined; + } + | { + tagParts: SemverTagParts; + error?: undefined; + }; + // Warning: (ae-forgotten-export) The symbol "GitReleaseApi" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "gitReleaseManagerApiRef" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -32,5 +154,135 @@ export const gitReleaseManagerPlugin: BackstagePlugin< {} >; +// Warning: (ae-missing-release-tag) "InfoCardPlus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const InfoCardPlus: ({ + children, +}: { + children?: React_2.ReactNode; +}) => JSX.Element; + +// Warning: (ae-missing-release-tag) "internals" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export const internals: { + components: typeof components; + constants: typeof constants; + helpers: typeof helpers; +}; + +// Warning: (ae-missing-release-tag) "isCalverTagParts" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +function isCalverTagParts( + project: Project, + _tagParts: unknown, +): _tagParts is CalverTagParts; + +// Warning: (ae-missing-release-tag) "isProjectValid" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +function isProjectValid(project: any): project is Project; + +// Warning: (ae-missing-release-tag) "LinearProgressWithLabel" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +function LinearProgressWithLabel(props: { + progress: number; + responseSteps: ResponseStep[]; +}): JSX.Element; + +// Warning: (ae-missing-release-tag) "NoLatestRelease" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const NoLatestRelease: () => JSX.Element; + +// Warning: (ae-forgotten-export) The symbol "DialogProps" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "ResponseStepDialog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const ResponseStepDialog: ({ + progress, + responseSteps, + title, +}: DialogProps) => JSX.Element; + +// Warning: (ae-forgotten-export) The symbol "ResponseStepListProps" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "ResponseStepList" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const ResponseStepList: ({ + responseSteps, + animationDelay, + loading, + denseList, + children, +}: PropsWithChildren) => JSX.Element; + +// Warning: (ae-forgotten-export) The symbol "ResponseStepListItemProps" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "ResponseStepListItem" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const ResponseStepListItem: ({ + responseStep, + animationDelay, +}: ResponseStepListItemProps) => JSX.Element; + +// Warning: (ae-missing-release-tag) "SEMVER_PARTS" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const SEMVER_PARTS: { + major: 'major'; + minor: 'minor'; + patch: 'patch'; +}; + +// Warning: (ae-missing-release-tag) "semverRegexp" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const semverRegexp: RegExp; + +// Warning: (ae-missing-release-tag) "TAG_OBJECT_MESSAGE" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const TAG_OBJECT_MESSAGE = + 'Tag generated by your friendly neighborhood Backstage Release Manager'; + +// Warning: (ae-missing-release-tag) "validateTagName" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const validateTagName: ({ + project, + tagName, +}: { + project: Project; + tagName?: string | undefined; +}) => + | { + tagNameError: null; + } + | { + tagNameError: AlertError | undefined; + }; + +// Warning: (ae-missing-release-tag) "VERSIONING_STRATEGIES" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const VERSIONING_STRATEGIES: { + semver: 'semver'; + calver: 'calver'; +}; + +// Warnings were encountered during analysis: +// +// src/components/ResponseStepDialog/LinearProgressWithLabel.d.ts:5:5 - (ae-forgotten-export) The symbol "ResponseStep" needs to be exported by the entry point index.d.ts +// src/helpers/getBumpedTag.d.ts:14:5 - (ae-forgotten-export) The symbol "Project" needs to be exported by the entry point index.d.ts +// src/helpers/getBumpedTag.d.ts:19:5 - (ae-forgotten-export) The symbol "CalverTagParts" needs to be exported by the entry point index.d.ts +// src/helpers/getBumpedTag.d.ts:31:5 - (ae-forgotten-export) The symbol "AlertError" needs to be exported by the entry point index.d.ts +// src/index.d.ts:4:5 - (ae-forgotten-export) The symbol "components" needs to be exported by the entry point index.d.ts +// src/index.d.ts:5:5 - (ae-forgotten-export) The symbol "constants" needs to be exported by the entry point index.d.ts +// src/index.d.ts:6:5 - (ae-forgotten-export) The symbol "helpers" needs to be exported by the entry point index.d.ts + // (No @packageDocumentation comment for this package) ``` From 5d369b603025e41347b6ce4df52a3d7e4ca17f43 Mon Sep 17 00:00:00 2001 From: Erik Engervall Date: Mon, 23 Aug 2021 17:17:00 +0200 Subject: [PATCH 20/33] Update plugin.test.ts with new exports Signed-off-by: Erik Engervall --- plugins/git-release-manager/src/plugin.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/git-release-manager/src/plugin.test.ts b/plugins/git-release-manager/src/plugin.test.ts index 69e2c47c6a..3431775fcb 100644 --- a/plugins/git-release-manager/src/plugin.test.ts +++ b/plugins/git-release-manager/src/plugin.test.ts @@ -21,6 +21,9 @@ describe('git-release-manager', () => { expect(Object.keys(plugin)).toMatchInlineSnapshot(` Array [ "gitReleaseManagerApiRef", + "constants", + "helpers", + "components", "gitReleaseManagerPlugin", "GitReleaseManagerPage", ] From c531345df68d943c4652c30b898f9c6e7d40de7e Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Mon, 23 Aug 2021 18:44:21 +0100 Subject: [PATCH 21/33] ability to discover catalog-infos from default branch Previously the github-discovery supported only repos from a specific branch. This change cause the discovery to work in two modes. It continues to work as it did with a wildcard for repositories on a specified branch. The new mode added causes the discovery processor to use the default branch and looks for a file called catalog-info.yaml. Signed-off-by: Brian Fletcher --- .../GithubDiscoveryProcessor.test.ts | 5 ++- .../processors/GithubDiscoveryProcessor.ts | 32 +++++++++++++++---- .../src/ingestion/processors/github/github.ts | 6 ++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index ca8c702c8f..0717e24452 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -45,12 +45,15 @@ describe('GithubDiscoveryProcessor', () => { repoSearchPath: /^proj.*$/, catalogPath: '/blob/master/catalog.yaml', }); + expect(parseUrl('https://github.com/foo')).toEqual({ + org: 'foo', + host: 'github.com', + }); }); it('throws on incorrectly formed URLs', () => { expect(() => parseUrl('https://github.com')).toThrow(); expect(() => parseUrl('https://github.com//')).toThrow(); - expect(() => parseUrl('https://github.com/foo')).toThrow(); expect(() => parseUrl('https://github.com//foo')).toThrow(); expect(() => parseUrl('https://github.com/org/teams')).toThrow(); expect(() => parseUrl('https://github.com/org//teams')).toThrow(); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index f7069a9f64..8333728fb9 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -28,7 +28,15 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; /** * Extracts repositories out of a GitHub org. - */ + * + * It can be configured in two modes. The first will create locations for all + * catalog-info.yaml files on the default branch. The second will create locations + * for all projects which have a catalog-info.yaml on the master branch. + * + * target: "https://github.com/backstage" + * or + * target: https://github.com/backstage/*\/blob/master/catalog-info.yaml + **/ export class GithubDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; private readonly logger: Logger; @@ -87,9 +95,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { this.logger.info(`Reading GitHub repositories from ${location.target}`); const { repositories } = await getOrganizationRepositories(client, org); - const matching = repositories.filter( - r => !r.isArchived && repoSearchPath.test(r.name), - ); + const matching = repoSearchPath + ? repositories.filter(r => !r.isArchived && repoSearchPath.test(r.name)) + : repositories; const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); this.logger.debug( @@ -97,11 +105,14 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { ); for (const repository of matching) { + const path = catalogPath + ? catalogPath + : `/blob/${repository.defaultBranchRef.name}/catalog-info.yaml`; emit( results.location( { type: 'url', - target: `${repository.url}${catalogPath}`, + target: `${repository.url}${path}`, }, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting @@ -121,14 +132,16 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { export function parseUrl(urlString: string): { org: string; - repoSearchPath: RegExp; - catalogPath: string; + repoSearchPath?: RegExp; + catalogPath?: string; host: string; } { const url = new URL(urlString); const path = url.pathname.substr(1).split('/'); // /backstage/techdocs-*/blob/master/catalog-info.yaml + // can also be + // /backstage if (path.length > 2 && path[0].length && path[1].length) { return { org: decodeURIComponent(path[0]), @@ -136,6 +149,11 @@ export function parseUrl(urlString: string): { catalogPath: `/${decodeURIComponent(path.slice(2).join('/'))}`, host: url.host, }; + } else if (path.length === 1 && path[0].length) { + return { + org: decodeURIComponent(path[0]), + host: url.host, + }; } throw new Error(`Failed to parse ${urlString}`); diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index 70479110c2..9eb5073710 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -60,6 +60,9 @@ export type Repository = { name: string; url: string; isArchived: boolean; + defaultBranchRef: { + name: string; + }; }; export type Connection = { @@ -252,6 +255,9 @@ export async function getOrganizationRepositories( name url isArchived + defaultBranchRef { + name + } } pageInfo { hasNextPage From 8b39242c4850725fe1b4661019c0ece7d0c05382 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Mon, 23 Aug 2021 20:52:05 +0100 Subject: [PATCH 22/33] adds changeset for github discovery change Signed-off-by: Brian Fletcher --- .changeset/big-jars-repair.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/big-jars-repair.md diff --git a/.changeset/big-jars-repair.md b/.changeset/big-jars-repair.md new file mode 100644 index 0000000000..63fef2c740 --- /dev/null +++ b/.changeset/big-jars-repair.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +GitHub discovery processor adds support for discovering the default GitHub branch From f337c2fa2fe75df9696b0c2e58d78f5fb18f9113 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Mon, 23 Aug 2021 20:54:41 +0100 Subject: [PATCH 23/33] fixes tests types Signed-off-by: Brian Fletcher --- .../GithubDiscoveryProcessor.test.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 0717e24452..7ebb5b0da1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -128,11 +128,17 @@ describe('GithubDiscoveryProcessor', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'master', + }, }, { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, ], }); @@ -171,16 +177,25 @@ describe('GithubDiscoveryProcessor', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'techdocs-cli', url: 'https://github.com/backstage/techdocs-cli', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'techdocs-container', url: 'https://github.com/backstage/techdocs-container', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, ], }); @@ -218,21 +233,33 @@ describe('GithubDiscoveryProcessor', () => { name: 'abstest', url: 'https://github.com/backstage/abctest', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'test', url: 'https://github.com/backstage/test', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'test-archived', url: 'https://github.com/backstage/test', isArchived: true, + defaultBranchRef: { + name: 'main', + }, }, { name: 'testxyz', url: 'https://github.com/backstage/testxyz', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, ], }); From 1a98eb41e6e096d182f185a72cdcf18246ad2e17 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Mon, 23 Aug 2021 21:08:01 +0100 Subject: [PATCH 24/33] fixing more tests mocks Signed-off-by: Brian Fletcher --- .../src/ingestion/processors/github/github.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts index 11cdf671c3..e481355fb4 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts @@ -201,11 +201,17 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + defaultBranchRef: { + name: 'main', + }, }, ], pageInfo: { @@ -221,11 +227,17 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + defaultBranchRef: { + name: 'main', + }, }, ], }; From 04ef773b94943a9828f270a12ae5253263964b76 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 24 Aug 2021 09:49:43 +0200 Subject: [PATCH 25/33] Reduce changset level to 'patch' Signed-off-by: Roy Jacobs --- .changeset/lucky-gifts-help.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/lucky-gifts-help.md b/.changeset/lucky-gifts-help.md index 1bc100d824..3ed6b12498 100644 --- a/.changeset/lucky-gifts-help.md +++ b/.changeset/lucky-gifts-help.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-catalog-backend': patch --- Added GitLabDiscoveryProcessor, which allows catalog discovery from a GitLab instance From bdf30530d80c3a3447dbc9d818010901366ecb9f Mon Sep 17 00:00:00 2001 From: Erik Engervall Date: Tue, 24 Aug 2021 13:28:50 +0200 Subject: [PATCH 26/33] Export testHelpers as well to further aid development of custom features Signed-off-by: Erik Engervall --- plugins/git-release-manager/src/index.ts | 3 ++- plugins/git-release-manager/src/plugin.ts | 3 ++- .../src/test-helpers/index.ts | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 plugins/git-release-manager/src/test-helpers/index.ts diff --git a/plugins/git-release-manager/src/index.ts b/plugins/git-release-manager/src/index.ts index cb903044f4..c325a3335b 100644 --- a/plugins/git-release-manager/src/index.ts +++ b/plugins/git-release-manager/src/index.ts @@ -20,10 +20,11 @@ export { gitReleaseManagerApiRef, } from './plugin'; -import { components, constants, helpers } from './plugin'; +import { components, constants, helpers, testHelpers } from './plugin'; export const internals = { components, constants, helpers, + testHelpers, }; diff --git a/plugins/git-release-manager/src/plugin.ts b/plugins/git-release-manager/src/plugin.ts index 03027a5369..a416ab9092 100644 --- a/plugins/git-release-manager/src/plugin.ts +++ b/plugins/git-release-manager/src/plugin.ts @@ -28,8 +28,9 @@ import { rootRouteRef } from './routes'; import * as constants from './constants/constants'; import * as helpers from './helpers'; import * as components from './components'; +import * as testHelpers from './test-helpers'; -export { gitReleaseManagerApiRef, constants, helpers, components }; +export { gitReleaseManagerApiRef, constants, helpers, components, testHelpers }; export const gitReleaseManagerPlugin = createPlugin({ id: 'git-release-manager', diff --git a/plugins/git-release-manager/src/test-helpers/index.ts b/plugins/git-release-manager/src/test-helpers/index.ts new file mode 100644 index 0000000000..07c59e68cd --- /dev/null +++ b/plugins/git-release-manager/src/test-helpers/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * as stats from './stats'; +export * as testHelpers from './test-helpers'; +export * as testIds from './test-ids'; From 9ad8641161c9bcf643daadc1343265885f9efc8d Mon Sep 17 00:00:00 2001 From: Erik Engervall Date: Tue, 24 Aug 2021 13:40:34 +0200 Subject: [PATCH 27/33] Export testHelpers properly Signed-off-by: Erik Engervall --- plugins/git-release-manager/src/test-helpers/index.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/git-release-manager/src/test-helpers/index.ts b/plugins/git-release-manager/src/test-helpers/index.ts index 07c59e68cd..2a91928321 100644 --- a/plugins/git-release-manager/src/test-helpers/index.ts +++ b/plugins/git-release-manager/src/test-helpers/index.ts @@ -14,6 +14,8 @@ * limitations under the License. */ -export * as stats from './stats'; -export * as testHelpers from './test-helpers'; -export * as testIds from './test-ids'; +import * as stats from './stats'; +import * as testHelpers from './test-helpers'; +import * as testIds from './test-ids'; + +export { stats, testHelpers, testIds }; From 33b5933c4b64744d7e00c03e88bcc17f7da47f60 Mon Sep 17 00:00:00 2001 From: Erik Engervall Date: Tue, 24 Aug 2021 13:43:05 +0200 Subject: [PATCH 28/33] Update api-report.md Signed-off-by: Erik Engervall --- plugins/git-release-manager/api-report.md | 261 ++++++++++++++++++++++ 1 file changed, 261 insertions(+) diff --git a/plugins/git-release-manager/api-report.md b/plugins/git-release-manager/api-report.md index 44fafd7387..2c3b55a110 100644 --- a/plugins/git-release-manager/api-report.md +++ b/plugins/git-release-manager/api-report.md @@ -17,6 +17,20 @@ import { RouteRef } from '@backstage/core-plugin-api'; // @public (undocumented) const calverRegexp: RegExp; +// Warning: (ae-forgotten-export) The symbol "GetCommitResult" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "createMockCommit" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const createMockCommit: ( + overrides: Partial, +) => GetCommitResult; + +// Warning: (ae-forgotten-export) The symbol "GetTagResult" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "createMockTag" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const createMockTag: (overrides: Partial) => GetTagResult; + // Warning: (ae-forgotten-export) The symbol "DifferProps" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "Differ" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -170,6 +184,7 @@ export const internals: { components: typeof components; constants: typeof constants; helpers: typeof helpers; + testHelpers: typeof testHelpers; }; // Warning: (ae-missing-release-tag) "isCalverTagParts" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -193,6 +208,153 @@ function LinearProgressWithLabel(props: { responseSteps: ResponseStep[]; }): JSX.Element; +// Warning: (ae-missing-release-tag) "mockApiClient" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +const mockApiClient: GitReleaseApi; + +// Warning: (ae-missing-release-tag) "mockBumpedTag" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockBumpedTag = 'rc-2020.01.01_1337'; + +// Warning: (ae-missing-release-tag) "mockCalverProject" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockCalverProject: Project; + +// Warning: (ae-missing-release-tag) "mockDefaultBranch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockDefaultBranch = 'mock_defaultBranch'; + +// Warning: (ae-forgotten-export) The symbol "getReleaseCandidateGitInfo" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "mockNextGitInfoCalver" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockNextGitInfoCalver: ReturnType; + +// Warning: (ae-missing-release-tag) "mockNextGitInfoSemver" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockNextGitInfoSemver: ReturnType; + +// Warning: (ae-missing-release-tag) "mockReleaseBranch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockReleaseBranch: { + name: string; + links: { + html: string; + }; + commit: { + sha: string; + commit: { + tree: { + sha: string; + }; + }; + }; +}; + +// Warning: (ae-missing-release-tag) "mockReleaseCandidateCalver" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockReleaseCandidateCalver: { + targetCommitish: string; + tagName: string; + prerelease: boolean; + id: number; + htmlUrl: string; + body?: string | null | undefined; +}; + +// Warning: (ae-missing-release-tag) "mockReleaseCandidateSemver" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockReleaseCandidateSemver: { + targetCommitish: string; + tagName: string; + prerelease: boolean; + id: number; + htmlUrl: string; + body?: string | null | undefined; +}; + +// Warning: (ae-forgotten-export) The symbol "ReleaseStats" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "mockReleaseStats" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockReleaseStats: ReleaseStats; + +// Warning: (ae-missing-release-tag) "mockReleaseVersionCalver" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockReleaseVersionCalver: { + targetCommitish: string; + tagName: string; + prerelease: boolean; + id: number; + htmlUrl: string; + body?: string | null | undefined; +}; + +// Warning: (ae-missing-release-tag) "mockReleaseVersionSemver" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockReleaseVersionSemver: { + targetCommitish: string; + tagName: string; + prerelease: boolean; + id: number; + htmlUrl: string; + body?: string | null | undefined; +}; + +// Warning: (ae-missing-release-tag) "mockSearchCalver" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockSearchCalver: string; + +// Warning: (ae-missing-release-tag) "mockSearchSemver" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockSearchSemver: string; + +// Warning: (ae-missing-release-tag) "mockSelectedPatchCommit" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockSelectedPatchCommit: { + htmlUrl: string; + sha: string; + author: { + htmlUrl?: string | undefined; + login?: string | undefined; + }; + commit: { + message: string; + }; + firstParentSha?: string | undefined; +}; + +// Warning: (ae-missing-release-tag) "mockSemverProject" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockSemverProject: Project; + +// Warning: (ae-missing-release-tag) "mockTagParts" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockTagParts: CalverTagParts; + +// Warning: (ae-missing-release-tag) "mockUser" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const mockUser: { + username: string; + email: string; +}; + // Warning: (ae-missing-release-tag) "NoLatestRelease" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -243,12 +405,110 @@ const SEMVER_PARTS: { // @public (undocumented) const semverRegexp: RegExp; +declare namespace stats { + export { mockReleaseStats }; +} + // Warning: (ae-missing-release-tag) "TAG_OBJECT_MESSAGE" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) const TAG_OBJECT_MESSAGE = 'Tag generated by your friendly neighborhood Backstage Release Manager'; +// Warning: (ae-missing-release-tag) "TEST_IDS" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const TEST_IDS: { + info: { + info: string; + infoFeaturePlus: string; + }; + createRc: { + cta: string; + semverSelect: string; + }; + promoteRc: { + mockedPromoteRcBody: string; + notRcWarning: string; + promoteRc: string; + cta: string; + }; + patch: { + error: string; + loading: string; + notPrerelease: string; + body: string; + }; + form: { + owner: { + loading: string; + select: string; + error: string; + empty: string; + }; + repo: { + loading: string; + select: string; + error: string; + empty: string; + }; + versioningStrategy: { + radioGroup: string; + }; + }; + components: { + divider: string; + noLatestRelease: string; + circularProgress: string; + responseStepListDialogContent: string; + responseStepListItem: string; + responseStepListItemIconSuccess: string; + responseStepListItemIconFailure: string; + responseStepListItemIconLink: string; + responseStepListItemIconDefault: string; + differ: { + current: string; + next: string; + icons: { + tag: string; + branch: string; + github: string; + slack: string; + versioning: string; + }; + }; + linearProgressWithLabel: string; + }; +}; + +declare namespace testHelpers_2 { + export { + createMockTag, + createMockCommit, + mockUser, + mockSemverProject, + mockCalverProject, + mockSearchCalver, + mockSearchSemver, + mockDefaultBranch, + mockNextGitInfoSemver, + mockNextGitInfoCalver, + mockTagParts, + mockBumpedTag, + mockReleaseCandidateCalver, + mockReleaseVersionCalver, + mockReleaseCandidateSemver, + mockReleaseVersionSemver, + mockReleaseBranch, + mockSelectedPatchCommit, + mockApiClient, + }; +} + +declare namespace testIds { + export { TEST_IDS }; +} + // Warning: (ae-missing-release-tag) "validateTagName" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -283,6 +543,7 @@ const VERSIONING_STRATEGIES: { // src/index.d.ts:4:5 - (ae-forgotten-export) The symbol "components" needs to be exported by the entry point index.d.ts // src/index.d.ts:5:5 - (ae-forgotten-export) The symbol "constants" needs to be exported by the entry point index.d.ts // src/index.d.ts:6:5 - (ae-forgotten-export) The symbol "helpers" needs to be exported by the entry point index.d.ts +// src/index.d.ts:7:5 - (ae-forgotten-export) The symbol "testHelpers" needs to be exported by the entry point index.d.ts // (No @packageDocumentation comment for this package) ``` From d3e9e2485bb7c6f7a1b2f47ccae2b1afa5f94bd4 Mon Sep 17 00:00:00 2001 From: Erik Engervall Date: Tue, 24 Aug 2021 14:00:09 +0200 Subject: [PATCH 29/33] Update plugin.test.ts with newly added testHelpers export Signed-off-by: Erik Engervall --- plugins/git-release-manager/src/plugin.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/git-release-manager/src/plugin.test.ts b/plugins/git-release-manager/src/plugin.test.ts index 3431775fcb..db60617c24 100644 --- a/plugins/git-release-manager/src/plugin.test.ts +++ b/plugins/git-release-manager/src/plugin.test.ts @@ -24,6 +24,7 @@ describe('git-release-manager', () => { "constants", "helpers", "components", + "testHelpers", "gitReleaseManagerPlugin", "GitReleaseManagerPage", ] From 5a1eb6bfc44305711b28c2acc408a2c9f130bf32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 24 Aug 2021 15:08:55 +0200 Subject: [PATCH 30/33] Memoize the context value in `EntityListProvider` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/lemon-dogs-switch.md | 14 ++++++++ .github/styles/vocab.txt | 2 ++ .../src/hooks/useEntityListProvider.tsx | 34 +++++++++++++------ 3 files changed, 39 insertions(+), 11 deletions(-) create mode 100644 .changeset/lemon-dogs-switch.md diff --git a/.changeset/lemon-dogs-switch.md b/.changeset/lemon-dogs-switch.md new file mode 100644 index 0000000000..aa86e3c2b0 --- /dev/null +++ b/.changeset/lemon-dogs-switch.md @@ -0,0 +1,14 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +Memoize the context value in `EntityListProvider`. + +This removes quite a few unnecessary rerenders of the inner components. + +When running the full `CatalogPage` test: + +- Before: 98 table render calls total, 16 seconds runtime +- After: 57 table render calls total, 14 seconds runtime + +This doesn't account for all of the slowness, but does give a minor difference in perceived speed in the browser too. diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 57510f2c3f..dd9224fc84 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -142,6 +142,7 @@ maintainership makefile md memcache +memoize memoized microservice microservices @@ -215,6 +216,7 @@ repo Repo repos rerender +rerenders Reusability reusability roadmaps diff --git a/plugins/catalog-react/src/hooks/useEntityListProvider.tsx b/plugins/catalog-react/src/hooks/useEntityListProvider.tsx index 54d2fdcd47..d5c4668079 100644 --- a/plugins/catalog-react/src/hooks/useEntityListProvider.tsx +++ b/plugins/catalog-react/src/hooks/useEntityListProvider.tsx @@ -22,6 +22,7 @@ import React, { PropsWithChildren, useCallback, useContext, + useMemo, useState, } from 'react'; import { useSearchParams } from 'react-router-dom'; @@ -197,18 +198,29 @@ export const EntityListProvider = ({ [], ); + const value = useMemo( + () => ({ + filters: outputState.appliedFilters, + entities: outputState.entities, + backendEntities: outputState.backendEntities, + updateFilters, + queryParameters: outputState.queryParameters, + loading, + error, + }), + [ + outputState.appliedFilters, + outputState.entities, + outputState.backendEntities, + updateFilters, + outputState.queryParameters, + loading, + error, + ], + ); + return ( - + {children} ); From 16ec8381ad9c16da7590f6a1ba9e65512803d41a Mon Sep 17 00:00:00 2001 From: Phil Kuang Date: Fri, 6 Aug 2021 15:42:39 -0400 Subject: [PATCH 31/33] fix(search): Fix search page to respond to searches made from sidebar Signed-off-by: Phil Kuang --- .changeset/wild-olives-applaud.md | 5 ++ .../src/components/SearchBar/SearchBar.tsx | 6 +- .../SearchContext/SearchContext.tsx | 2 +- .../components/SearchPage/SearchPage.test.tsx | 29 ++++-- .../src/components/SearchPage/SearchPage.tsx | 89 ++++++++++++------- 5 files changed, 89 insertions(+), 42 deletions(-) create mode 100644 .changeset/wild-olives-applaud.md diff --git a/.changeset/wild-olives-applaud.md b/.changeset/wild-olives-applaud.md new file mode 100644 index 0000000000..14637cc842 --- /dev/null +++ b/.changeset/wild-olives-applaud.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search': patch +--- + +Fix search page to respond to searches made from sidebar search diff --git a/plugins/search/src/components/SearchBar/SearchBar.tsx b/plugins/search/src/components/SearchBar/SearchBar.tsx index a1db818488..bdff003b7a 100644 --- a/plugins/search/src/components/SearchBar/SearchBar.tsx +++ b/plugins/search/src/components/SearchBar/SearchBar.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { ChangeEvent, useState } from 'react'; +import React, { ChangeEvent, useEffect, useState } from 'react'; import { useDebounce } from 'react-use'; import { InputBase, InputAdornment, IconButton } from '@material-ui/core'; import SearchIcon from '@material-ui/icons/Search'; @@ -31,6 +31,10 @@ export const SearchBar = ({ className, debounceTime = 0 }: Props) => { const { term, setTerm } = useSearch(); const [value, setValue] = useState(term); + useEffect(() => { + setValue(prevValue => (prevValue !== term ? term : prevValue)); + }, [term]); + useDebounce(() => setTerm(value), debounceTime, [value]); const handleQuery = (e: ChangeEvent) => { diff --git a/plugins/search/src/components/SearchContext/SearchContext.tsx b/plugins/search/src/components/SearchContext/SearchContext.tsx index 619471d490..45a74f5355 100644 --- a/plugins/search/src/components/SearchContext/SearchContext.tsx +++ b/plugins/search/src/components/SearchContext/SearchContext.tsx @@ -54,7 +54,7 @@ export const SearchContextProvider = ({ term: '', pageCursor: '', filters: {}, - types: ['*'], + types: [], }, children, }: PropsWithChildren<{ initialState?: SettableSearchContext }>) => { diff --git a/plugins/search/src/components/SearchPage/SearchPage.test.tsx b/plugins/search/src/components/SearchPage/SearchPage.test.tsx index 4c1ae38005..c387d57b60 100644 --- a/plugins/search/src/components/SearchPage/SearchPage.test.tsx +++ b/plugins/search/src/components/SearchPage/SearchPage.test.tsx @@ -18,7 +18,7 @@ import React from 'react'; import { renderInTestApp } from '@backstage/test-utils'; import { useLocation, useOutlet } from 'react-router'; -import { useSearch, SearchContextProvider } from '../SearchContext'; +import { useSearch } from '../SearchContext'; import { SearchPage } from './'; jest.mock('react-router', () => ({ @@ -29,6 +29,11 @@ jest.mock('react-router', () => ({ useOutlet: jest.fn().mockReturnValue('Route Children'), })); +const setTermMock = jest.fn(); +const setTypesMock = jest.fn(); +const setFiltersMock = jest.fn(); +const setPageCursorMock = jest.fn(); + jest.mock('../SearchContext', () => ({ ...jest.requireActual('../SearchContext'), SearchContextProvider: jest @@ -36,9 +41,13 @@ jest.mock('../SearchContext', () => ({ .mockImplementation(({ children }) => children), useSearch: jest.fn().mockReturnValue({ term: '', + setTerm: (term: any) => setTermMock(term), types: [], + setTypes: (types: any) => setTypesMock(types), filters: {}, + setFilters: (filters: any) => setFiltersMock(filters), pageCursor: '', + setPageCursor: (pageCursor: any) => setPageCursorMock(pageCursor), }), })); @@ -58,7 +67,7 @@ describe('SearchPage', () => { window.history.replaceState = origReplaceState; }); - it('uses initial term state from location', async () => { + it('sets term state from location', async () => { // Given this initial location.search value... const expectedFilterField = 'anyKey'; const expectedFilterValue = 'anyValue'; @@ -75,13 +84,11 @@ describe('SearchPage', () => { // When we render the page... await renderInTestApp(); - // Then search context should be initialized with these values... - const calls = (SearchContextProvider as jest.Mock).mock.calls[0]; - const actualInitialState = calls[0].initialState; - expect(actualInitialState.term).toEqual(expectedTerm); - expect(actualInitialState.types).toEqual(expectedTypes); - expect(actualInitialState.pageCursor).toEqual(expectedPageCursor); - expect(actualInitialState.filters).toStrictEqual(expectedFilters); + // Then search context should be set with these values... + expect(setTermMock).toHaveBeenCalledWith(expectedTerm); + expect(setTypesMock).toHaveBeenCalledWith(expectedTypes); + expect(setPageCursorMock).toHaveBeenCalledWith(expectedPageCursor); + expect(setFiltersMock).toHaveBeenCalledWith(expectedFilters); }); it('renders provided router element', async () => { @@ -103,6 +110,10 @@ describe('SearchPage', () => { types: ['software-catalog'], pageCursor: 'page2-or-something', filters: { anyKey: 'anyValue' }, + setTerm: setTermMock, + setTypes: setTypesMock, + setFilters: setFiltersMock, + setPageCursor: setPageCursorMock, }); const expectedLocation = encodeURI( '?query=bieber&types[]=software-catalog&pageCursor=page2-or-something&filters[anyKey]=anyValue', diff --git a/plugins/search/src/components/SearchPage/SearchPage.tsx b/plugins/search/src/components/SearchPage/SearchPage.tsx index 8a85879a48..fb18833da6 100644 --- a/plugins/search/src/components/SearchPage/SearchPage.tsx +++ b/plugins/search/src/components/SearchPage/SearchPage.tsx @@ -14,7 +14,8 @@ * limitations under the License. */ -import React from 'react'; +import React, { useEffect } from 'react'; +import { usePrevious } from 'react-use'; import qs from 'qs'; import { useLocation, useOutlet } from 'react-router'; import { SearchContextProvider, useSearch } from '../SearchContext'; @@ -22,46 +23,72 @@ import { JsonObject } from '@backstage/config'; import { LegacySearchPage } from '../LegacySearchPage'; export const UrlUpdater = () => { - const { term, types, pageCursor, filters } = useSearch(); + const location = useLocation(); + const { + term, + setTerm, + types, + setTypes, + pageCursor, + setPageCursor, + filters, + setFilters, + } = useSearch(); - const newParams = qs.stringify( - { - query: term, - types, - pageCursor, - filters, - }, - { arrayFormat: 'brackets' }, - ); - const newUrl = `${window.location.pathname}?${newParams}`; + const prevQueryParams = usePrevious(location.search); + useEffect(() => { + // Only respond to changes to url query params + if (location.search === prevQueryParams) { + return; + } - // We directly manipulate window history here in order to not re-render - // infinitely (state => location => state => etc). The intention of this - // code is just to ensure the right query/filters are loaded when a user - // clicks the "back" button after clicking a result. - window.history.replaceState(null, document.title, newUrl); + const query = + qs.parse(location.search.substring(1), { arrayLimit: 0 }) || {}; + + if (query.filters) { + setFilters(query.filters as JsonObject); + } + + if (query.query) { + setTerm(query.query as string); + } + + if (query.pageCursor) { + setPageCursor(query.pageCursor as string); + } + + if (query.types) { + setTypes(query.types as string[]); + } + }, [prevQueryParams, location, setTerm, setTypes, setPageCursor, setFilters]); + + useEffect(() => { + const newParams = qs.stringify( + { + query: term, + types, + pageCursor, + filters, + }, + { arrayFormat: 'brackets' }, + ); + const newUrl = `${window.location.pathname}?${newParams}`; + + // We directly manipulate window history here in order to not re-render + // infinitely (state => location => state => etc). The intention of this + // code is just to ensure the right query/filters are loaded when a user + // clicks the "back" button after clicking a result. + window.history.replaceState(null, document.title, newUrl); + }, [term, types, pageCursor, filters]); return null; }; export const SearchPage = () => { - const location = useLocation(); const outlet = useOutlet(); - const query = qs.parse(location.search.substring(1), { arrayLimit: 0 }) || {}; - const filters = (query.filters as JsonObject) || {}; - const queryString = (query.query as string) || ''; - const pageCursor = (query.pageCursor as string) || ''; - const types = (query.types as string[]) || []; - - const initialState = { - term: queryString || '', - types, - pageCursor, - filters, - }; return ( - + {outlet || } From b000c79abfefe506f3173565fde7e7ad681382da Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 24 Aug 2021 16:23:55 +0100 Subject: [PATCH 32/33] Support full gamit of path options for discovery It'll now support anything from: https://github.com/backstage to https://github.com/backstage/*\/blob/-/catalog-info.yaml Signed-off-by: Brian Fletcher --- .../GithubDiscoveryProcessor.test.ts | 9 +++-- .../processors/GithubDiscoveryProcessor.ts | 36 +++++++++++-------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 7ebb5b0da1..f8647cb5d2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -35,7 +35,8 @@ describe('GithubDiscoveryProcessor', () => { org: 'foo', host: 'github.com', repoSearchPath: /^proj$/, - catalogPath: '/blob/master/catalog.yaml', + branch: 'master', + catalogPath: '/catalog.yaml', }); expect( parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'), @@ -43,11 +44,15 @@ describe('GithubDiscoveryProcessor', () => { org: 'foo', host: 'github.com', repoSearchPath: /^proj.*$/, - catalogPath: '/blob/master/catalog.yaml', + branch: 'master', + catalogPath: '/catalog.yaml', }); expect(parseUrl('https://github.com/foo')).toEqual({ org: 'foo', host: 'github.com', + repoSearchPath: /^.*$/, + branch: '-', + catalogPath: '/catalog-info.yaml', }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index 8333728fb9..567f6536d0 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -29,13 +29,16 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; /** * Extracts repositories out of a GitHub org. * - * It can be configured in two modes. The first will create locations for all - * catalog-info.yaml files on the default branch. The second will create locations - * for all projects which have a catalog-info.yaml on the master branch. + * The following will create locations for all projects which have a catalog-info.yaml + * on the default branch. The first is shorthand for the second. * * target: "https://github.com/backstage" * or - * target: https://github.com/backstage/*\/blob/master/catalog-info.yaml + * target: https://github.com/backstage/*\/blob/-/catalog-info.yaml + * + * You may also explicitly specify the source branch: + * + * target: https://github.com/backstage/*\/blob/main/catalog-info.yaml **/ export class GithubDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; @@ -73,7 +76,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { ); } - const { org, repoSearchPath, catalogPath, host } = parseUrl( + const { org, repoSearchPath, catalogPath, branch, host } = parseUrl( location.target, ); @@ -95,9 +98,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { this.logger.info(`Reading GitHub repositories from ${location.target}`); const { repositories } = await getOrganizationRepositories(client, org); - const matching = repoSearchPath - ? repositories.filter(r => !r.isArchived && repoSearchPath.test(r.name)) - : repositories; + const matching = repositories.filter( + r => !r.isArchived && repoSearchPath.test(r.name), + ); const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); this.logger.debug( @@ -105,9 +108,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { ); for (const repository of matching) { - const path = catalogPath - ? catalogPath - : `/blob/${repository.defaultBranchRef.name}/catalog-info.yaml`; + const path = `/blob/${ + branch === '-' ? repository.defaultBranchRef.name : branch + }${catalogPath}`; emit( results.location( { @@ -132,8 +135,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { export function parseUrl(urlString: string): { org: string; - repoSearchPath?: RegExp; - catalogPath?: string; + repoSearchPath: RegExp; + catalogPath: string; + branch: string; host: string; } { const url = new URL(urlString); @@ -146,13 +150,17 @@ export function parseUrl(urlString: string): { return { org: decodeURIComponent(path[0]), repoSearchPath: escapeRegExp(decodeURIComponent(path[1])), - catalogPath: `/${decodeURIComponent(path.slice(2).join('/'))}`, + branch: decodeURIComponent(path[3]), + catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`, host: url.host, }; } else if (path.length === 1 && path[0].length) { return { org: decodeURIComponent(path[0]), host: url.host, + repoSearchPath: escapeRegExp('*'), + catalogPath: '/catalog-info.yaml', + branch: '-', }; } From ea9fe95674abf480421d2b96c35809cb32e6e0a6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 24 Aug 2021 17:42:04 +0200 Subject: [PATCH 33/33] auth-backend: fix but where undefined state values where being stringified Signed-off-by: Patrik Oldsberg --- .changeset/light-spoons-listen.md | 5 +++ plugins/auth-backend/package.json | 1 + .../src/lib/oauth/helpers.test.ts | 32 ++++++++++++++++++- plugins/auth-backend/src/lib/oauth/helpers.ts | 5 ++- 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 .changeset/light-spoons-listen.md diff --git a/.changeset/light-spoons-listen.md b/.changeset/light-spoons-listen.md new file mode 100644 index 0000000000..ec3a85cb0d --- /dev/null +++ b/.changeset/light-spoons-listen.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Fixed a bug where OAuth state parameters would be serialized as the string `'undefined'`. diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 26bd5b408e..38ad491b19 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -50,6 +50,7 @@ "jose": "^1.27.1", "jwt-decode": "^3.1.0", "knex": "^0.95.1", + "lodash": "^4.17.21", "luxon": "^2.0.2", "minimatch": "^3.0.3", "morgan": "^1.10.0", diff --git a/plugins/auth-backend/src/lib/oauth/helpers.test.ts b/plugins/auth-backend/src/lib/oauth/helpers.test.ts index 00161ff8cf..8af1d2f2ad 100644 --- a/plugins/auth-backend/src/lib/oauth/helpers.test.ts +++ b/plugins/auth-backend/src/lib/oauth/helpers.test.ts @@ -15,9 +15,39 @@ */ import express from 'express'; -import { verifyNonce, encodeState } from './helpers'; +import { verifyNonce, encodeState, readState } from './helpers'; describe('OAuthProvider Utils', () => { + describe('encodeState', () => { + it('should serialized values', () => { + const state = { + nonce: '123', + env: 'development', + origin: 'https://example.com', + }; + + const encoded = encodeState(state); + expect(encoded).toBe( + Buffer.from( + 'nonce=123&env=development&origin=https%3A%2F%2Fexample.com', + ).toString('hex'), + ); + + expect(readState(encoded)).toEqual(state); + }); + + it('should not include undefined values', () => { + const state = { nonce: '123', env: 'development', origin: undefined }; + + const encoded = encodeState(state); + expect(encoded).toBe( + Buffer.from('nonce=123&env=development').toString('hex'), + ); + + expect(readState(encoded)).toEqual(state); + }); + }); + describe('verifyNonce', () => { it('should throw error if cookie nonce missing', () => { const state = { nonce: 'NONCE', env: 'development' }; diff --git a/plugins/auth-backend/src/lib/oauth/helpers.ts b/plugins/auth-backend/src/lib/oauth/helpers.ts index ead9acae27..17e3769d21 100644 --- a/plugins/auth-backend/src/lib/oauth/helpers.ts +++ b/plugins/auth-backend/src/lib/oauth/helpers.ts @@ -16,6 +16,7 @@ import express from 'express'; import { OAuthState } from './types'; +import pickBy from 'lodash/pickBy'; export const readState = (stateString: string): OAuthState => { const state = Object.fromEntries( @@ -34,7 +35,9 @@ export const readState = (stateString: string): OAuthState => { }; export const encodeState = (state: OAuthState): string => { - const stateString = new URLSearchParams(state).toString(); + const stateString = new URLSearchParams( + pickBy(state, value => value !== undefined), + ).toString(); return Buffer.from(stateString, 'utf-8').toString('hex'); };