From a0b80e7bce9120e696086f1f8c0498ef6e980c7e Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 14 Jul 2023 11:43:47 +0100 Subject: [PATCH 001/131] Add users and groups ingestion for gitlab.com Signed-off-by: angom1 --- .../src/lib/client.ts | 165 +++++++++++++++++- .../src/lib/types.ts | 52 ++++++ .../GitlabOrgDiscoveryEntityProvider.ts | 25 +-- 3 files changed, 231 insertions(+), 11 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index f0e169d8df..d2a3fa01e0 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -20,7 +20,13 @@ import { GitLabIntegrationConfig, } from '@backstage/integration'; import { Logger } from 'winston'; -import { GitLabGroup, GitLabGroupMembersResponse, GitLabUser } from './types'; +import { + GitLabGroup, + GitLabSaasUsersResponse, + GitLabSaasGroupsResponse, + GitLabGroupMembersResponse, + GitLabUser, +} from './types'; export type CommonListOptions = { [key: string]: string | number | boolean | undefined; @@ -91,6 +97,163 @@ export class GitLabClient { return this.pagedRequest(`/groups`, options); } + async listSaasUsers(groupPath: string): Promise> { + const items: GitLabUser[] = []; + let hasNextPage: boolean = false; + let endCursor: string | null = null; + + if (!groupPath) { + throw new Error( + `Missing required 'group' value in gitlab provider config.`, + ); + } + + do { + const response: GitLabSaasUsersResponse = await fetch( + `${this.config.baseUrl}/api/graphql`, + { + method: 'POST', + headers: { + ...getGitLabRequestOptions(this.config).headers, + ['Content-Type']: 'application/json', + }, + body: JSON.stringify({ + variables: { group: groupPath, endCursor }, + query: `query($group: ID!, $endCursor: String) { + group(fullPath: $group) { + groupMembers(first: 100, relations: [DESCENDANTS], after: $endCursor) { + nodes { + user { + id + username + commitEmail + name + state + webUrl + avatarUrl + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + }`, + }), + }, + ).then(r => r.json()); + if (response.errors) { + throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`); + } + + if (!response.data.group?.groupMembers?.nodes) { + this.logger.warn( + `Couldn't get members under ${groupPath}. The provided token might not have sufficient permissions`, + ); + continue; + } + + const usersData = response.data.group.groupMembers.nodes; + + for (let i = 0; i < usersData.length; i++) { + const userItem = usersData[i]; + + const formattedUserResponse = { + id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), + username: userItem.user.username, + email: userItem.user.commitEmail, + name: userItem.user.name, + state: userItem.user.state, + web_url: userItem.user.webUrl, + avatar_url: userItem.user.avatarUrl, + }; + + items.push(formattedUserResponse); + } + ({ hasNextPage, endCursor } = response.data.group.groupMembers.pageInfo); + } while (hasNextPage); + return { items }; + } + + async listSaasGroups(groupPath: string): Promise> { + const items: GitLabGroup[] = []; + let hasNextPage: boolean = false; + let endCursor: string | null = null; + + if (!groupPath) { + throw new Error( + `Missing required 'group' value in gitlab provider config.`, + ); + } + + do { + const response: GitLabSaasGroupsResponse = await fetch( + `${this.config.baseUrl}/api/graphql`, + { + method: 'POST', + headers: { + ...getGitLabRequestOptions(this.config).headers, + ['Content-Type']: 'application/json', + }, + body: JSON.stringify({ + variables: { group: groupPath, endCursor }, + query: `query($group: ID!, $endCursor: String) { + group(fullPath: $group) { + descendantGroups(first: 100, after: $endCursor){ + nodes{ + id + name + description + fullPath + parent{ + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + }`, + }), + }, + ).then(r => r.json()); + if (response.errors) { + throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`); + } + + if (!response.data.group?.descendantGroups.nodes) { + this.logger.warn( + `Could not get groups under ${groupPath}. The provided token might not have sufficient permissions`, + ); + continue; + } + + const groupsData = response.data.group.descendantGroups.nodes; + + for (let i = 0; i < groupsData.length; i++) { + const groupItem = groupsData[i]; + + const formattedGroupResponse = { + id: Number(groupItem.id.replace(/^gid:\/\/gitlab\/Group\//, '')), + name: groupItem.name, + description: groupItem.description, + full_path: groupItem.fullPath, + parent_id: Number( + groupItem.parent.id.replace(/^gid:\/\/gitlab\/Group\//, ''), + ), + }; + + items.push(formattedGroupResponse); + } + ({ hasNextPage, endCursor } = + response.data.group.descendantGroups.pageInfo); + } while (hasNextPage); + return { items }; + } + async getGroupMembers(groupPath: string): Promise { const memberIds = []; let hasNextPage: boolean = false; diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 341e2a9d9a..a00c66b4a7 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -70,6 +70,58 @@ export type GitLabGroupMembersResponse = { }; }; +export type GitLabSaasUsersResponse = { + errors: { message: string }[]; + data: { + group: { + groupMembers: { + nodes: [ + { + user: { + id: string; + username: string; + commitEmail: string; + name: string; + state: string; + webUrl: string; + avatarUrl: string; + }; + }, + ]; + pageInfo: { + endCursor: string; + hasNextPage: boolean; + }; + }; + }; + }; +}; + +export type GitLabSaasGroupsResponse = { + errors: { message: string }[]; + data: { + group: { + descendantGroups: { + nodes: [ + { + id: string; + name: string; + description: string; + fullPath: string; + parent: { + id: string; + }; + }, + ]; + pageInfo: { + endCursor: string; + hasNextPage: false; + }; + }; + }; + }; +}; + export type GitlabProviderConfig = { host: string; group: string; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 02c036f27a..5fd5094943 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -168,19 +168,24 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { logger: logger, }); - const users = paginated(options => client.listUsers(options), { - page: 1, - per_page: 100, - active: true, - }); + let groups; + let users; - const groups = paginated( - options => client.listGroups(options), - { + if (client.isSelfManaged()) { + groups = paginated(options => client.listGroups(options), { page: 1, per_page: 100, - }, - ); + }); + + users = paginated(options => client.listUsers(options), { + page: 1, + per_page: 100, + active: true, + }); + } else { + groups = (await client.listSaasGroups(this.config.group)).items; + users = (await client.listSaasUsers(this.config.group)).items; + } const idMappedUser: { [userId: number]: GitLabUser } = {}; From d2f13494d84013a1b85e55a7cc494d4c0a03f0f5 Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 14 Jul 2023 16:11:43 +0100 Subject: [PATCH 002/131] Update error handling and documentation Signed-off-by: angom1 --- docs/integrations/gitlab/org.md | 4 +++- .../src/lib/client.ts | 14 +------------- .../providers/GitlabOrgDiscoveryEntityProvider.ts | 6 ++++++ 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index d022641ab6..6c154938ee 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -58,6 +58,8 @@ amount of data, this can take significant time and resources. The token used must have the `read_api` scope, and the Users and Groups fetched will be those visible to the account which provisioned the token. +**NOTE**: If any groups that are being ingested are empty groups and the account which provisioned the token is shared at a higher level group via group sharing and you don't see the expected number of Groups in the catalog you may be hitting this [GitLab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996). + ```yaml catalog: providers: @@ -65,7 +67,7 @@ catalog: yourProviderId: host: gitlab.com orgEnabled: true - group: org/teams # Optional. Must not end with slash. Accepts only groups under the provided path (which will be stripped) + group: org/teams # Required for gitlab.com. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped) groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything ``` diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index d2a3fa01e0..1e5e3ea32b 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -102,12 +102,6 @@ export class GitLabClient { let hasNextPage: boolean = false; let endCursor: string | null = null; - if (!groupPath) { - throw new Error( - `Missing required 'group' value in gitlab provider config.`, - ); - } - do { const response: GitLabSaasUsersResponse = await fetch( `${this.config.baseUrl}/api/graphql`, @@ -181,12 +175,6 @@ export class GitLabClient { let hasNextPage: boolean = false; let endCursor: string | null = null; - if (!groupPath) { - throw new Error( - `Missing required 'group' value in gitlab provider config.`, - ); - } - do { const response: GitLabSaasGroupsResponse = await fetch( `${this.config.baseUrl}/api/graphql`, @@ -226,7 +214,7 @@ export class GitLabClient { if (!response.data.group?.descendantGroups.nodes) { this.logger.warn( - `Could not get groups under ${groupPath}. The provided token might not have sufficient permissions`, + `Couldn't get groups under ${groupPath}. The provided token might not have sufficient permissions`, ); continue; } diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 5fd5094943..94d3e64a02 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -89,6 +89,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { ); } + if (!providerConfig.group && providerConfig.host === 'gitlab.com') { + throw new Error( + `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:${providerConfig.id}.`, + ); + } + if (!options.schedule && !providerConfig.schedule) { throw new Error( `No schedule provided neither via code nor config for GitlabOrgDiscoveryEntityProvider:${providerConfig.id}.`, From 1d2d469d31c76661defc3659c6b2db79b1fffa09 Mon Sep 17 00:00:00 2001 From: angom1 Date: Mon, 17 Jul 2023 12:04:12 +0100 Subject: [PATCH 003/131] Add test when group parameter is not set Signed-off-by: angom1 --- .../GitlabOrgDiscoveryEntityProvider.test.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index e29cdccc04..fea973333a 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -543,6 +543,42 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ); }); + it('fail with scheduler but no group config when host is gitlab.com', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'test-gitlab', + apiBaseUrl: 'https://api.gitlab.example/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + orgEnabled: true, + }, + }, + }, + }, + }); + + expect(() => + GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + `Missing 'group' value for GitlabOrgDiscoveryEntityProvider:test-id`, + ); + }); + it('single simple provider config with schedule in config', async () => { const schedule = new PersistingTaskRunner(); const scheduler = { From a446d9eadcc3e3d52a0a7b805228214fad32e9d2 Mon Sep 17 00:00:00 2001 From: angom1 Date: Mon, 17 Jul 2023 12:27:46 +0100 Subject: [PATCH 004/131] Fix host and url Signed-off-by: angom1 --- .../src/providers/GitlabOrgDiscoveryEntityProvider.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index fea973333a..b1d95b63fa 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -551,8 +551,8 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { integrations: { gitlab: [ { - host: 'test-gitlab', - apiBaseUrl: 'https://api.gitlab.example/api/v4', + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', token: '1234', }, ], From 8f8391b768678a10a6acf300a448ed0b1bb2b771 Mon Sep 17 00:00:00 2001 From: angom1 Date: Wed, 19 Jul 2023 16:43:59 +0100 Subject: [PATCH 005/131] Add test for listSaasUsers function Signed-off-by: angom1 --- .../src/lib/client.test.ts | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index f4504e8937..61376e33b5 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -398,6 +398,149 @@ describe('GitLabClient', () => { ]); }); + describe('listSaasUsers', () => { + it('gets all users under group', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + commitEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasMembers = (await client.listSaasUsers('group1')).items; + + expect(saasMembers.length).toEqual(1); + }); + + it('gets all users with token without full permissions', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.data({ + group: {}, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasMembers = (await client.listSaasUsers('group1')).items; + + expect(saasMembers).toEqual([]); + }); + + it('rejects when GraphQL returns errors', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + await expect(() => client.listSaasUsers('group1')).rejects.toThrow( + 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', + ); + }); + it('traverses multi-page results', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((req, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: req.variables.endCursor + ? [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + commitEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ] + : [ + { + user: { + id: 'gid://gitlab/User/2', + username: 'user2', + commitEmail: 'user2@example.com', + name: 'user2', + state: 'active', + webUrl: 'user2.com', + avatarUrl: 'user2', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasMembers = (await client.listSaasUsers('group1')).items; + + expect(saasMembers.length).toEqual(2); + }); + }); + describe('getGroupMembers', () => { it('gets member IDs', async () => { server.use( From 80a049c361066746458ff5fa95f61f864cd2021e Mon Sep 17 00:00:00 2001 From: angom1 Date: Thu, 20 Jul 2023 10:38:45 +0100 Subject: [PATCH 006/131] fix key names Signed-off-by: angom1 --- .../src/lib/client.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 61376e33b5..c718398582 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -413,11 +413,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - commitEmail: 'user1@example.com', + email: 'user1@example.com', name: 'user1', state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', + web_url: 'user1.com', + avatar_url: 'user1', }, }, ], @@ -499,11 +499,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - commitEmail: 'user1@example.com', + email: 'user1@example.com', name: 'user1', state: 'active', - webUrl: 'user1.com', - avatarUrl: 'user1', + web_url: 'user1.com', + avatar_url: 'user1', }, }, ] @@ -512,11 +512,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/2', username: 'user2', - commitEmail: 'user2@example.com', + email: 'user2@example.com', name: 'user2', state: 'active', - webUrl: 'user2.com', - avatarUrl: 'user2', + web_url: 'user2.com', + avatar_url: 'user2', }, }, ], From 3be6da71a1976820400fc299965caaf1295e2e15 Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 21 Jul 2023 12:47:17 +0100 Subject: [PATCH 007/131] Add tests for listSaasGroups Signed-off-by: angom1 --- .../src/lib/client.test.ts | 155 +++++++++++++++++- .../src/lib/client.ts | 2 +- 2 files changed, 147 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index c718398582..b702889835 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -413,11 +413,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - email: 'user1@example.com', + commitEmail: 'user1@example.com', name: 'user1', state: 'active', - web_url: 'user1.com', - avatar_url: 'user1', + webUrl: 'user1.com', + avatarUrl: 'user1', }, }, ], @@ -499,11 +499,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - email: 'user1@example.com', + commitEmail: 'user1@example.com', name: 'user1', state: 'active', - web_url: 'user1.com', - avatar_url: 'user1', + webUrl: 'user1.com', + avatarUrl: 'user1', }, }, ] @@ -512,11 +512,11 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/2', username: 'user2', - email: 'user2@example.com', + commitEmail: 'user2@example.com', name: 'user2', state: 'active', - web_url: 'user2.com', - avatar_url: 'user2', + webUrl: 'user2.com', + avatarUrl: 'user2', }, }, ], @@ -541,6 +541,143 @@ describe('GitLabClient', () => { }); }); + describe('listSaasGroups', () => { + it('gets all groups under root', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/1', + name: 'group1', + description: 'description1', + fullPath: 'path/user1', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasGroups = (await client.listSaasGroups('group1')).items; + + expect(saasGroups.length).toEqual(1); + }); + + it('gets all descendant groups with token without full permissions', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.data({ + group: {}, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasGroups = (await client.listSaasGroups('group1')).items; + + expect(saasGroups).toEqual([]); + }); + + it('rejects when GraphQL returns errors', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((_, res, ctx) => + res( + ctx.errors([ + { message: 'Unexpected end of document', locations: [] }, + ]), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + await expect(() => client.listSaasGroups('group1')).rejects.toThrow( + 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', + ); + }); + it('traverses multi-page results', async () => { + server.use( + graphql + .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) + .operation((req, res, ctx) => + res( + ctx.data({ + group: { + descendantGroups: { + nodes: req.variables.endCursor + ? [ + { + id: 'gid://gitlab/Group/1', + name: 'group1', + description: 'description1', + fullPath: 'root/group1', + parent: { + id: '123', + }, + }, + ] + : [ + { + id: 'gid://gitlab/Group/2', + name: 'group2', + description: 'description2', + fullPath: 'root/group2', + parent: { + id: '123', + }, + }, + ], + pageInfo: { + endCursor: req.variables.endCursor ? 'end' : 'next', + hasNextPage: !req.variables.endCursor, + }, + }, + }, + }), + ), + ), + ); + const client = new GitLabClient({ + config: MOCK_CONFIG, + logger: getVoidLogger(), + }); + + const saasGroups = (await client.listSaasGroups('root')).items; + + expect(saasGroups.length).toEqual(2); + }); + }); + describe('getGroupMembers', () => { it('gets member IDs', async () => { server.use( diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 1e5e3ea32b..a015cd300a 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -212,7 +212,7 @@ export class GitLabClient { throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`); } - if (!response.data.group?.descendantGroups.nodes) { + if (!response.data.group?.descendantGroups?.nodes) { this.logger.warn( `Couldn't get groups under ${groupPath}. The provided token might not have sufficient permissions`, ); From 79dbbc0307756ab5441d13e99337e7fdac37f57f Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 21 Jul 2023 13:00:58 +0100 Subject: [PATCH 008/131] Fix name in test Signed-off-by: angom1 --- plugins/catalog-backend-module-gitlab/src/lib/client.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index b702889835..14bd51d938 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -556,7 +556,7 @@ describe('GitLabClient', () => { id: 'gid://gitlab/Group/1', name: 'group1', description: 'description1', - fullPath: 'path/user1', + fullPath: 'path/group1', parent: { id: '123', }, From 096e2c909cb6bce4afcaf0f3d04634600c064d1f Mon Sep 17 00:00:00 2001 From: angom1 Date: Wed, 26 Jul 2023 15:27:32 +0100 Subject: [PATCH 009/131] Add draft test Signed-off-by: angom1 --- .../GitlabOrgDiscoveryEntityProvider.test.ts | 312 ++++++++++++++++++ 1 file changed, 312 insertions(+) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index b1d95b63fa..eb070a54ac 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -475,6 +475,318 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { }); }); + // it('apply full update on scheduled execution for gitlab.com', async () => { + // const config = new ConfigReader({ + // integrations: { + // gitlab: [ + // { + // host: 'gitlab.com', + // apiBaseUrl: 'https://gitlab.com/api/v4', + // token: '1234', + // }, + // ], + // }, + // catalog: { + // providers: { + // gitlab: { + // 'test-id': { + // host: 'gitlab.com', + // group: 'test-group', + // orgEnabled: true, + // }, + // }, + // }, + // }, + // }); + // const schedule = new PersistingTaskRunner(); + // const entityProviderConnection: EntityProviderConnection = { + // applyMutation: jest.fn(), + // refresh: jest.fn(), + // }; + // const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + // logger, + // schedule, + // })[0]; + // expect(provider.getProviderName()).toEqual( + // 'GitlabOrgDiscoveryEntityProvider:test-id', + // ); + + // server.use( + // graphql + // .link('https://gitlab.com/api/graphql') + // .operation( + // (_req, res, ctx) => { + // const response = + // { + // id: 123, + // default_branch: 'master', + // archived: false, + // last_activity_at: new Date().toString(), + // web_url: 'https://gitlab.com/test-group/test-repo', + // path_with_namespace: 'test-group/test-repo', + // }; + // return res(ctx.data(response)); + // }, + // ), + // graphql + // .link('https://gitlab.com/api/graphql') + // .operation((_req, res, ctx) => { + // const response = [ + // { + // id: 1, + // username: 'test1', + // name: 'Test Testit', + // state: 'active', + // avatar_url: 'https://secure.gravatar.com/', + // web_url: 'https://gitlab.com/test1', + // created_at: '2023-01-19T07:27:03.333Z', + // bio: '', + // location: null, + // public_email: null, + // skype: '', + // linkedin: '', + // twitter: '', + // website_url: '', + // organization: null, + // job_title: '', + // pronouns: null, + // bot: false, + // work_information: null, + // followers: 0, + // following: 0, + // is_followed: false, + // local_time: null, + // last_sign_in_at: '2023-01-19T07:27:49.601Z', + // confirmed_at: '2023-01-19T07:27:02.905Z', + // last_activity_on: '2023-01-19', + // email: 'test@example.com', + // theme_id: 1, + // color_scheme_id: 1, + // projects_limit: 100000, + // current_sign_in_at: '2023-01-19T09:09:10.676Z', + // identities: [], + // can_create_group: true, + // can_create_project: true, + // two_factor_enabled: false, + // external: false, + // private_profile: false, + // commit_email: 'test@example.com', + // is_admin: false, + // note: '', + // }, + // { + // id: 2, + // username: 'test2', + // name: '', + // state: 'active', + // avatar_url: '', + // web_url: 'https://gitlab.com/test2', + // created_at: '2023-01-19T07:27:03.333Z', + // bio: '', + // location: null, + // public_email: null, + // skype: '', + // linkedin: '', + // twitter: '', + // website_url: '', + // organization: null, + // job_title: '', + // pronouns: null, + // bot: false, + // work_information: null, + // followers: 0, + // following: 0, + // is_followed: false, + // local_time: null, + // last_sign_in_at: '2023-01-19T07:27:49.601Z', + // confirmed_at: '2023-01-19T07:27:02.905Z', + // last_activity_on: '2023-01-19', + // email: 'test@example.com', + // theme_id: 1, + // color_scheme_id: 1, + // projects_limit: 100000, + // current_sign_in_at: '2023-01-19T09:09:10.676Z', + // identities: [], + // can_create_group: true, + // can_create_project: true, + // two_factor_enabled: false, + // external: false, + // private_profile: false, + // commit_email: 'test@example.com', + // is_admin: false, + // note: '', + // }, + // ]; + // return res(ctx.data(response)); + // }), + // graphql + // .link('https://gitlab.com/api/graphql') + // .operation((_req, res, ctx) => { + // const response = [ + // { + // id: 1, + // web_url: 'https://gitlab.com/groups/group1', + // name: 'group1', + // path: 'group1', + // description: '', + // visibility: 'internal', + // share_with_group_lock: false, + // require_two_factor_authentication: false, + // two_factor_grace_period: 48, + // project_creation_level: 'developer', + // auto_devops_enabled: null, + // subgroup_creation_level: 'owner', + // emails_disabled: null, + // mentions_disabled: null, + // lfs_enabled: true, + // default_branch_protection: 2, + // avatar_url: null, + // request_access_enabled: false, + // full_name: '8020', + // full_path: '8020', + // created_at: '2017-06-19T06:42:34.160Z', + // parent_id: null, + // }, + // { + // id: 2, + // web_url: 'https://gitlab.com/groups/group1/group2', + // name: 'group2', + // path: 'group1/group2', + // description: 'Group2', + // visibility: 'internal', + // share_with_group_lock: false, + // require_two_factor_authentication: false, + // two_factor_grace_period: 48, + // project_creation_level: 'developer', + // auto_devops_enabled: null, + // subgroup_creation_level: 'owner', + // emails_disabled: null, + // mentions_disabled: null, + // lfs_enabled: true, + // request_access_enabled: false, + // full_name: 'group2', + // full_path: 'group1/group2', + // created_at: '2017-12-07T13:20:40.675Z', + // parent_id: null, + // }, + // ]; + // return res(ctx.data(response)); + // }), + // graphql + // .link('https://gitlab.com/api/graphql') + // .operation(async (req, res, ctx) => + // res( + // ctx.data({ + // group: { + // groupMembers: { + // nodes: + // req.variables.group === 'group1/group2' + // ? [{ user: { id: 'gid://gitlab/User/1' } }] + // : [], + // pageInfo: { + // endCursor: 'end', + // hasNextPage: false, + // }, + // }, + // }, + // }), + // ), + // ), + // ); + + // await provider.connect(entityProviderConnection); + + // const taskDef = schedule.getTasks()[0]; + // expect(taskDef.id).toEqual( + // 'GitlabOrgDiscoveryEntityProvider:test-id:refresh', + // ); + // await (taskDef.fn as () => Promise)(); + + // const expectedEntities = [ + // { + // entity: { + // apiVersion: 'backstage.io/v1alpha1', + // kind: 'User', + // metadata: { + // annotations: { + // 'backstage.io/managed-by-location': + // 'url:https://gitlab.com/test1', + // 'backstage.io/managed-by-origin-location': + // 'url:https://gitlab.com/test1', + // 'gitlab.com/user-login': 'https://gitlab.com/test1', + // }, + // name: 'test1', + // }, + // spec: { + // memberOf: ['group1-group2'], + // profile: { + // displayName: 'Test Testit', + // email: 'test@example.com', + // picture: 'https://secure.gravatar.com/', + // }, + // }, + // }, + // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + // }, + // { + // entity: { + // apiVersion: 'backstage.io/v1alpha1', + // kind: 'User', + // metadata: { + // annotations: { + // 'backstage.io/managed-by-location': + // 'url:https://gitlab.com/test2', + // 'backstage.io/managed-by-origin-location': + // 'url:https://gitlab.com/test2', + // 'gitlab.com/user-login': 'https://gitlab.com/test2', + // }, + // name: 'test2', + // }, + // spec: { + // memberOf: [], + // profile: { + // displayName: undefined, + // email: 'test@example.com', + // picture: undefined, + // }, + // }, + // }, + // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + // }, + // { + // entity: { + // apiVersion: 'backstage.io/v1alpha1', + // kind: 'Group', + // metadata: { + // annotations: { + // 'backstage.io/managed-by-location': + // 'url:https://gitlab.com/group1/group2', + // 'backstage.io/managed-by-origin-location': + // 'url:https://gitlab.com/group1/group2', + // 'gitlab.com/team-path': 'group1/group2', + // }, + // description: 'Group2', + // name: 'group1-group2', + // }, + // spec: { + // children: [], + // profile: { + // displayName: 'group2', + // }, + // type: 'team', + // }, + // }, + // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + // }, + // ]; + + // expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + // expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + // type: 'full', + // entities: expectedEntities, + // }); + // }); + it('fail without schedule and scheduler', () => { const config = new ConfigReader({ integrations: { From f0b82fcf2603bd8689f0e3fb30b8aa5641160ce0 Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 28 Jul 2023 12:32:23 +0100 Subject: [PATCH 010/131] Update draft test Signed-off-by: angom1 --- .../src/lib/client.ts | 6 +- .../GitlabOrgDiscoveryEntityProvider.test.ts | 492 +++++++----------- 2 files changed, 189 insertions(+), 309 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index a015cd300a..e37d7f27dc 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -113,7 +113,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, endCursor }, - query: `query($group: ID!, $endCursor: String) { + query: `query listSaasUsers($group: ID!, $endCursor: String) { group(fullPath: $group) { groupMembers(first: 100, relations: [DESCENDANTS], after: $endCursor) { nodes { @@ -186,7 +186,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, endCursor }, - query: `query($group: ID!, $endCursor: String) { + query: `query listSaasGroups($group: ID!, $endCursor: String) { group(fullPath: $group) { descendantGroups(first: 100, after: $endCursor){ nodes{ @@ -257,7 +257,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, endCursor }, - query: `query($group: ID!, $endCursor: String) { + query: `query getGroupMembers($group: ID!, $endCursor: String) { group(fullPath: $group) { groupMembers(first: 100, relations: [DIRECT], after: $endCursor) { nodes { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index eb070a54ac..b6c62be5a2 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -475,317 +475,197 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { }); }); - // it('apply full update on scheduled execution for gitlab.com', async () => { - // const config = new ConfigReader({ - // integrations: { - // gitlab: [ - // { - // host: 'gitlab.com', - // apiBaseUrl: 'https://gitlab.com/api/v4', - // token: '1234', - // }, - // ], - // }, - // catalog: { - // providers: { - // gitlab: { - // 'test-id': { - // host: 'gitlab.com', - // group: 'test-group', - // orgEnabled: true, - // }, - // }, - // }, - // }, - // }); - // const schedule = new PersistingTaskRunner(); - // const entityProviderConnection: EntityProviderConnection = { - // applyMutation: jest.fn(), - // refresh: jest.fn(), - // }; - // const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { - // logger, - // schedule, - // })[0]; - // expect(provider.getProviderName()).toEqual( - // 'GitlabOrgDiscoveryEntityProvider:test-id', - // ); + it('apply full update on scheduled execution for gitlab.com', async () => { + const config = new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + token: '1234', + }, + ], + }, + catalog: { + providers: { + gitlab: { + 'test-id': { + host: 'gitlab.com', + group: 'group1', + orgEnabled: true, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const provider = GitlabOrgDiscoveryEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id', + ); - // server.use( - // graphql - // .link('https://gitlab.com/api/graphql') - // .operation( - // (_req, res, ctx) => { - // const response = - // { - // id: 123, - // default_branch: 'master', - // archived: false, - // last_activity_at: new Date().toString(), - // web_url: 'https://gitlab.com/test-group/test-repo', - // path_with_namespace: 'test-group/test-repo', - // }; - // return res(ctx.data(response)); - // }, - // ), - // graphql - // .link('https://gitlab.com/api/graphql') - // .operation((_req, res, ctx) => { - // const response = [ - // { - // id: 1, - // username: 'test1', - // name: 'Test Testit', - // state: 'active', - // avatar_url: 'https://secure.gravatar.com/', - // web_url: 'https://gitlab.com/test1', - // created_at: '2023-01-19T07:27:03.333Z', - // bio: '', - // location: null, - // public_email: null, - // skype: '', - // linkedin: '', - // twitter: '', - // website_url: '', - // organization: null, - // job_title: '', - // pronouns: null, - // bot: false, - // work_information: null, - // followers: 0, - // following: 0, - // is_followed: false, - // local_time: null, - // last_sign_in_at: '2023-01-19T07:27:49.601Z', - // confirmed_at: '2023-01-19T07:27:02.905Z', - // last_activity_on: '2023-01-19', - // email: 'test@example.com', - // theme_id: 1, - // color_scheme_id: 1, - // projects_limit: 100000, - // current_sign_in_at: '2023-01-19T09:09:10.676Z', - // identities: [], - // can_create_group: true, - // can_create_project: true, - // two_factor_enabled: false, - // external: false, - // private_profile: false, - // commit_email: 'test@example.com', - // is_admin: false, - // note: '', - // }, - // { - // id: 2, - // username: 'test2', - // name: '', - // state: 'active', - // avatar_url: '', - // web_url: 'https://gitlab.com/test2', - // created_at: '2023-01-19T07:27:03.333Z', - // bio: '', - // location: null, - // public_email: null, - // skype: '', - // linkedin: '', - // twitter: '', - // website_url: '', - // organization: null, - // job_title: '', - // pronouns: null, - // bot: false, - // work_information: null, - // followers: 0, - // following: 0, - // is_followed: false, - // local_time: null, - // last_sign_in_at: '2023-01-19T07:27:49.601Z', - // confirmed_at: '2023-01-19T07:27:02.905Z', - // last_activity_on: '2023-01-19', - // email: 'test@example.com', - // theme_id: 1, - // color_scheme_id: 1, - // projects_limit: 100000, - // current_sign_in_at: '2023-01-19T09:09:10.676Z', - // identities: [], - // can_create_group: true, - // can_create_project: true, - // two_factor_enabled: false, - // external: false, - // private_profile: false, - // commit_email: 'test@example.com', - // is_admin: false, - // note: '', - // }, - // ]; - // return res(ctx.data(response)); - // }), - // graphql - // .link('https://gitlab.com/api/graphql') - // .operation((_req, res, ctx) => { - // const response = [ - // { - // id: 1, - // web_url: 'https://gitlab.com/groups/group1', - // name: 'group1', - // path: 'group1', - // description: '', - // visibility: 'internal', - // share_with_group_lock: false, - // require_two_factor_authentication: false, - // two_factor_grace_period: 48, - // project_creation_level: 'developer', - // auto_devops_enabled: null, - // subgroup_creation_level: 'owner', - // emails_disabled: null, - // mentions_disabled: null, - // lfs_enabled: true, - // default_branch_protection: 2, - // avatar_url: null, - // request_access_enabled: false, - // full_name: '8020', - // full_path: '8020', - // created_at: '2017-06-19T06:42:34.160Z', - // parent_id: null, - // }, - // { - // id: 2, - // web_url: 'https://gitlab.com/groups/group1/group2', - // name: 'group2', - // path: 'group1/group2', - // description: 'Group2', - // visibility: 'internal', - // share_with_group_lock: false, - // require_two_factor_authentication: false, - // two_factor_grace_period: 48, - // project_creation_level: 'developer', - // auto_devops_enabled: null, - // subgroup_creation_level: 'owner', - // emails_disabled: null, - // mentions_disabled: null, - // lfs_enabled: true, - // request_access_enabled: false, - // full_name: 'group2', - // full_path: 'group1/group2', - // created_at: '2017-12-07T13:20:40.675Z', - // parent_id: null, - // }, - // ]; - // return res(ctx.data(response)); - // }), - // graphql - // .link('https://gitlab.com/api/graphql') - // .operation(async (req, res, ctx) => - // res( - // ctx.data({ - // group: { - // groupMembers: { - // nodes: - // req.variables.group === 'group1/group2' - // ? [{ user: { id: 'gid://gitlab/User/1' } }] - // : [], - // pageInfo: { - // endCursor: 'end', - // hasNextPage: false, - // }, - // }, - // }, - // }), - // ), - // ), - // ); + server.use( + graphql + .link('https://gitlab.com/api/graphql') + .query('listSaasGroups', async (_, res, ctx) => + res( + ctx.data({ + group: { + descendantGroups: { + nodes: [ + { + id: 'gid://gitlab/Group/456', + name: 'group2', + description: 'Group2', + fullPath: 'group1/group2', + parent: { + id: 'gid://gitlab/Group/123', + }, + }, + { + id: 'gid://gitlab/Group/789', + name: 'group3', + description: 'Group3', + fullPath: 'group1/group3', + parent: { + id: 'gid://gitlab/Group/123', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + graphql + .link('https://gitlab.com/api/graphql') + .query('listSaasUsers', async (_, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: [ + { + user: { + id: 'gid://gitlab/User/1234', + username: 'testuser', + commitEmail: 'testuser@example.com', + state: 'active', + name: 'Test User', + webUrl: 'https://gitlab.com/testuser', + avatarUrl: 'https://secure.gravatar.com/', + }, + }, + ], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + graphql + .link('https://gitlab.com/api/graphql') + .query('getGroupMembers', async (req, res, ctx) => + res( + ctx.data({ + group: { + groupMembers: { + nodes: + req.variables.group === 'group1/group2' + ? [{ user: { id: 'gid://gitlab/User/1234' } }] + : [], + pageInfo: { + endCursor: 'end', + hasNextPage: false, + }, + }, + }, + }), + ), + ), + ); - // await provider.connect(entityProviderConnection); + await provider.connect(entityProviderConnection); - // const taskDef = schedule.getTasks()[0]; - // expect(taskDef.id).toEqual( - // 'GitlabOrgDiscoveryEntityProvider:test-id:refresh', - // ); - // await (taskDef.fn as () => Promise)(); + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual( + 'GitlabOrgDiscoveryEntityProvider:test-id:refresh', + ); + await (taskDef.fn as () => Promise)(); - // const expectedEntities = [ - // { - // entity: { - // apiVersion: 'backstage.io/v1alpha1', - // kind: 'User', - // metadata: { - // annotations: { - // 'backstage.io/managed-by-location': - // 'url:https://gitlab.com/test1', - // 'backstage.io/managed-by-origin-location': - // 'url:https://gitlab.com/test1', - // 'gitlab.com/user-login': 'https://gitlab.com/test1', - // }, - // name: 'test1', - // }, - // spec: { - // memberOf: ['group1-group2'], - // profile: { - // displayName: 'Test Testit', - // email: 'test@example.com', - // picture: 'https://secure.gravatar.com/', - // }, - // }, - // }, - // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - // }, - // { - // entity: { - // apiVersion: 'backstage.io/v1alpha1', - // kind: 'User', - // metadata: { - // annotations: { - // 'backstage.io/managed-by-location': - // 'url:https://gitlab.com/test2', - // 'backstage.io/managed-by-origin-location': - // 'url:https://gitlab.com/test2', - // 'gitlab.com/user-login': 'https://gitlab.com/test2', - // }, - // name: 'test2', - // }, - // spec: { - // memberOf: [], - // profile: { - // displayName: undefined, - // email: 'test@example.com', - // picture: undefined, - // }, - // }, - // }, - // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - // }, - // { - // entity: { - // apiVersion: 'backstage.io/v1alpha1', - // kind: 'Group', - // metadata: { - // annotations: { - // 'backstage.io/managed-by-location': - // 'url:https://gitlab.com/group1/group2', - // 'backstage.io/managed-by-origin-location': - // 'url:https://gitlab.com/group1/group2', - // 'gitlab.com/team-path': 'group1/group2', - // }, - // description: 'Group2', - // name: 'group1-group2', - // }, - // spec: { - // children: [], - // profile: { - // displayName: 'group2', - // }, - // type: 'team', - // }, - // }, - // locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', - // }, - // ]; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser', + 'gitlab.com/user-login': 'https://gitlab.com/testuser', + }, + name: 'testuser', + }, + spec: { + memberOf: ['group2'], + profile: { + displayName: 'Test User', + email: 'testuser@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/group1/group2', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/group1/group2', + 'gitlab.com/team-path': 'group1/group2', + }, + description: 'Group2', + name: 'group2', + }, + spec: { + children: [], + profile: { + displayName: 'group2', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + ]; - // expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); - // expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - // type: 'full', - // entities: expectedEntities, - // }); - // }); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); it('fail without schedule and scheduler', () => { const config = new ConfigReader({ From f85f6730a61e36d921d453fe6b9da3a414145cfd Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 28 Jul 2023 14:34:58 +0100 Subject: [PATCH 011/131] Update client tests Signed-off-by: angom1 --- .../src/lib/client.test.ts | 91 ++++++++++++++++--- 1 file changed, 77 insertions(+), 14 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 14bd51d938..b9fdf905ee 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -403,7 +403,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasUsers', async (_, res, ctx) => res( ctx.data({ group: { @@ -437,15 +437,27 @@ describe('GitLabClient', () => { }); const saasMembers = (await client.listSaasUsers('group1')).items; + const expectedSaasMember = [ + { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }, + ]; expect(saasMembers.length).toEqual(1); + expect(saasMembers).toEqual(expectedSaasMember); }); it('gets all users with token without full permissions', async () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasUsers', async (_, res, ctx) => res( ctx.data({ group: {}, @@ -467,7 +479,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasUsers', async (_, res, ctx) => res( ctx.errors([ { message: 'Unexpected end of document', locations: [] }, @@ -488,7 +500,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((req, res, ctx) => + .query('listSaasUsers', async (req, res, ctx) => res( ctx.data({ group: { @@ -537,7 +549,29 @@ describe('GitLabClient', () => { const saasMembers = (await client.listSaasUsers('group1')).items; + const expectedSaasMember1 = { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }; + + const expectedSaasMember2 = { + id: 2, + username: 'user2', + email: 'user2@example.com', + name: 'user2', + state: 'active', + web_url: 'user2.com', + avatar_url: 'user2', + }; + expect(saasMembers.length).toEqual(2); + expect(saasMembers[0]).toEqual(expectedSaasMember2); + expect(saasMembers[1]).toEqual(expectedSaasMember1); }); }); @@ -546,7 +580,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasGroups', async (_, res, ctx) => res( ctx.data({ group: { @@ -579,14 +613,25 @@ describe('GitLabClient', () => { const saasGroups = (await client.listSaasGroups('group1')).items; + const expectedSaasGroup = [ + { + id: 1, + name: 'group1', + description: 'description1', + full_path: 'path/group1', + parent_id: 123, + }, + ]; + expect(saasGroups.length).toEqual(1); + expect(saasGroups).toEqual(expectedSaasGroup); }); it('gets all descendant groups with token without full permissions', async () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasGroups', async (_, res, ctx) => res( ctx.data({ group: {}, @@ -608,7 +653,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('listSaasGroups', async (_, res, ctx) => res( ctx.errors([ { message: 'Unexpected end of document', locations: [] }, @@ -629,7 +674,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((req, res, ctx) => + .query('listSaasGroups', async (req, res, ctx) => res( ctx.data({ group: { @@ -640,7 +685,7 @@ describe('GitLabClient', () => { id: 'gid://gitlab/Group/1', name: 'group1', description: 'description1', - fullPath: 'root/group1', + fullPath: 'path/group1', parent: { id: '123', }, @@ -651,7 +696,7 @@ describe('GitLabClient', () => { id: 'gid://gitlab/Group/2', name: 'group2', description: 'description2', - fullPath: 'root/group2', + fullPath: 'path/group2', parent: { id: '123', }, @@ -674,7 +719,25 @@ describe('GitLabClient', () => { const saasGroups = (await client.listSaasGroups('root')).items; + const expectedSaasGroup1 = { + id: 1, + name: 'group1', + description: 'description1', + full_path: 'path/group1', + parent_id: 123, + }; + + const expectedSaasGroup2 = { + id: 2, + name: 'group2', + description: 'description2', + full_path: 'path/group2', + parent_id: 123, + }; + expect(saasGroups.length).toEqual(2); + expect(saasGroups[0]).toEqual(expectedSaasGroup2); + expect(saasGroups[1]).toEqual(expectedSaasGroup1); }); }); @@ -683,7 +746,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: { @@ -713,7 +776,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: {}, @@ -735,7 +798,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.errors([ { message: 'Unexpected end of document', locations: [] }, @@ -757,7 +820,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .operation((req, res, ctx) => + .query('getGroupMembers', async (req, res, ctx) => res( ctx.data({ group: { From 0b569236eb2e2f82664dd5e7d6f482735c9f8ae8 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 1 Aug 2023 10:37:31 +0100 Subject: [PATCH 012/131] Switch from commitEmail to publicEmail for email attribute Signed-off-by: Stephen Barry --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 4 ++-- plugins/catalog-backend-module-gitlab/src/lib/types.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index e37d7f27dc..fa4ccf9f97 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -120,7 +120,7 @@ export class GitLabClient { user { id username - commitEmail + publicEmail name state webUrl @@ -156,7 +156,7 @@ export class GitLabClient { const formattedUserResponse = { id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), username: userItem.user.username, - email: userItem.user.commitEmail, + email: userItem.user.publicEmail, name: userItem.user.name, state: userItem.user.state, web_url: userItem.user.webUrl, diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index a00c66b4a7..651fbb00a6 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -80,7 +80,7 @@ export type GitLabSaasUsersResponse = { user: { id: string; username: string; - commitEmail: string; + publicEmail: string; name: string; state: string; webUrl: string; From 74f7fa6dce4a6d3378df9a7ba900de16f79fe61b Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 1 Aug 2023 10:38:59 +0100 Subject: [PATCH 013/131] Update tests after switch to publicEmail and add additional test data to SaaS full mutation test Signed-off-by: Stephen Barry --- .../src/lib/client.test.ts | 6 +- .../GitlabOrgDiscoveryEntityProvider.test.ts | 87 ++++++++++++++++--- 2 files changed, 77 insertions(+), 16 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index b9fdf905ee..a62a3874d5 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -413,7 +413,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - commitEmail: 'user1@example.com', + publicEmail: 'user1@example.com', name: 'user1', state: 'active', webUrl: 'user1.com', @@ -511,7 +511,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - commitEmail: 'user1@example.com', + publicEmail: 'user1@example.com', name: 'user1', state: 'active', webUrl: 'user1.com', @@ -524,7 +524,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/2', username: 'user2', - commitEmail: 'user2@example.com', + publicEmail: 'user2@example.com', name: 'user2', state: 'active', webUrl: 'user2.com', diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index b6c62be5a2..7339dd4a29 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -558,12 +558,23 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { nodes: [ { user: { - id: 'gid://gitlab/User/1234', - username: 'testuser', - commitEmail: 'testuser@example.com', + id: 'gid://gitlab/User/12', + username: 'testuser1', + publicEmail: 'testuser1@example.com', state: 'active', - name: 'Test User', - webUrl: 'https://gitlab.com/testuser', + name: 'Test User 1', + webUrl: 'https://gitlab.com/testuser1', + avatarUrl: 'https://secure.gravatar.com/', + }, + }, + { + user: { + id: 'gid://gitlab/User/34', + username: 'testuser2', + publicEmail: 'testuser2@example.com', + state: 'active', + name: 'Test User 2', + webUrl: 'https://gitlab.com/testuser2', avatarUrl: 'https://secure.gravatar.com/', }, }, @@ -586,8 +597,8 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { groupMembers: { nodes: req.variables.group === 'group1/group2' - ? [{ user: { id: 'gid://gitlab/User/1234' } }] - : [], + ? [{ user: { id: 'gid://gitlab/User/12' } }] + : [{ user: { id: 'gid://gitlab/User/34' } }], pageInfo: { endCursor: 'end', hasNextPage: false, @@ -615,18 +626,43 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://gitlab.com/testuser', + 'url:https://gitlab.com/testuser1', 'backstage.io/managed-by-origin-location': - 'url:https://gitlab.com/testuser', - 'gitlab.com/user-login': 'https://gitlab.com/testuser', + 'url:https://gitlab.com/testuser1', + 'gitlab.com/user-login': 'https://gitlab.com/testuser1', }, - name: 'testuser', + name: 'testuser1', }, spec: { memberOf: ['group2'], profile: { - displayName: 'Test User', - email: 'testuser@example.com', + displayName: 'Test User 1', + email: 'testuser1@example.com', + picture: 'https://secure.gravatar.com/', + }, + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/testuser2', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/testuser2', + 'gitlab.com/user-login': 'https://gitlab.com/testuser2', + }, + name: 'testuser2', + }, + spec: { + memberOf: ['group3'], + profile: { + displayName: 'Test User 2', + email: 'testuser2@example.com', picture: 'https://secure.gravatar.com/', }, }, @@ -658,6 +694,31 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { }, locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://gitlab.com/group1/group3', + 'backstage.io/managed-by-origin-location': + 'url:https://gitlab.com/group1/group3', + 'gitlab.com/team-path': 'group1/group3', + }, + description: 'Group3', + name: 'group3', + }, + spec: { + children: [], + profile: { + displayName: 'group3', + }, + type: 'team', + }, + }, + locationKey: 'GitlabOrgDiscoveryEntityProvider:test-id', + }, ]; expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); From 017e9e22eef53b0ca7dc36781e5954d3b9c6071e Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 1 Aug 2023 10:39:41 +0100 Subject: [PATCH 014/131] Fetch users from root SaaS group Signed-off-by: Stephen Barry --- .../src/providers/GitlabOrgDiscoveryEntityProvider.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 94d3e64a02..2b614d69c9 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -190,7 +190,8 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); } else { groups = (await client.listSaasGroups(this.config.group)).items; - users = (await client.listSaasUsers(this.config.group)).items; + users = (await client.listSaasUsers(this.config.group.split('/')[0])) + .items; } const idMappedUser: { [userId: number]: GitLabUser } = {}; From 86e2f78a12a90a4500e9e88c36ae635f16a3b23d Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 1 Aug 2023 11:32:06 +0100 Subject: [PATCH 015/131] Update Gitlab Org Data doc Signed-off-by: Stephen Barry --- docs/integrations/gitlab/org.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index 6c154938ee..7090dab26e 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -58,7 +58,7 @@ amount of data, this can take significant time and resources. The token used must have the `read_api` scope, and the Users and Groups fetched will be those visible to the account which provisioned the token. -**NOTE**: If any groups that are being ingested are empty groups and the account which provisioned the token is shared at a higher level group via group sharing and you don't see the expected number of Groups in the catalog you may be hitting this [GitLab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996). +**NOTE**: If any groups that are being ingested are empty groups and the user which provisioned the token is shared with a higher level group via [group sharing](https://docs.gitlab.com/ee/user/group/manage.html#share-a-group-with-another-group) and you don't see the expected number of `Group` entities in the catalog you may be hitting this [Gitlab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996). ```yaml catalog: From 3d73bafd85c9423948e7c8fca6a86dc37ed78019 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 1 Aug 2023 11:42:52 +0100 Subject: [PATCH 016/131] Add changeset Signed-off-by: Stephen Barry --- .changeset/thick-pets-drive.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .changeset/thick-pets-drive.md diff --git a/.changeset/thick-pets-drive.md b/.changeset/thick-pets-drive.md new file mode 100644 index 0000000000..a4d8a35029 --- /dev/null +++ b/.changeset/thick-pets-drive.md @@ -0,0 +1,17 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Fix Gitlab.com user ingestion by scoping GitlabOrgDiscoveryEntityProvider to a group. + +**BREAKING** The `group` parameter is now required Gitlab.com Org Data integrations and the backend will fail to start without this option configured. + +```diff +catalog: + providers: + gitlab: + yourProviderId: + host: gitlab.com + orgEnabled: true ++ group: org/teams +``` From 49c3657ac6d4176e315c6c596c3000ee875be1ed Mon Sep 17 00:00:00 2001 From: Ilya Katlinski Date: Tue, 8 Aug 2023 08:59:15 +0200 Subject: [PATCH 017/131] feat: allow to override vault secret engine in entity level Signed-off-by: Ilya Katlinski --- plugins/vault-backend/README.md | 17 ++++++++++++- .../vault-backend/src/service/VaultBuilder.ts | 7 +++++- plugins/vault-backend/src/service/vaultApi.ts | 25 +++++++++++++------ plugins/vault/README.md | 18 ++++++++++++- plugins/vault/src/api.ts | 18 ++++++++++--- .../EntityVaultTable/EntityVaultTable.tsx | 16 ++++++++---- plugins/vault/src/constants.ts | 1 + 7 files changed, 84 insertions(+), 18 deletions(-) diff --git a/plugins/vault-backend/README.md b/plugins/vault-backend/README.md index c1b49f623a..753281c1dc 100644 --- a/plugins/vault-backend/README.md +++ b/plugins/vault-backend/README.md @@ -66,7 +66,7 @@ To get started, first you need a running instance of Vault. You can follow [this baseUrl: http://your-internal-vault-url.svc publicUrl: https://your-vault-url.example.com token: - secretEngine: 'customSecretEngine' # Optional. By default it uses 'secrets' + secretEngine: 'customSecretEngine' # Optional. By default it uses 'secrets'. Can be overwritten by the annotation of the entity kvVersion: # Optional. The K/V version that your instance is using. The available options are '1' or '2' ``` @@ -106,6 +106,20 @@ The path is relative to your secrets engine folder. So if you want to get the se You will set the `vault.io/secret-path` to `test/backstage`. If the folder `backstage` contains other sub-folders, the plugin will fetch the secrets inside them and adapt the **View** and **Edit** URLs to point to the correct place. +In case you need to support different secret engines for entities of the catalog you can proivde optional annotion to the entity in `catalog-info.yaml`: + +```diff + apiVersion: backstage.io/v1alpha1 + kind: Component + metadata: + # ... + annotations: + vault.io/secrets-path: path/to/secrets ++ vault.io/secret-engine: customSecretEngine # Optional. By default it uses 'secertEngine' value from configuration. +``` + +That will overwrite the default secret engine from the configuration. + ## Renew token In a secure Vault instance, it's usual that the tokens are refreshed after some time. In order to always have a valid token to fetch the secrets, it might be necessary to execute a renew action after some time. By default this is deactivated, but it can be easily activated and configured to be executed periodically (hourly by default, but customizable by the user). In order to do that, modify your `src/plugins/vault.ts` file to look like this one: @@ -138,6 +152,7 @@ export default async function createPlugin( ## Features - List the secrets present in a certain path +- Use different secret engines for different components - Open a link to view the secret - Open a link to edit the secret - Renew the token automatically with a defined periodicity diff --git a/plugins/vault-backend/src/service/VaultBuilder.ts b/plugins/vault-backend/src/service/VaultBuilder.ts index a61d99ac70..ec118d7648 100644 --- a/plugins/vault-backend/src/service/VaultBuilder.ts +++ b/plugins/vault-backend/src/service/VaultBuilder.ts @@ -140,11 +140,16 @@ export class VaultBuilder { router.get('/v1/secrets/:path', async (req, res) => { const { path } = req.params; + const { engine } = req.query; + if (typeof path !== 'string') { throw new InputError(`Invalid path: ${path}`); } + if (engine && typeof engine !== 'string') { + throw new InputError(`Invalid engine: ${engine}`); + } - const secrets = await vaultApi.listSecrets(path); + const secrets = await vaultApi.listSecrets(path, engine); res.json({ items: secrets }); }); diff --git a/plugins/vault-backend/src/service/vaultApi.ts b/plugins/vault-backend/src/service/vaultApi.ts index b75ea76368..adcaf07d56 100644 --- a/plugins/vault-backend/src/service/vaultApi.ts +++ b/plugins/vault-backend/src/service/vaultApi.ts @@ -59,11 +59,17 @@ export interface VaultApi { * Returns the URL to access the Vault UI with the defined config. */ getFrontendSecretsUrl(): string; + /** * Returns a list of secrets used to show in a table. * @param secretPath - The path where the secrets are stored in Vault + * @param secretMount - The mount point of the secrets engine, optional, overrides default secret engine */ - listSecrets(secretPath: string): Promise; + listSecrets( + secretPath: string, + secretMount?: string | undefined, + ): Promise; + /** * Optional, to renew the token used to list the secrets. Throws an * error if the token renewal went wrong. @@ -115,11 +121,16 @@ export class VaultClient implements VaultApi { return `${this.vaultConfig.baseUrl}/ui/vault/secrets/${this.vaultConfig.secretEngine}`; } - async listSecrets(secretPath: string): Promise { + async listSecrets( + secretPath: string, + secretEngine?: string | undefined, + ): Promise { + const mount = secretEngine || this.vaultConfig.secretEngine; + const listUrl = this.vaultConfig.kvVersion === 2 - ? `v1/${this.vaultConfig.secretEngine}/metadata/${secretPath}` - : `v1/${this.vaultConfig.secretEngine}/${secretPath}`; + ? `v1/${mount}/metadata/${secretPath}` + : `v1/${mount}/${secretPath}`; const result = await this.limit(() => this.callApi(listUrl, { list: true }), ); @@ -131,7 +142,7 @@ export class VaultClient implements VaultApi { if (secret.endsWith('/')) { secrets.push( ...(await this.limit(() => - this.listSecrets(`${secretPath}/${secret.slice(0, -1)}`), + this.listSecrets(`${secretPath}/${secret.slice(0, -1)}`, mount), )), ); } else { @@ -140,8 +151,8 @@ export class VaultClient implements VaultApi { secrets.push({ name: secret, path: secretPath, - editUrl: `${vaultUrl}/ui/vault/secrets/${this.vaultConfig.secretEngine}/edit/${secretPath}/${secret}`, - showUrl: `${vaultUrl}/ui/vault/secrets/${this.vaultConfig.secretEngine}/show/${secretPath}/${secret}`, + editUrl: `${vaultUrl}/ui/vault/secrets/${mount}/edit/${secretPath}/${secret}`, + showUrl: `${vaultUrl}/ui/vault/secrets/${mount}/show/${secretPath}/${secret}`, }); } }), diff --git a/plugins/vault/README.md b/plugins/vault/README.md index 824293632f..d128d15dd9 100644 --- a/plugins/vault/README.md +++ b/plugins/vault/README.md @@ -42,7 +42,7 @@ To get started, first you need a running instance of Vault. You can follow [this vault: baseUrl: http://your-vault-url token: - secretEngine: 'customSecretEngine' # Optional. By default it uses 'secrets' + secretEngine: 'customSecretEngine' # Optional. By default it uses 'secrets'. Can be overwritten by the annotation of the entity kvVersion: # Optional. The K/V version that your instance is using. The available options are '1' or '2' ``` @@ -67,6 +67,7 @@ metadata: # ... annotations: vault.io/secrets-path: path/to/secrets + vault.io/secret-engine: customSecretEngine # Optional. By default it uses 'secertEngine' value from configuration. ``` The path is relative to your secrets engine folder. So if you want to get the secrets for backstage and you have the following directory structure: @@ -86,9 +87,24 @@ If the annotation is missing for a certain component, then the card will show so ![Screenshot of the vault plugin with missing annotation](images/annotation-missing.png) +In case you need to support different secret engines for entities of the catalog you can proivde optional annotion to the entity in `catalog-info.yaml`: + +```diff + apiVersion: backstage.io/v1alpha1 + kind: Component + metadata: + # ... + annotations: + vault.io/secrets-path: path/to/secrets ++ vault.io/secret-engine: customSecretEngine # Optional. By default it uses 'secertEngine' value from configuration. +``` + +That will overwrite the default secret engine from the configuration. + ## Features - List the secrets present in a certain path +- Use different secret engines for different components - Open a link to view the secret - Open a link to edit the secret - Renew the token automatically with a defined periodicity diff --git a/plugins/vault/src/api.ts b/plugins/vault/src/api.ts index d924336a2e..fa6a65a77d 100644 --- a/plugins/vault/src/api.ts +++ b/plugins/vault/src/api.ts @@ -46,8 +46,12 @@ export interface VaultApi { /** * Returns a list of secrets used to show in a table. * @param secretPath - The path where the secrets are stored in Vault + * @param secretMount - The mount point of the secrets engine, optional, overrides default secret engine */ - listSecrets(secretPath: string): Promise; + listSecrets( + secretPath: string, + secretMount?: string | undefined, + ): Promise; } /** @@ -90,10 +94,18 @@ export class VaultClient implements VaultApi { throw await ResponseError.fromResponse(response); } - async listSecrets(secretPath: string): Promise { + async listSecrets( + secretPath: string, + secretMount?: string | undefined, + ): Promise { + const query: { [key in string]: any } = {}; + if (secretMount) { + query.engine = secretMount; + } + const result = await this.callApi<{ items: VaultSecret[] }>( `v1/secrets/${encodeURIComponent(secretPath)}`, - {}, + query, ); return result.items; } diff --git a/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.tsx b/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.tsx index 9191d54b0b..c7fad01e0c 100644 --- a/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.tsx +++ b/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.tsx @@ -23,27 +23,33 @@ import Visibility from '@material-ui/icons/Visibility'; import Alert from '@material-ui/lab/Alert'; import useAsync from 'react-use/lib/useAsync'; import { VaultSecret, vaultApiRef } from '../../api'; -import { VAULT_SECRET_PATH_ANNOTATION } from '../../constants'; +import { + VAULT_SECRET_ENGINE_ANNOTATION, + VAULT_SECRET_PATH_ANNOTATION, +} from '../../constants'; -export const vaultSecretPath = (entity: Entity) => { +export const vaultSecretConfig = (entity: Entity) => { const secretPath = entity.metadata.annotations?.[VAULT_SECRET_PATH_ANNOTATION]; + const secretEngine = + entity.metadata.annotations?.[VAULT_SECRET_ENGINE_ANNOTATION]; - return { secretPath }; + return { secretPath, secretEngine }; }; export const EntityVaultTable = ({ entity }: { entity: Entity }) => { const vaultApi = useApi(vaultApiRef); - const { secretPath } = vaultSecretPath(entity); + const { secretPath, secretEngine } = vaultSecretConfig(entity); if (!secretPath) { throw Error( `The secret path is undefined. Please, define the annotation ${VAULT_SECRET_PATH_ANNOTATION}`, ); } + const { value, loading, error } = useAsync(async (): Promise< VaultSecret[] > => { - return vaultApi.listSecrets(secretPath); + return vaultApi.listSecrets(secretPath, secretEngine); }, []); const columns: TableColumn[] = [ diff --git a/plugins/vault/src/constants.ts b/plugins/vault/src/constants.ts index 0d99c261d4..7398bc59ee 100644 --- a/plugins/vault/src/constants.ts +++ b/plugins/vault/src/constants.ts @@ -17,4 +17,5 @@ /** * @public */ +export const VAULT_SECRET_ENGINE_ANNOTATION = 'vault.io/secrets-engine'; export const VAULT_SECRET_PATH_ANNOTATION = 'vault.io/secrets-path'; From 858a18800870dd44f16b8206a5c2c7a60d8980de Mon Sep 17 00:00:00 2001 From: Ilya Katlinski Date: Tue, 8 Aug 2023 09:06:03 +0200 Subject: [PATCH 018/131] chore: add changeset record for vault plugin changes Signed-off-by: Ilya Katlinski --- .changeset/silent-numbers-retire.md | 6 ++++++ plugins/vault-backend/README.md | 2 +- plugins/vault/README.md | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changeset/silent-numbers-retire.md diff --git a/.changeset/silent-numbers-retire.md b/.changeset/silent-numbers-retire.md new file mode 100644 index 0000000000..6f260c05ef --- /dev/null +++ b/.changeset/silent-numbers-retire.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-vault-backend': minor +'@backstage/plugin-vault': minor +--- + +Added ability to override vault secretEngine value on catalog entity level using annotation `vault.io/secrets-engine` diff --git a/plugins/vault-backend/README.md b/plugins/vault-backend/README.md index 753281c1dc..250bca1ffd 100644 --- a/plugins/vault-backend/README.md +++ b/plugins/vault-backend/README.md @@ -115,7 +115,7 @@ In case you need to support different secret engines for entities of the catalog # ... annotations: vault.io/secrets-path: path/to/secrets -+ vault.io/secret-engine: customSecretEngine # Optional. By default it uses 'secertEngine' value from configuration. ++ vault.io/secrets-engine: customSecretEngine # Optional. By default it uses 'secertEngine' value from configuration. ``` That will overwrite the default secret engine from the configuration. diff --git a/plugins/vault/README.md b/plugins/vault/README.md index d128d15dd9..434227561d 100644 --- a/plugins/vault/README.md +++ b/plugins/vault/README.md @@ -96,7 +96,7 @@ In case you need to support different secret engines for entities of the catalog # ... annotations: vault.io/secrets-path: path/to/secrets -+ vault.io/secret-engine: customSecretEngine # Optional. By default it uses 'secertEngine' value from configuration. ++ vault.io/secrets-engine: customSecretEngine # Optional. By default it uses 'secertEngine' value from configuration. ``` That will overwrite the default secret engine from the configuration. From 2e60250c7dbecaf6340597a4f4c700823a8d1aa8 Mon Sep 17 00:00:00 2001 From: Ilya Katlinski Date: Tue, 8 Aug 2023 10:58:46 +0200 Subject: [PATCH 019/131] test: update unit test for vault plugin api and table Signed-off-by: Ilya Katlinski --- plugins/vault/src/api.test.ts | 34 +++++++++----- .../EntityVaultTable.test.tsx | 44 +++++++++++++++---- 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/plugins/vault/src/api.test.ts b/plugins/vault/src/api.test.ts index ec88412303..47d33f2d99 100644 --- a/plugins/vault/src/api.test.ts +++ b/plugins/vault/src/api.test.ts @@ -21,6 +21,9 @@ import { VaultSecret, VaultClient } from './api'; import { UrlPatternDiscovery } from '@backstage/core-app-api'; describe('api', () => { + let api: VaultClient; + let fetchApiSpy: jest.SpyInstance>; + const server = setupServer(); setupRequestMockHandlers(server); @@ -64,37 +67,48 @@ describe('api', () => { ); }; - it('should return secrets', async () => { + beforeEach(() => { setupHandlers(); - const api = new VaultClient({ discoveryApi, fetchApi }); + + api = new VaultClient({ discoveryApi, fetchApi }); + fetchApiSpy = jest.spyOn(fetchApi, 'fetch'); + }); + + it('should return secrets', async () => { expect(await api.listSecrets('test/success')).toEqual( mockSecretsResult.items, ); + expect(fetchApiSpy).toHaveBeenCalledWith( + `${mockBaseUrl}/v1/secrets/test%2Fsuccess?`, + expect.anything(), + ); + }); + + it('should return secrets with custom engine', async () => { + expect(await api.listSecrets('test/success', 'kv')).toEqual( + mockSecretsResult.items, + ); + expect(fetchApiSpy).toHaveBeenCalledWith( + `${mockBaseUrl}/v1/secrets/test%2Fsuccess?engine=kv`, + expect.anything(), + ); }); it('should return empty secret list', async () => { - setupHandlers(); - const api = new VaultClient({ discoveryApi, fetchApi }); expect(await api.listSecrets('test/empty')).toEqual([]); }); it('should return all the secrets if no path defined', async () => { - setupHandlers(); - const api = new VaultClient({ discoveryApi, fetchApi }); expect(await api.listSecrets('')).toEqual(mockSecretsResult.items); }); it('should throw an error if the Vault API responds with an HTTP 404', async () => { - setupHandlers(); - const api = new VaultClient({ discoveryApi, fetchApi }); await expect(api.listSecrets('test/not-found')).rejects.toThrow( "No secrets found in path 'v1/secrets/test%2Fnot-found'", ); }); it('should throw an error if the Vault API responds with a non-successful HTTP status code', async () => { - setupHandlers(); - const api = new VaultClient({ discoveryApi, fetchApi }); await expect(api.listSecrets('test/error')).rejects.toThrow( 'Request failed with 400 Error', ); diff --git a/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.test.tsx b/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.test.tsx index e5bd5b43de..4cf5b60026 100644 --- a/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.test.tsx +++ b/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.test.tsx @@ -29,23 +29,31 @@ import { VaultSecret, vaultApiRef, VaultClient } from '../../api'; import { rest } from 'msw'; describe('EntityVaultTable', () => { + let vaultClient: VaultClient; + let apis: TestApiRegistry; + let listSecretsSpy: jest.SpyInstance>; + const server = setupServer(); setupRequestMockHandlers(server); - let apis: TestApiRegistry; const mockBaseUrl = 'https://api-vault.com/api/vault'; const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl); const fetchApi = new MockFetchApi(); - const entity = (secretPath: string) => { + const entity = (secretPath: string, secretEngine?: string | undefined) => { + const annotations: Record = { + 'vault.io/secrets-path': secretPath, + }; + if (secretEngine) { + annotations['vault.io/secrets-engine'] = secretEngine; + } + return { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { name: 'test', description: 'This is the description', - annotations: { - 'vault.io/secrets-path': secretPath, - }, + annotations, }, spec: { lifecycle: 'production', @@ -56,6 +64,7 @@ describe('EntityVaultTable', () => { }; const entityOk = entity('test/success'); + const entityOkWithEngine = entity('test/success', 'kv'); const entityEmpty = entity('test/empty'); const entityNotOk = entity('test/error'); @@ -91,10 +100,9 @@ describe('EntityVaultTable', () => { }; beforeEach(() => { - apis = TestApiRegistry.from([ - vaultApiRef, - new VaultClient({ discoveryApi, fetchApi }), - ]); + vaultClient = new VaultClient({ discoveryApi, fetchApi }); + apis = TestApiRegistry.from([vaultApiRef, vaultClient]); + listSecretsSpy = jest.spyOn(vaultClient, 'listSecrets'); }); it('should render secrets', async () => { @@ -107,6 +115,24 @@ describe('EntityVaultTable', () => { expect(await rendered.findAllByText(/secret::one/)).toBeDefined(); expect(await rendered.findAllByText(/secret::two/)).toBeDefined(); + + expect(listSecretsSpy).toHaveBeenCalledTimes(1); + expect(listSecretsSpy).toHaveBeenCalledWith('test/success', undefined); + }); + + it('should render secrets with custom engine', async () => { + setupHandlers(); + const rendered = await renderInTestApp( + + + , + ); + + expect(await rendered.findAllByText(/secret::one/)).toBeDefined(); + expect(await rendered.findAllByText(/secret::two/)).toBeDefined(); + + expect(listSecretsSpy).toHaveBeenCalledTimes(1); + expect(listSecretsSpy).toHaveBeenCalledWith('test/success', 'kv'); }); it('should render no secrets found', async () => { From ad9544c483a1490bb7c4debf717bde237a43c770 Mon Sep 17 00:00:00 2001 From: Ilya Katlinski Date: Tue, 8 Aug 2023 12:10:19 +0200 Subject: [PATCH 020/131] test: update unit test for vault backend plugin api Signed-off-by: Ilya Katlinski --- .../src/service/vaultApi.test.ts | 56 +++++++++++-------- plugins/vault/src/api.test.ts | 2 + .../EntityVaultTable.test.tsx | 1 + 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/plugins/vault-backend/src/service/vaultApi.test.ts b/plugins/vault-backend/src/service/vaultApi.test.ts index 1fb869ef60..1d037bb77a 100644 --- a/plugins/vault-backend/src/service/vaultApi.test.ts +++ b/plugins/vault-backend/src/service/vaultApi.test.ts @@ -21,6 +21,8 @@ import { VaultSecret, VaultClient, VaultSecretList } from './vaultApi'; import { ConfigReader } from '@backstage/config'; describe('VaultApi', () => { + let api: VaultClient; + const server = setupServer(); setupRequestMockHandlers(server); @@ -43,20 +45,24 @@ describe('VaultApi', () => { }, }; - const mockSecretsResult: VaultSecret[] = [ - { - name: 'secret::one', - path: 'test/success', - editUrl: `${mockBaseUrl}/ui/vault/secrets/secrets/edit/test/success/secret::one`, - showUrl: `${mockBaseUrl}/ui/vault/secrets/secrets/show/test/success/secret::one`, - }, - { - name: 'secret::two', - path: 'test/success', - editUrl: `${mockBaseUrl}/ui/vault/secrets/secrets/edit/test/success/secret::two`, - showUrl: `${mockBaseUrl}/ui/vault/secrets/secrets/show/test/success/secret::two`, - }, - ]; + const mockSecretsResult = ( + secretEngine: string = 'secrets', + ): VaultSecret[] => { + return [ + { + name: 'secret::one', + path: 'test/success', + editUrl: `${mockBaseUrl}/ui/vault/secrets/${secretEngine}/edit/test/success/secret::one`, + showUrl: `${mockBaseUrl}/ui/vault/secrets/${secretEngine}/show/test/success/secret::one`, + }, + { + name: 'secret::two', + path: 'test/success', + editUrl: `${mockBaseUrl}/ui/vault/secrets/${secretEngine}/edit/test/success/secret::two`, + showUrl: `${mockBaseUrl}/ui/vault/secrets/${secretEngine}/show/test/success/secret::two`, + }, + ]; + }; const setupHandlers = () => { server.use( @@ -66,6 +72,9 @@ describe('VaultApi', () => { return res(ctx.json(mockListResult)); }, ), + rest.get(`${mockBaseUrl}/v1/kv/metadata/test/success`, (_, res, ctx) => { + return res(ctx.json(mockListResult)); + }), rest.get( `${mockBaseUrl}/v1/secrets/metadata/test/error`, (_, res, ctx) => { @@ -78,28 +87,31 @@ describe('VaultApi', () => { ); }; - it('should return secrets', async () => { + beforeEach(() => { setupHandlers(); - const api = new VaultClient({ config }); + api = new VaultClient({ config }); + }); + + it('should return secrets', async () => { const secrets = await api.listSecrets('test/success'); - expect(secrets).toEqual(mockSecretsResult); + expect(secrets).toEqual(mockSecretsResult()); + }); + + it('should return secrets for custom engine', async () => { + const secrets = await api.listSecrets('test/success', 'kv'); + expect(secrets).toEqual(mockSecretsResult('kv')); }); it('should return empty secret list', async () => { - setupHandlers(); - const api = new VaultClient({ config }); const secrets = await api.listSecrets('test/error'); expect(secrets).toEqual([]); }); it('should return success token renew', async () => { - setupHandlers(); - const api = new VaultClient({ config }); expect(await api.renewToken()).toBe(undefined); }); it('should render frontend url', () => { - const api = new VaultClient({ config }); const url = api.getFrontendSecretsUrl(); expect(url).toEqual(`${mockBaseUrl}/ui/vault/secrets/secrets`); }); diff --git a/plugins/vault/src/api.test.ts b/plugins/vault/src/api.test.ts index 47d33f2d99..03c75fbce3 100644 --- a/plugins/vault/src/api.test.ts +++ b/plugins/vault/src/api.test.ts @@ -68,6 +68,8 @@ describe('api', () => { }; beforeEach(() => { + jest.resetAllMocks(); + setupHandlers(); api = new VaultClient({ discoveryApi, fetchApi }); diff --git a/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.test.tsx b/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.test.tsx index 4cf5b60026..69fa5fec4e 100644 --- a/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.test.tsx +++ b/plugins/vault/src/components/EntityVaultTable/EntityVaultTable.test.tsx @@ -100,6 +100,7 @@ describe('EntityVaultTable', () => { }; beforeEach(() => { + jest.resetAllMocks(); vaultClient = new VaultClient({ discoveryApi, fetchApi }); apis = TestApiRegistry.from([vaultApiRef, vaultClient]); listSecretsSpy = jest.spyOn(vaultClient, 'listSecrets'); From 1dd90e437cf5ee60adfe4ee171275325380c2923 Mon Sep 17 00:00:00 2001 From: Ilya Katlinski Date: Tue, 8 Aug 2023 12:23:43 +0200 Subject: [PATCH 021/131] doc: fix documentation types based on reviewdog checks Signed-off-by: Ilya Katlinski --- .changeset/silent-numbers-retire.md | 2 +- plugins/vault-backend/README.md | 2 +- plugins/vault/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/silent-numbers-retire.md b/.changeset/silent-numbers-retire.md index 6f260c05ef..b386d04b14 100644 --- a/.changeset/silent-numbers-retire.md +++ b/.changeset/silent-numbers-retire.md @@ -3,4 +3,4 @@ '@backstage/plugin-vault': minor --- -Added ability to override vault secretEngine value on catalog entity level using annotation `vault.io/secrets-engine` +Added ability to override vault secret engine value on catalog entity level using annotation `vault.io/secrets-engine` diff --git a/plugins/vault-backend/README.md b/plugins/vault-backend/README.md index 250bca1ffd..aac2b68f60 100644 --- a/plugins/vault-backend/README.md +++ b/plugins/vault-backend/README.md @@ -106,7 +106,7 @@ The path is relative to your secrets engine folder. So if you want to get the se You will set the `vault.io/secret-path` to `test/backstage`. If the folder `backstage` contains other sub-folders, the plugin will fetch the secrets inside them and adapt the **View** and **Edit** URLs to point to the correct place. -In case you need to support different secret engines for entities of the catalog you can proivde optional annotion to the entity in `catalog-info.yaml`: +In case you need to support different secret engines for entities of the catalog you can provide optional annotation to the entity in `catalog-info.yaml`: ```diff apiVersion: backstage.io/v1alpha1 diff --git a/plugins/vault/README.md b/plugins/vault/README.md index 434227561d..3fe87398ce 100644 --- a/plugins/vault/README.md +++ b/plugins/vault/README.md @@ -87,7 +87,7 @@ If the annotation is missing for a certain component, then the card will show so ![Screenshot of the vault plugin with missing annotation](images/annotation-missing.png) -In case you need to support different secret engines for entities of the catalog you can proivde optional annotion to the entity in `catalog-info.yaml`: +In case you need to support different secret engines for entities of the catalog you can provide optional annotation to the entity in `catalog-info.yaml`: ```diff apiVersion: backstage.io/v1alpha1 From cfa3b6e9e0467e2813ec87845b7ec74ad32e568d Mon Sep 17 00:00:00 2001 From: Ilya Katlinski Date: Tue, 8 Aug 2023 12:49:41 +0200 Subject: [PATCH 022/131] doc: update api reports as public api changed Signed-off-by: Ilya Katlinski --- plugins/vault-backend/api-report.md | 10 ++++++++-- plugins/vault/api-report.md | 5 ++++- plugins/vault/src/constants.ts | 3 +++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/plugins/vault-backend/api-report.md b/plugins/vault-backend/api-report.md index 4890e585b4..47275693ec 100644 --- a/plugins/vault-backend/api-report.md +++ b/plugins/vault-backend/api-report.md @@ -25,7 +25,10 @@ export interface RouterOptions { // @public export interface VaultApi { getFrontendSecretsUrl(): string; - listSecrets(secretPath: string): Promise; + listSecrets( + secretPath: string, + secretMount?: string | undefined, + ): Promise; renewToken?(): Promise; } @@ -49,7 +52,10 @@ export class VaultClient implements VaultApi { // (undocumented) getFrontendSecretsUrl(): string; // (undocumented) - listSecrets(secretPath: string): Promise; + listSecrets( + secretPath: string, + secretEngine?: string | undefined, + ): Promise; // (undocumented) renewToken(): Promise; } diff --git a/plugins/vault/api-report.md b/plugins/vault/api-report.md index 8c4e5da829..ee85145202 100644 --- a/plugins/vault/api-report.md +++ b/plugins/vault/api-report.md @@ -20,7 +20,10 @@ export const VAULT_SECRET_PATH_ANNOTATION = 'vault.io/secrets-path'; // @public export interface VaultApi { - listSecrets(secretPath: string): Promise; + listSecrets( + secretPath: string, + secretMount?: string | undefined, + ): Promise; } // @public (undocumented) diff --git a/plugins/vault/src/constants.ts b/plugins/vault/src/constants.ts index 7398bc59ee..a71a1c104f 100644 --- a/plugins/vault/src/constants.ts +++ b/plugins/vault/src/constants.ts @@ -18,4 +18,7 @@ * @public */ export const VAULT_SECRET_ENGINE_ANNOTATION = 'vault.io/secrets-engine'; +/** + * @public + */ export const VAULT_SECRET_PATH_ANNOTATION = 'vault.io/secrets-path'; From c2988744e8da93f63d9e757aa50543e58a10ca16 Mon Sep 17 00:00:00 2001 From: Vladimir Masarik Date: Mon, 14 Aug 2023 16:36:52 +0200 Subject: [PATCH 023/131] Add warning about backend APIs not having auth by default Signed-off-by: Vladimir Masarik --- .changeset/slimy-spiders-pull.md | 5 +++++ docs/auth/index.md | 2 ++ docs/permissions/getting-started.md | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changeset/slimy-spiders-pull.md diff --git a/.changeset/slimy-spiders-pull.md b/.changeset/slimy-spiders-pull.md new file mode 100644 index 0000000000..12bd0e23e4 --- /dev/null +++ b/.changeset/slimy-spiders-pull.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Add warning about backend APIs not having auth by default diff --git a/docs/auth/index.md b/docs/auth/index.md index b2aac47e2a..6f3250559c 100644 --- a/docs/auth/index.md +++ b/docs/auth/index.md @@ -10,6 +10,8 @@ configure Backstage to have any number of authentication providers, but only one of these will typically be used for sign-in, with the rest being used to provide access external resources. +> Note: Backstage backend APIs are by default unauthenticated. Thus, if your Backstage instance is exposed to the Internet, anyone can access information in the Backstage. If you would like to learn more, read about how to [authenticate API requests"](https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/authenticate-api-requests.md). + ## Built-in Authentication Providers Backstage comes with many common authentication providers in the core library: diff --git a/docs/permissions/getting-started.md b/docs/permissions/getting-started.md index 01a268f7ef..0d542a6502 100644 --- a/docs/permissions/getting-started.md +++ b/docs/permissions/getting-started.md @@ -36,7 +36,7 @@ Like many other parts of Backstage, the permissions framework relies on informat ## Optionally add cookie-based authentication -Asset requests initiated by the browser will not include a token in the `Authorization` header. If these requests check authorization through the permission framework, as done in plugins like TechDocs, then you'll need to set up cookie-based authentication. Refer to the ["Authenticate API requests"](https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/authenticate-api-requests.md) tutorial for a demonstration on how to implement this behavior. +Frontend requests initiated by the browser will not include a token in the `Authorization` header. If these requests check authorization through the permission framework, as done in plugins like TechDocs, then you'll need to set up cookie-based authentication. Refer to the ["Authenticate API requests"](https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/authenticate-api-requests.md) tutorial for a demonstration on how to implement this behavior. ## Integrating the permission framework with your Backstage instance From fe55b95b133d062857c0f8cc219ce853aa8b4437 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 15 Aug 2023 12:13:52 +0100 Subject: [PATCH 024/131] Remove GitlabSaasUsersResponse and merge with GitlabGroupMemberResponse with additional fields Signed-off-by: Stephen Barry --- .../src/lib/types.ts | 39 ++++++------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 09a9fb92c3..1c08a0e253 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -60,34 +60,17 @@ export type GitLabGroupMembersResponse = { data: { group: { groupMembers: { - nodes: { user: { id: string } }[]; - pageInfo: { - endCursor: string; - hasNextPage: boolean; - }; - }; - }; - }; -}; - -export type GitLabSaasUsersResponse = { - errors: { message: string }[]; - data: { - group: { - groupMembers: { - nodes: [ - { - user: { - id: string; - username: string; - publicEmail: string; - name: string; - state: string; - webUrl: string; - avatarUrl: string; - }; - }, - ]; + nodes: { + user: { + id: string; + username: string; + publicEmail: string; + name: string; + state: string; + webUrl: string; + avatarUrl: string; + }; + }[]; pageInfo: { endCursor: string; hasNextPage: boolean; From 9e5f59fdd2bc5abc8d2f1086f7038ff0e2ba9207 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 15 Aug 2023 12:16:28 +0100 Subject: [PATCH 025/131] Rename listSaasGroups -> listDescendantGroups Signed-off-by: Stephen Barry --- .../src/lib/client.test.ts | 18 +++++++++--------- .../src/lib/client.ts | 9 +++++---- .../GitlabOrgDiscoveryEntityProvider.test.ts | 2 +- .../GitlabOrgDiscoveryEntityProvider.ts | 2 +- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index a62a3874d5..79dd79dffa 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -575,12 +575,12 @@ describe('GitLabClient', () => { }); }); - describe('listSaasGroups', () => { + describe('listDescendantGroups', () => { it('gets all groups under root', async () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasGroups', async (_, res, ctx) => + .query('listDescendantGroups', async (_, res, ctx) => res( ctx.data({ group: { @@ -611,7 +611,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasGroups = (await client.listSaasGroups('group1')).items; + const saasGroups = (await client.listDescendantGroups('group1')).items; const expectedSaasGroup = [ { @@ -631,7 +631,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasGroups', async (_, res, ctx) => + .query('listDescendantGroups', async (_, res, ctx) => res( ctx.data({ group: {}, @@ -644,7 +644,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasGroups = (await client.listSaasGroups('group1')).items; + const saasGroups = (await client.listDescendantGroups('group1')).items; expect(saasGroups).toEqual([]); }); @@ -653,7 +653,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasGroups', async (_, res, ctx) => + .query('listDescendantGroups', async (_, res, ctx) => res( ctx.errors([ { message: 'Unexpected end of document', locations: [] }, @@ -666,7 +666,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - await expect(() => client.listSaasGroups('group1')).rejects.toThrow( + await expect(() => client.listDescendantGroups('group1')).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); @@ -674,7 +674,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasGroups', async (req, res, ctx) => + .query('listDescendantGroups', async (req, res, ctx) => res( ctx.data({ group: { @@ -717,7 +717,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasGroups = (await client.listSaasGroups('root')).items; + const saasGroups = (await client.listDescendantGroups('root')).items; const expectedSaasGroup1 = { id: 1, diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index fa4ccf9f97..d49811b714 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -22,7 +22,6 @@ import { import { Logger } from 'winston'; import { GitLabGroup, - GitLabSaasUsersResponse, GitLabSaasGroupsResponse, GitLabGroupMembersResponse, GitLabUser, @@ -103,7 +102,7 @@ export class GitLabClient { let endCursor: string | null = null; do { - const response: GitLabSaasUsersResponse = await fetch( + const response: GitLabGroupMembersResponse = await fetch( `${this.config.baseUrl}/api/graphql`, { method: 'POST', @@ -170,7 +169,9 @@ export class GitLabClient { return { items }; } - async listSaasGroups(groupPath: string): Promise> { + async listDescendantGroups( + groupPath: string, + ): Promise> { const items: GitLabGroup[] = []; let hasNextPage: boolean = false; let endCursor: string | null = null; @@ -186,7 +187,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, endCursor }, - query: `query listSaasGroups($group: ID!, $endCursor: String) { + query: `query listDescendantGroups($group: ID!, $endCursor: String) { group(fullPath: $group) { descendantGroups(first: 100, after: $endCursor){ nodes{ diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 7339dd4a29..0a1284b22c 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -514,7 +514,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { server.use( graphql .link('https://gitlab.com/api/graphql') - .query('listSaasGroups', async (_, res, ctx) => + .query('listDescendantGroups', async (_, res, ctx) => res( ctx.data({ group: { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 2b614d69c9..a14a00d718 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -189,7 +189,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { active: true, }); } else { - groups = (await client.listSaasGroups(this.config.group)).items; + groups = (await client.listDescendantGroups(this.config.group)).items; users = (await client.listSaasUsers(this.config.group.split('/')[0])) .items; } From b9578fd485daade2723444ea4eb067327fb1d691 Mon Sep 17 00:00:00 2001 From: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> Date: Tue, 15 Aug 2023 15:10:03 +0100 Subject: [PATCH 026/131] Update .changeset/thick-pets-drive.md Co-authored-by: Jamie Klassen Signed-off-by: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> --- .changeset/thick-pets-drive.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/thick-pets-drive.md b/.changeset/thick-pets-drive.md index a4d8a35029..ee09f8af06 100644 --- a/.changeset/thick-pets-drive.md +++ b/.changeset/thick-pets-drive.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend-module-gitlab': patch +'@backstage/plugin-catalog-backend-module-gitlab': minor --- Fix Gitlab.com user ingestion by scoping GitlabOrgDiscoveryEntityProvider to a group. From 5a3842761c2d6f0964d754d99ff231b0ed589bd4 Mon Sep 17 00:00:00 2001 From: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> Date: Tue, 15 Aug 2023 15:10:28 +0100 Subject: [PATCH 027/131] Update plugins/catalog-backend-module-gitlab/src/lib/client.ts Co-authored-by: Jamie Klassen Signed-off-by: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index fa4ccf9f97..dca545eab3 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -118,13 +118,13 @@ export class GitLabClient { groupMembers(first: 100, relations: [DESCENDANTS], after: $endCursor) { nodes { user { - id - username + id + username publicEmail name state webUrl - avatarUrl + avatarUrl } } pageInfo { From ecf970e4fe8479d475740a2517e1ba8e7adf720b Mon Sep 17 00:00:00 2001 From: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> Date: Tue, 15 Aug 2023 15:10:47 +0100 Subject: [PATCH 028/131] Update plugins/catalog-backend-module-gitlab/src/lib/client.ts Co-authored-by: Jamie Klassen Signed-off-by: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> --- .../src/lib/client.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index dca545eab3..f44aec9b20 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -187,24 +187,24 @@ export class GitLabClient { body: JSON.stringify({ variables: { group: groupPath, endCursor }, query: `query listSaasGroups($group: ID!, $endCursor: String) { - group(fullPath: $group) { - descendantGroups(first: 100, after: $endCursor){ - nodes{ - id - name - description - fullPath - parent{ - id - } - } - pageInfo { - endCursor - hasNextPage + group(fullPath: $group) { + descendantGroups(first: 100, after: $endCursor) { + nodes { + id + name + description + fullPath + parent { + id } + } + pageInfo { + endCursor + hasNextPage + } } - } - }`, + } + }`, }), }, ).then(r => r.json()); From 31d50133b4bc80fe07c8064d3fa58fcb16222347 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Wed, 16 Aug 2023 10:15:12 +0100 Subject: [PATCH 029/131] Remove listSaasUsers method and switch to paramterized getGroupMembers passing relations and update tests Signed-off-by: Stephen Barry --- .../src/lib/client.test.ts | 67 +++++++--- .../src/lib/client.ts | 116 +++++------------- .../GitlabOrgDiscoveryEntityProvider.test.ts | 6 +- .../GitlabOrgDiscoveryEntityProvider.ts | 17 ++- 4 files changed, 97 insertions(+), 109 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 79dd79dffa..28c5eb2775 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -398,12 +398,12 @@ describe('GitLabClient', () => { ]); }); - describe('listSaasUsers', () => { + describe('get gitlab.com users', () => { it('gets all users under group', async () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasUsers', async (_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: { @@ -436,7 +436,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasMembers = (await client.listSaasUsers('group1')).items; + const saasMembers = ( + await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + ).items; const expectedSaasMember = [ { id: 1, @@ -457,7 +459,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasUsers', async (_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: {}, @@ -470,7 +472,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasMembers = (await client.listSaasUsers('group1')).items; + const saasMembers = ( + await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + ).items; expect(saasMembers).toEqual([]); }); @@ -479,7 +483,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasUsers', async (_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.errors([ { message: 'Unexpected end of document', locations: [] }, @@ -492,7 +496,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - await expect(() => client.listSaasUsers('group1')).rejects.toThrow( + await expect(() => + client.getGroupMembers('group1', 'DIRECT, DESCENDANTS'), + ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); @@ -500,7 +506,7 @@ describe('GitLabClient', () => { server.use( graphql .link(`${MOCK_CONFIG.baseUrl}/api/graphql`) - .query('listSaasUsers', async (req, res, ctx) => + .query('getGroupMembers', async (req, res, ctx) => res( ctx.data({ group: { @@ -547,7 +553,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const saasMembers = (await client.listSaasUsers('group1')).items; + const saasMembers = ( + await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + ).items; const expectedSaasMember1 = { id: 1, @@ -751,7 +759,19 @@ describe('GitLabClient', () => { ctx.data({ group: { groupMembers: { - nodes: [{ user: { id: 'gid://gitlab/User/1' } }], + nodes: [ + { + user: { + id: 'gid://gitlab/User/1', + username: 'user1', + publicEmail: 'user1@example.com', + name: 'user1', + state: 'active', + webUrl: 'user1.com', + avatarUrl: 'user1', + }, + }, + ], pageInfo: { endCursor: 'end', hasNextPage: false, @@ -767,9 +787,19 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1'); + const members = await client.getGroupMembers('group1', 'DIRECT'); - expect(members).toEqual([1]); + const user = { + id: 1, + username: 'user1', + email: 'user1@example.com', + name: 'user1', + state: 'active', + web_url: 'user1.com', + avatar_url: 'user1', + }; + + expect(members.items).toEqual([user]); }); it('gets member IDs with token without full permissions', async () => { @@ -789,9 +819,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1'); + const members = await client.getGroupMembers('group1', 'DIRECT'); - expect(members).toEqual([]); + expect(members.items).toEqual([]); }); it('rejects when GraphQL returns errors', async () => { @@ -811,7 +841,9 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - await expect(() => client.getGroupMembers('group1')).rejects.toThrow( + await expect(() => + client.getGroupMembers('group1', 'DIRECT'), + ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); }); @@ -843,9 +875,10 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1'); + const members = await client.getGroupMembers('group1', 'DIRECT'); - expect(members).toEqual([1, 2]); + expect(members.items[0].id).toEqual(1); + expect(members.items[1].id).toEqual(2); }); }); }); diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index dee9f4ec51..5eee68cbe2 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -96,79 +96,6 @@ export class GitLabClient { return this.pagedRequest(`/groups`, options); } - async listSaasUsers(groupPath: string): Promise> { - const items: GitLabUser[] = []; - let hasNextPage: boolean = false; - let endCursor: string | null = null; - - do { - const response: GitLabGroupMembersResponse = await fetch( - `${this.config.baseUrl}/api/graphql`, - { - method: 'POST', - headers: { - ...getGitLabRequestOptions(this.config).headers, - ['Content-Type']: 'application/json', - }, - body: JSON.stringify({ - variables: { group: groupPath, endCursor }, - query: `query listSaasUsers($group: ID!, $endCursor: String) { - group(fullPath: $group) { - groupMembers(first: 100, relations: [DESCENDANTS], after: $endCursor) { - nodes { - user { - id - username - publicEmail - name - state - webUrl - avatarUrl - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - }`, - }), - }, - ).then(r => r.json()); - if (response.errors) { - throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`); - } - - if (!response.data.group?.groupMembers?.nodes) { - this.logger.warn( - `Couldn't get members under ${groupPath}. The provided token might not have sufficient permissions`, - ); - continue; - } - - const usersData = response.data.group.groupMembers.nodes; - - for (let i = 0; i < usersData.length; i++) { - const userItem = usersData[i]; - - const formattedUserResponse = { - id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), - username: userItem.user.username, - email: userItem.user.publicEmail, - name: userItem.user.name, - state: userItem.user.state, - web_url: userItem.user.webUrl, - avatar_url: userItem.user.avatarUrl, - }; - - items.push(formattedUserResponse); - } - ({ hasNextPage, endCursor } = response.data.group.groupMembers.pageInfo); - } while (hasNextPage); - return { items }; - } - async listDescendantGroups( groupPath: string, ): Promise> { @@ -243,8 +170,11 @@ export class GitLabClient { return { items }; } - async getGroupMembers(groupPath: string): Promise { - const memberIds = []; + async getGroupMembers( + groupPath: string, + relations: string, + ): Promise> { + const items: GitLabUser[] = []; let hasNextPage: boolean = false; let endCursor: string | null = null; do { @@ -257,13 +187,19 @@ export class GitLabClient { ['Content-Type']: 'application/json', }, body: JSON.stringify({ - variables: { group: groupPath, endCursor }, + variables: { group: groupPath, relations: relations, endCursor }, query: `query getGroupMembers($group: ID!, $endCursor: String) { group(fullPath: $group) { - groupMembers(first: 100, relations: [DIRECT], after: $endCursor) { + groupMembers(first: 100, relations: [$relations], after: $endCursor) { nodes { user { id + username + publicEmail + name + state + webUrl + avatarUrl } } pageInfo { @@ -287,16 +223,26 @@ export class GitLabClient { continue; } - memberIds.push( - ...response.data.group.groupMembers.nodes - .filter(n => n.user) - .map(node => - Number(node.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), - ), - ); + const usersData = response.data.group.groupMembers.nodes; + + for (let i = 0; i < usersData.length; i++) { + const userItem = usersData[i]; + + const formattedUserResponse = { + id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), + username: userItem.user.username, + email: userItem.user.publicEmail, + name: userItem.user.name, + state: userItem.user.state, + web_url: userItem.user.webUrl, + avatar_url: userItem.user.avatarUrl, + }; + + items.push(formattedUserResponse); + } ({ hasNextPage, endCursor } = response.data.group.groupMembers.pageInfo); } while (hasNextPage); - return memberIds; + return { items }; } /** diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 0a1284b22c..1b1357c1bf 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -550,7 +550,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { ), graphql .link('https://gitlab.com/api/graphql') - .query('listSaasUsers', async (_, res, ctx) => + .query('getGroupMembers', async (_, res, ctx) => res( ctx.data({ group: { @@ -634,7 +634,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { name: 'testuser1', }, spec: { - memberOf: ['group2'], + memberOf: ['group2', 'group3'], profile: { displayName: 'Test User 1', email: 'testuser1@example.com', @@ -659,7 +659,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { name: 'testuser2', }, spec: { - memberOf: ['group3'], + memberOf: ['group2', 'group3'], profile: { displayName: 'Test User 2', email: 'testuser2@example.com', diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index a14a00d718..bec3b06661 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -190,8 +190,12 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { }); } else { groups = (await client.listDescendantGroups(this.config.group)).items; - users = (await client.listSaasUsers(this.config.group.split('/')[0])) - .items; + users = ( + await client.getGroupMembers( + this.config.group.split('/')[0], + 'DIRECT, DESCENDANTS', + ) + ).items; } const idMappedUser: { [userId: number]: GitLabUser } = {}; @@ -236,8 +240,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groupRes.scanned++; groupRes.matches.push(group); - for (const id of await client.getGroupMembers(group.full_path)) { - const user = idMappedUser[id]; + const groupUsers = await client.getGroupMembers( + group.full_path, + 'DIRECT', + ); + + for (const groupUser of groupUsers.items) { + const user = idMappedUser[groupUser.id]; if (user) { user.groups = (user.groups ?? []).concat(group); } From aef4f7263ae1bf8f13f2a025279819f6d92e3439 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Wed, 16 Aug 2023 11:41:33 +0100 Subject: [PATCH 030/131] Fix relations argument declaration for getGroupMembers query and fix relation for fetching SaaS users Signed-off-by: Stephen Barry --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 2 +- .../src/providers/GitlabOrgDiscoveryEntityProvider.ts | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 5eee68cbe2..52829c343b 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -188,7 +188,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, relations: relations, endCursor }, - query: `query getGroupMembers($group: ID!, $endCursor: String) { + query: `query getGroupMembers($group: ID!, $relations: GroupMemberRelation!, $endCursor: String) { group(fullPath: $group) { groupMembers(first: 100, relations: [$relations], after: $endCursor) { nodes { diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index bec3b06661..5c4624c43a 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -191,10 +191,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { } else { groups = (await client.listDescendantGroups(this.config.group)).items; users = ( - await client.getGroupMembers( - this.config.group.split('/')[0], - 'DIRECT, DESCENDANTS', - ) + await client.getGroupMembers(this.config.group.split('/')[0], 'DIRECT') ).items; } From cb61a63af3d8d14c65501c54f2ce49a763a6d36c Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Wed, 16 Aug 2023 14:43:25 +0100 Subject: [PATCH 031/131] Switch relations argument to string[] and update references Signed-off-by: Stephen Barry --- .../src/lib/client.test.ts | 16 ++++++++-------- .../src/lib/client.ts | 6 +++--- .../GitlabOrgDiscoveryEntityProvider.ts | 10 ++++++---- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 28c5eb2775..2bb1f713b5 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -437,7 +437,7 @@ describe('GitLabClient', () => { }); const saasMembers = ( - await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) ).items; const expectedSaasMember = [ { @@ -473,7 +473,7 @@ describe('GitLabClient', () => { }); const saasMembers = ( - await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) ).items; expect(saasMembers).toEqual([]); @@ -497,7 +497,7 @@ describe('GitLabClient', () => { }); await expect(() => - client.getGroupMembers('group1', 'DIRECT, DESCENDANTS'), + client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']), ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); @@ -554,7 +554,7 @@ describe('GitLabClient', () => { }); const saasMembers = ( - await client.getGroupMembers('group1', 'DIRECT, DESCENDANTS') + await client.getGroupMembers('group1', ['DIRECT, DESCENDANTS']) ).items; const expectedSaasMember1 = { @@ -787,7 +787,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', 'DIRECT'); + const members = await client.getGroupMembers('group1', ['DIRECT']); const user = { id: 1, @@ -819,7 +819,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', 'DIRECT'); + const members = await client.getGroupMembers('group1', ['DIRECT']); expect(members.items).toEqual([]); }); @@ -842,7 +842,7 @@ describe('GitLabClient', () => { }); await expect(() => - client.getGroupMembers('group1', 'DIRECT'), + client.getGroupMembers('group1', ['DIRECT']), ).rejects.toThrow( 'GraphQL errors: [{"message":"Unexpected end of document","locations":[]}]', ); @@ -875,7 +875,7 @@ describe('GitLabClient', () => { logger: getVoidLogger(), }); - const members = await client.getGroupMembers('group1', 'DIRECT'); + const members = await client.getGroupMembers('group1', ['DIRECT']); expect(members.items[0].id).toEqual(1); expect(members.items[1].id).toEqual(2); diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 006a345c4a..e2c4914979 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -173,7 +173,7 @@ export class GitLabClient { async getGroupMembers( groupPath: string, - relations: string, + relations: string[], ): Promise> { const items: GitLabUser[] = []; let hasNextPage: boolean = false; @@ -189,9 +189,9 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, relations: relations, endCursor }, - query: `query getGroupMembers($group: ID!, $relations: GroupMemberRelation!, $endCursor: String) { + query: `query getGroupMembers($group: ID!, $relations: [GroupMemberRelation!], $endCursor: String) { group(fullPath: $group) { - groupMembers(first: 100, relations: [$relations], after: $endCursor) { + groupMembers(first: 100, relations: $relations, after: $endCursor) { nodes { user { id diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts index 5c4624c43a..0ef0e2b23c 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts @@ -191,7 +191,10 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { } else { groups = (await client.listDescendantGroups(this.config.group)).items; users = ( - await client.getGroupMembers(this.config.group.split('/')[0], 'DIRECT') + await client.getGroupMembers(this.config.group.split('/')[0], [ + 'DIRECT', + 'DESCENDANTS', + ]) ).items; } @@ -237,10 +240,9 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider { groupRes.scanned++; groupRes.matches.push(group); - const groupUsers = await client.getGroupMembers( - group.full_path, + const groupUsers = await client.getGroupMembers(group.full_path, [ 'DIRECT', - ); + ]); for (const groupUser of groupUsers.items) { const user = idMappedUser[groupUser.id]; From 1825c689d0605a295d603d182269f9e216db9567 Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Wed, 16 Aug 2023 14:45:00 +0100 Subject: [PATCH 032/131] Update GitLab Org Data docs Signed-off-by: Stephen Barry --- docs/integrations/gitlab/org.md | 45 ++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index 7090dab26e..5078e93d77 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -58,8 +58,6 @@ amount of data, this can take significant time and resources. The token used must have the `read_api` scope, and the Users and Groups fetched will be those visible to the account which provisioned the token. -**NOTE**: If any groups that are being ingested are empty groups and the user which provisioned the token is shared with a higher level group via [group sharing](https://docs.gitlab.com/ee/user/group/manage.html#share-a-group-with-another-group) and you don't see the expected number of `Group` entities in the catalog you may be hitting this [Gitlab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996). - ```yaml catalog: providers: @@ -67,11 +65,52 @@ catalog: yourProviderId: host: gitlab.com orgEnabled: true - group: org/teams # Required for gitlab.com. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped) + group: org/teams # Required for gitlab.com when `orgEnabled: true`. Optional for self managed. Must not end with slash. Accepts only groups under the provided path (which will be stripped) groupPattern: '[\s\S]*' # Optional. Filters found groups based on provided pattern. Defaults to `[\s\S]*`, which means to not filter anything ``` +### Groups + When the `group` parameter is provided, the corresponding path prefix will be stripped out from each matching group when computing the unique entity name. e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will be `avengers-gotg`. + +For gitlab.com, when `orgEnabled: true`, the `group` parameter is required in +order to limit the ingestion to a group within your organisation. `Group` +entities will only be ingested for the configured group, or it's descendant groups, +but not any ancestor groups higher than the configured group path. Only groups +which contain members will be ingested. + +### Users + +For self hosted, all `User` entities are ingested from the entire instance. + +For gitlab.com `User` entities for users who have [direct or inherited membership](https://docs.gitlab.com/ee/user/project/members/index.html#membership-types) +of the top-level group for the configured group path will be ingested. + +### Limiting `User` and `Group` entity ingestion in the provider + +Optionally, you can limit the entity types ingested by the provider when using +`orgEnabled: true` with the following `rules` configuration to limit it to only +`User` and `Group` entities. + +```yaml +catalog: + providers: + gitlab: + yourOrgDataProviderId: + host: gitlab.com + orgEnabled: true + group: org/teams + rules: + - allow: [Group, User] +``` + +## Troubleshooting + +**NOTE**: If any groups that are being ingested are empty groups (i.e. do not +contain any projects) and the user which provisioned the token is shared with a +higher level group via [group sharing](https://docs.gitlab.com/ee/user/group/manage.html#share-a-group-with-another-group) +and you don't see the expected number of `Group` entities in the catalog you may +be hitting this [Gitlab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996). From 14ae8a9497e2a0ccc7465f52b60afb1f45c4a0b9 Mon Sep 17 00:00:00 2001 From: Vladimir Masarik Date: Mon, 21 Aug 2023 19:45:25 +0200 Subject: [PATCH 033/131] Remove useless change set Signed-off-by: Vladimir Masarik --- .changeset/slimy-spiders-pull.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/slimy-spiders-pull.md diff --git a/.changeset/slimy-spiders-pull.md b/.changeset/slimy-spiders-pull.md deleted file mode 100644 index 12bd0e23e4..0000000000 --- a/.changeset/slimy-spiders-pull.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-auth-backend': patch ---- - -Add warning about backend APIs not having auth by default From 8bc51413d9069cf6cc50ac6c7f57adbaf658505a Mon Sep 17 00:00:00 2001 From: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> Date: Tue, 22 Aug 2023 08:26:01 +0100 Subject: [PATCH 034/131] Apply suggestions from code review Co-authored-by: Jamie Klassen Signed-off-by: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> --- docs/integrations/gitlab/org.md | 2 +- .../src/lib/client.ts | 34 ++++++++++--------- .../src/lib/types.ts | 2 +- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/docs/integrations/gitlab/org.md b/docs/integrations/gitlab/org.md index 5078e93d77..ca40cefa9e 100644 --- a/docs/integrations/gitlab/org.md +++ b/docs/integrations/gitlab/org.md @@ -78,7 +78,7 @@ e.g. If `group` is `org/teams`, the name for `org/teams/avengers/gotg` will be For gitlab.com, when `orgEnabled: true`, the `group` parameter is required in order to limit the ingestion to a group within your organisation. `Group` -entities will only be ingested for the configured group, or it's descendant groups, +entities will only be ingested for the configured group, or its descendant groups, but not any ancestor groups higher than the configured group path. Only groups which contain members will be ingested. diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index e2c4914979..3675d7e779 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -115,25 +115,27 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, endCursor }, - query: `query listDescendantGroups($group: ID!, $endCursor: String) { - group(fullPath: $group) { - descendantGroups(first: 100, after: $endCursor) { - nodes { - id - name - description - fullPath - parent { + query: /* GraphQL */ ` + query listDescendantGroups($group: ID!, $endCursor: String) { + group(fullPath: $group) { + descendantGroups(first: 100, after: $endCursor) { + nodes { id + name + description + fullPath + parent { + id + } + } + pageInfo { + endCursor + hasNextPage } } - pageInfo { - endCursor - hasNextPage - } - } + } } - }`, + `, }), }, ).then(r => r.json()); @@ -189,7 +191,7 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, relations: relations, endCursor }, - query: `query getGroupMembers($group: ID!, $relations: [GroupMemberRelation!], $endCursor: String) { + query: /* GraphQL */ `query getGroupMembers($group: ID!, $relations: [GroupMemberRelation!], $endCursor: String) { group(fullPath: $group) { groupMembers(first: 100, relations: $relations, after: $endCursor) { nodes { diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 1c08a0e253..5a4082eec3 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -80,7 +80,7 @@ export type GitLabGroupMembersResponse = { }; }; -export type GitLabSaasGroupsResponse = { +export type GitLabDescendantGroupsResponse = { errors: { message: string }[]; data: { group: { From 880c08c0b0b9afed2fe555692e28e8ce5344b00b Mon Sep 17 00:00:00 2001 From: Stephen Barry Date: Tue, 22 Aug 2023 08:33:19 +0100 Subject: [PATCH 035/131] Apply prettier lint and rename GitlabSaasGroupsResponse -> GitlabDescendantGroupsResponse Signed-off-by: Stephen Barry --- .../src/lib/client.ts | 48 +++++++++++-------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 3675d7e779..681c0c7603 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -22,7 +22,7 @@ import { import { Logger } from 'winston'; import { GitLabGroup, - GitLabSaasGroupsResponse, + GitLabDescendantGroupsResponse, GitLabGroupMembersResponse, GitLabUser, } from './types'; @@ -105,7 +105,7 @@ export class GitLabClient { let endCursor: string | null = null; do { - const response: GitLabSaasGroupsResponse = await fetch( + const response: GitLabDescendantGroupsResponse = await fetch( `${this.config.baseUrl}/api/graphql`, { method: 'POST', @@ -191,27 +191,37 @@ export class GitLabClient { }, body: JSON.stringify({ variables: { group: groupPath, relations: relations, endCursor }, - query: /* GraphQL */ `query getGroupMembers($group: ID!, $relations: [GroupMemberRelation!], $endCursor: String) { - group(fullPath: $group) { - groupMembers(first: 100, relations: $relations, after: $endCursor) { - nodes { - user { - id - username - publicEmail - name - state - webUrl - avatarUrl + query: /* GraphQL */ ` + query getGroupMembers( + $group: ID! + $relations: [GroupMemberRelation!] + $endCursor: String + ) { + group(fullPath: $group) { + groupMembers( + first: 100 + relations: $relations + after: $endCursor + ) { + nodes { + user { + id + username + publicEmail + name + state + webUrl + avatarUrl + } + } + pageInfo { + endCursor + hasNextPage } - } - pageInfo { - endCursor - hasNextPage } } } - }`, + `, }), }, ).then(r => r.json()); From 72611992911ab35f7d73ee91def4dda17c99564b Mon Sep 17 00:00:00 2001 From: Enrico Alvarenga Date: Thu, 24 Aug 2023 16:06:11 -0700 Subject: [PATCH 036/131] feat(scaffolder-backend): allow default kind and namespace args on parseEntityRef filter Signed-off-by: Enrico Alvarenga --- .../src/lib/templating/filters.ts | 10 +- .../tasks/NunjucksWorkflowRunner.test.ts | 131 +++++++++++++++--- 2 files changed, 119 insertions(+), 22 deletions(-) diff --git a/plugins/scaffolder-backend/src/lib/templating/filters.ts b/plugins/scaffolder-backend/src/lib/templating/filters.ts index 204d0015d8..0400adf159 100644 --- a/plugins/scaffolder-backend/src/lib/templating/filters.ts +++ b/plugins/scaffolder-backend/src/lib/templating/filters.ts @@ -27,7 +27,15 @@ export const createDefaultFilters = ({ }): Record => { return { parseRepoUrl: url => parseRepoUrl(url as string, integrations), - parseEntityRef: ref => parseEntityRef(ref as string), + parseEntityRef: ( + ref, + defaultKind?: JsonValue, + defaultNamespace?: JsonValue, + ) => + parseEntityRef(ref as string, { + defaultKind: defaultKind?.toString(), + defaultNamespace: defaultNamespace?.toString(), + }), pick: (obj: JsonValue, key: JsonValue) => get(obj, key as string), projectSlug: repoUrl => { const { owner, repo } = parseRepoUrl(repoUrl as string, integrations); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts index 5d4f7b73cc..bf40357cfe 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts @@ -713,32 +713,121 @@ describe('DefaultWorkflowRunner', () => { }); }); - it('provides the parseEntityRef filter', async () => { - const task = createMockTaskWithSpec({ - apiVersion: 'scaffolder.backstage.io/v1beta3', - steps: [ - { - id: 'test', - name: 'name', - action: 'output-action', - input: {}, + describe('parseEntityRef', () => { + it('parses entity ref', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'scaffolder.backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'output-action', + input: {}, + }, + ], + output: { + foo: '${{ parameters.entity | parseEntityRef }}', }, - ], - output: { - foo: '${{ parameters.entity | parseEntityRef }}', - }, - parameters: { - entity: 'component:default/ben', - }, + parameters: { + entity: 'component:default/ben', + }, + }); + + const { output } = await runner.execute(task); + + expect(output.foo).toEqual({ + kind: 'component', + namespace: 'default', + name: 'ben', + }); }); - const { output } = await runner.execute(task); + it('provides default kind for parsing entity ref', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'scaffolder.backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'output-action', + input: {}, + }, + ], + output: { + foo: "${{ parameters.entity | parseEntityRef('user') }}", + }, + parameters: { + entity: 'ben', + }, + }); - expect(output.foo).toEqual({ - kind: 'component', - namespace: 'default', - name: 'ben', + const { output } = await runner.execute(task); + + expect(output.foo).toEqual({ + kind: 'user', + namespace: 'default', + name: 'ben', + }); }); + + it('provides default kind and namespace for parsing entity ref', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'scaffolder.backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'output-action', + input: {}, + }, + ], + output: { + foo: "${{ parameters.entity | parseEntityRef('user', 'namespace-b') }}", + }, + parameters: { + entity: 'ben', + }, + }); + + const { output } = await runner.execute(task); + + expect(output.foo).toEqual({ + kind: 'user', + namespace: 'namespace-b', + name: 'ben', + }); + }); + + it.each(['undefined', 'null', 'None'])( + 'provides default namespace and kind as "%s" value for parsing entity ref', + async kind => { + const task = createMockTaskWithSpec({ + apiVersion: 'scaffolder.backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'output-action', + input: {}, + }, + ], + output: { + foo: `\${{ parameters.entity | parseEntityRef(${kind}, 'namespace-b') }}`, + }, + parameters: { + entity: 'resource:infra-workspace', + }, + }); + + const { output } = await runner.execute(task); + + expect(output.foo).toEqual({ + kind: 'resource', + namespace: 'namespace-b', + name: 'infra-workspace', + }); + }, + ); }); it('provides the pick filter', async () => { From b5f239b50bcf4814e5190de0637a65f7b79029c5 Mon Sep 17 00:00:00 2001 From: Enrico Alvarenga Date: Thu, 24 Aug 2023 16:33:52 -0700 Subject: [PATCH 037/131] chore: add changeset Signed-off-by: Enrico Alvarenga --- .changeset/tame-jokes-do.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .changeset/tame-jokes-do.md diff --git a/.changeset/tame-jokes-do.md b/.changeset/tame-jokes-do.md new file mode 100644 index 0000000000..02e597e57b --- /dev/null +++ b/.changeset/tame-jokes-do.md @@ -0,0 +1,20 @@ +--- +'@backstage/plugin-scaffolder-backend': minor +--- + +Improved the `parseEntityRef` Scaffolder filter by introducing the ability for users to provide default kind and namespace values. The filter now takes +3 arguments: + +1. Entity reference +2. (Optional) Default kind +3. (Optional) Default namespace + +So you can now provide default `kind` and/or `namespace`. Check out the following examples: + +- Without default values: `${{ parameters.entity | parseEntityRef | pick('name') }}` +- With default kind: `${{ parameters.entity | parseEntityRef('group') | pick('name') }}` +- With default kind and namespace: `${{ parameters.entity | parseEntityRef('group', 'another-namespace') | pick('name') }}` +- With default namespace: + - `${{ parameters.entity | parseEntityRef(null, 'another-namespace') | pick('name') }}` + - `${{ parameters.entity | parseEntityRef(undefined, 'another-namespace') | pick('name') }}` + - `${{ parameters.entity | parseEntityRef('', 'another-namespace') | pick('name') }}` From bc799fbaebc03c2a0be8dacf99fbd55d9b95d25c Mon Sep 17 00:00:00 2001 From: Enrico Alvarenga Date: Mon, 28 Aug 2023 10:24:43 -0700 Subject: [PATCH 038/131] refactor(scaffolder-backend): parseEntityRef filter takes context object arg Signed-off-by: Enrico Alvarenga --- .changeset/tame-jokes-do.md | 19 ++--- .../src/lib/templating/filters.ts | 13 +--- .../tasks/NunjucksWorkflowRunner.test.ts | 72 ++++++++++++++++--- 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/.changeset/tame-jokes-do.md b/.changeset/tame-jokes-do.md index 02e597e57b..35a508e692 100644 --- a/.changeset/tame-jokes-do.md +++ b/.changeset/tame-jokes-do.md @@ -2,19 +2,14 @@ '@backstage/plugin-scaffolder-backend': minor --- -Improved the `parseEntityRef` Scaffolder filter by introducing the ability for users to provide default kind and namespace values. The filter now takes -3 arguments: +Improved the `parseEntityRef` Scaffolder filter by introducing the ability for users to provide default kind and/or namespace values. The filter now takes +2 arguments, similarly to the original [parseEntityRef](<(https://github.com/backstage/backstage/blob/v1.17.2/packages/catalog-model/src/entity/ref.ts#L77)>): 1. Entity reference -2. (Optional) Default kind -3. (Optional) Default namespace +2. [Context optional object](https://github.com/backstage/backstage/blob/v1.17.2/packages/catalog-model/src/entity/ref.ts#L77) -So you can now provide default `kind` and/or `namespace`. Check out the following examples: +Check out the following examples: -- Without default values: `${{ parameters.entity | parseEntityRef | pick('name') }}` -- With default kind: `${{ parameters.entity | parseEntityRef('group') | pick('name') }}` -- With default kind and namespace: `${{ parameters.entity | parseEntityRef('group', 'another-namespace') | pick('name') }}` -- With default namespace: - - `${{ parameters.entity | parseEntityRef(null, 'another-namespace') | pick('name') }}` - - `${{ parameters.entity | parseEntityRef(undefined, 'another-namespace') | pick('name') }}` - - `${{ parameters.entity | parseEntityRef('', 'another-namespace') | pick('name') }}` +- `${{ parameters.entityRef | parseEntityRef | pick('name') }}` +- `${{ parameters.entityRef | parseEntityRef({ defaultKind:"group", defaultNamespace:"default" }) | pick('name') }}` +- `${{ parameters.entityRef | parseEntityRef({ defaultKind:"group" }) | pick('name') }}` diff --git a/plugins/scaffolder-backend/src/lib/templating/filters.ts b/plugins/scaffolder-backend/src/lib/templating/filters.ts index 0400adf159..81f326b889 100644 --- a/plugins/scaffolder-backend/src/lib/templating/filters.ts +++ b/plugins/scaffolder-backend/src/lib/templating/filters.ts @@ -15,7 +15,7 @@ */ import { parseEntityRef } from '@backstage/catalog-model'; import { ScmIntegrations } from '@backstage/integration'; -import { JsonValue } from '@backstage/types'; +import type { JsonObject, JsonValue } from '@backstage/types'; import { TemplateFilter } from '..'; import { parseRepoUrl } from '../../scaffolder/actions/builtin/publish/util'; import get from 'lodash/get'; @@ -27,15 +27,8 @@ export const createDefaultFilters = ({ }): Record => { return { parseRepoUrl: url => parseRepoUrl(url as string, integrations), - parseEntityRef: ( - ref, - defaultKind?: JsonValue, - defaultNamespace?: JsonValue, - ) => - parseEntityRef(ref as string, { - defaultKind: defaultKind?.toString(), - defaultNamespace: defaultNamespace?.toString(), - }), + parseEntityRef: (ref, context?) => + parseEntityRef(ref as string, context as JsonObject), pick: (obj: JsonValue, key: JsonValue) => get(obj, key as string), projectSlug: repoUrl => { const { owner, repo } = parseRepoUrl(repoUrl as string, integrations); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts index bf40357cfe..213270b109 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts @@ -754,7 +754,7 @@ describe('DefaultWorkflowRunner', () => { }, ], output: { - foo: "${{ parameters.entity | parseEntityRef('user') }}", + foo: `\${{ parameters.entity | parseEntityRef({ defaultKind:"user" }) }}`, }, parameters: { entity: 'ben', @@ -770,6 +770,34 @@ describe('DefaultWorkflowRunner', () => { }); }); + it('provides default namespace for parsing entity ref', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'scaffolder.backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'output-action', + input: {}, + }, + ], + output: { + foo: `\${{ parameters.entity | parseEntityRef({ defaultNamespace:"namespace-b" }) }}`, + }, + parameters: { + entity: 'user:ben', + }, + }); + + const { output } = await runner.execute(task); + + expect(output.foo).toEqual({ + kind: 'user', + namespace: 'namespace-b', + name: 'ben', + }); + }); + it('provides default kind and namespace for parsing entity ref', async () => { const task = createMockTaskWithSpec({ apiVersion: 'scaffolder.backstage.io/v1beta3', @@ -782,7 +810,7 @@ describe('DefaultWorkflowRunner', () => { }, ], output: { - foo: "${{ parameters.entity | parseEntityRef('user', 'namespace-b') }}", + foo: `\${{ parameters.entity | parseEntityRef({ defaultKind:"user", defaultNamespace:"namespace-b" }) }}`, }, parameters: { entity: 'ben', @@ -798,8 +826,8 @@ describe('DefaultWorkflowRunner', () => { }); }); - it.each(['undefined', 'null', 'None'])( - 'provides default namespace and kind as "%s" value for parsing entity ref', + it.each(['undefined', 'null', 'None', 'group', 0, '{}', '[]'])( + 'ignores invalid context "%s" for parsing entity refF', async kind => { const task = createMockTaskWithSpec({ apiVersion: 'scaffolder.backstage.io/v1beta3', @@ -812,22 +840,48 @@ describe('DefaultWorkflowRunner', () => { }, ], output: { - foo: `\${{ parameters.entity | parseEntityRef(${kind}, 'namespace-b') }}`, + foo: `\${{ parameters.entity | parseEntityRef(${kind}) }}`, }, parameters: { - entity: 'resource:infra-workspace', + entity: 'user:default/ben', }, }); const { output } = await runner.execute(task); expect(output.foo).toEqual({ - kind: 'resource', - namespace: 'namespace-b', - name: 'infra-workspace', + kind: 'user', + namespace: 'default', + name: 'ben', }); }, ); + + it('fails when unable to parse entity ref', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'scaffolder.backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'output-action', + input: {}, + }, + ], + output: { + foo: `\${{ parameters.entity | parseEntityRef({ defaultNamespace:"namespace-b" }) }}`, + }, + parameters: { + entity: 'ben', + }, + }); + + const { output } = await runner.execute(task); + + expect(output.foo).toEqual( + `\${{ parameters.entity | parseEntityRef({ defaultNamespace:"namespace-b" }) }}`, + ); + }); }); it('provides the pick filter', async () => { From d7403140aed62e63b09630ab8c5788f3defd06a4 Mon Sep 17 00:00:00 2001 From: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> Date: Tue, 29 Aug 2023 10:53:13 +0100 Subject: [PATCH 039/131] Update plugins/catalog-backend-module-gitlab/src/lib/client.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply suggestion Co-authored-by: Fredrik Adelöw Signed-off-by: sbarrypoppulo <101636916+sbarrypoppulo@users.noreply.github.com> --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 681c0c7603..4a5d70b0bf 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -187,7 +187,7 @@ export class GitLabClient { method: 'POST', headers: { ...getGitLabRequestOptions(this.config).headers, - ['Content-Type']: 'application/json', + 'Content-Type': 'application/json', }, body: JSON.stringify({ variables: { group: groupPath, relations: relations, endCursor }, From b75655659d5f175df3e4d8c264bcc5ddcf32ec64 Mon Sep 17 00:00:00 2001 From: Enrico Alvarenga Date: Tue, 29 Aug 2023 09:21:23 -0700 Subject: [PATCH 040/131] fix: include explicit types to function args Co-authored-by: Ben Lambert Signed-off-by: Enrico Alvarenga --- plugins/scaffolder-backend/src/lib/templating/filters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/lib/templating/filters.ts b/plugins/scaffolder-backend/src/lib/templating/filters.ts index 81f326b889..7808d6d136 100644 --- a/plugins/scaffolder-backend/src/lib/templating/filters.ts +++ b/plugins/scaffolder-backend/src/lib/templating/filters.ts @@ -27,7 +27,7 @@ export const createDefaultFilters = ({ }): Record => { return { parseRepoUrl: url => parseRepoUrl(url as string, integrations), - parseEntityRef: (ref, context?) => + parseEntityRef: (ref: JsonValue, context?: JsonValue) => parseEntityRef(ref as string, context as JsonObject), pick: (obj: JsonValue, key: JsonValue) => get(obj, key as string), projectSlug: repoUrl => { From 9665ecee847ebcce4113b9b1346f4d70ff148258 Mon Sep 17 00:00:00 2001 From: Enrico Alvarenga Date: Tue, 29 Aug 2023 10:45:58 -0700 Subject: [PATCH 041/131] docs(writing-templates): add built in filters section Signed-off-by: Enrico Alvarenga --- .changeset/tame-jokes-do.md | 11 +-- .../software-templates/writing-templates.md | 95 +++++++++++++++++++ 2 files changed, 96 insertions(+), 10 deletions(-) diff --git a/.changeset/tame-jokes-do.md b/.changeset/tame-jokes-do.md index 35a508e692..8e9b5a6056 100644 --- a/.changeset/tame-jokes-do.md +++ b/.changeset/tame-jokes-do.md @@ -3,13 +3,4 @@ --- Improved the `parseEntityRef` Scaffolder filter by introducing the ability for users to provide default kind and/or namespace values. The filter now takes -2 arguments, similarly to the original [parseEntityRef](<(https://github.com/backstage/backstage/blob/v1.17.2/packages/catalog-model/src/entity/ref.ts#L77)>): - -1. Entity reference -2. [Context optional object](https://github.com/backstage/backstage/blob/v1.17.2/packages/catalog-model/src/entity/ref.ts#L77) - -Check out the following examples: - -- `${{ parameters.entityRef | parseEntityRef | pick('name') }}` -- `${{ parameters.entityRef | parseEntityRef({ defaultKind:"group", defaultNamespace:"default" }) | pick('name') }}` -- `${{ parameters.entityRef | parseEntityRef({ defaultKind:"group" }) | pick('name') }}` +2 arguments, similarly to the original [parseEntityRef](https://github.com/backstage/backstage/blob/v1.17.2/packages/catalog-model/src/entity/ref.ts#L77). diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md index 67bcb6c179..3594c395eb 100644 --- a/docs/features/software-templates/writing-templates.md +++ b/docs/features/software-templates/writing-templates.md @@ -601,3 +601,98 @@ output things. You can grab that output using `steps.$stepId.output.$property`. You can read more about all the `inputs` and `outputs` defined in the actions in code part of the `JSONSchema`, or you can read more about our [built in actions](./builtin-actions.md). + +## Built in Filters + +Template filters are functions that help you transform data, extract specific information, +and perform various operations in Scaffolder Templates. + +This section introduces the built-in filters provided by Backstage and offers examples of +how to use them in the Scaffolder templates. It's important to mention that Backstage also leverages the +native filters from the Nunjucks library. For a complete list of these native filters and their usage, +refer to the [Nunjucks documentation](https://mozilla.github.io/nunjucks/templating.html#builtin-filters). + +### parseRepoUrl + +The `parseRepoUrl` filter parse a repository URL into +its components, such as `owner`, repository `name`, and more. + +**Usage Example:** + +```yaml +- id: log + name: Parse Repo URL + action: debug:log + input: + extra: ${{ parameters.repoUrl | parseRepoUrl }} +``` + +- **Input**: `https://github.com/backstage/backstage` +- **Output**: [RepoSpec](https://github.com/backstage/backstage/blob/v1.17.2/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts#L39) + +### parseEntityRef + +The `parseEntityRef` filter allows you to extract different parts of +an entity reference, such as the `kind`, `namespace`, and `name`. + +**Usage example** + +1. Without context + +```yaml +- id: log + name: Parse Entity Reference + action: debug:log + input: + extra: ${{ parameters.owner | parseEntityRef }} +``` + +- **Input**: `group:techdocs` +- **Output**: [CompoundEntityRef](https://github.com/backstage/backstage/blob/v1.17.2/packages/catalog-model/src/types.ts#L23) + +2. With context + +```yaml +- id: log + name: Parse Entity Reference + action: debug:log + input: + extra: ${{ parameters.owner | parseEntityRef({ defaultKind:"group", defaultNamespace:"another-namespace" }) }} +``` + +- **Input**: `techdocs` +- **Output**: [CompoundEntityRef](https://github.com/backstage/backstage/blob/v1.17.2/packages/catalog-model/src/types.ts#L23) + +### pick + +This `pick` filter allows you to select specific properties from an object. + +**Usage Example** + +```yaml +- id: log + name: Pick + action: debug:log + input: + extra: ${{ parameters.owner | parseEntityRef | pick('name') }} +``` + +- **Input**: `group:techdocs` +- **Output**: `techndocs` + +### projectSlug + +The `projectSlug` filter generates a project slug from a repository URL + +**Usage Example** + +```yaml +- id: log + name: Project Slug + action: debug:log + input: + extra: ${{ parameters.repoUrl | projectSlug }} +``` + +- **Input**: `https://github.com/backstage/backstage` +- **Output**: `backstage/backstage` From 69405c52f8d1d72726994705e41798616b426110 Mon Sep 17 00:00:00 2001 From: angom1 Date: Thu, 31 Aug 2023 12:29:12 +0100 Subject: [PATCH 042/131] Simplify for loops Signed-off-by: angom1 --- .../catalog-backend-module-gitlab/src/lib/client.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 4a5d70b0bf..65eb3f326a 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -150,11 +150,7 @@ export class GitLabClient { continue; } - const groupsData = response.data.group.descendantGroups.nodes; - - for (let i = 0; i < groupsData.length; i++) { - const groupItem = groupsData[i]; - + for (const groupItem of response.data.group.descendantGroups.nodes) { const formattedGroupResponse = { id: Number(groupItem.id.replace(/^gid:\/\/gitlab\/Group\//, '')), name: groupItem.name, @@ -236,11 +232,7 @@ export class GitLabClient { continue; } - const usersData = response.data.group.groupMembers.nodes; - - for (let i = 0; i < usersData.length; i++) { - const userItem = usersData[i]; - + for (const userItem of response.data.group.groupMembers.nodes) { const formattedUserResponse = { id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), username: userItem.user.username, From 1d8b7a51ea9a4f21ab602e67d54580f7a095e8af Mon Sep 17 00:00:00 2001 From: angom1 Date: Fri, 1 Sep 2023 11:11:43 +0100 Subject: [PATCH 043/131] Switch to commitEmail Signed-off-by: angom1 --- .../catalog-backend-module-gitlab/src/lib/client.test.ts | 8 ++++---- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 2 +- plugins/catalog-backend-module-gitlab/src/lib/types.ts | 2 +- .../providers/GitlabOrgDiscoveryEntityProvider.test.ts | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts index 2bb1f713b5..dda30fd86a 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.test.ts @@ -413,7 +413,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - publicEmail: 'user1@example.com', + commitEmail: 'user1@example.com', name: 'user1', state: 'active', webUrl: 'user1.com', @@ -517,7 +517,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - publicEmail: 'user1@example.com', + commitEmail: 'user1@example.com', name: 'user1', state: 'active', webUrl: 'user1.com', @@ -530,7 +530,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/2', username: 'user2', - publicEmail: 'user2@example.com', + commitEmail: 'user2@example.com', name: 'user2', state: 'active', webUrl: 'user2.com', @@ -764,7 +764,7 @@ describe('GitLabClient', () => { user: { id: 'gid://gitlab/User/1', username: 'user1', - publicEmail: 'user1@example.com', + commitEmail: 'user1@example.com', name: 'user1', state: 'active', webUrl: 'user1.com', diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 65eb3f326a..26d1b2a6fb 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -236,7 +236,7 @@ export class GitLabClient { const formattedUserResponse = { id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), username: userItem.user.username, - email: userItem.user.publicEmail, + email: userItem.user.commitEmail, name: userItem.user.name, state: userItem.user.state, web_url: userItem.user.webUrl, diff --git a/plugins/catalog-backend-module-gitlab/src/lib/types.ts b/plugins/catalog-backend-module-gitlab/src/lib/types.ts index 5a4082eec3..a339ac57e4 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/types.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/types.ts @@ -64,7 +64,7 @@ export type GitLabGroupMembersResponse = { user: { id: string; username: string; - publicEmail: string; + commitEmail: string; name: string; state: string; webUrl: string; diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts index 1b1357c1bf..ce00510140 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.test.ts @@ -560,7 +560,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { user: { id: 'gid://gitlab/User/12', username: 'testuser1', - publicEmail: 'testuser1@example.com', + commitEmail: 'testuser1@example.com', state: 'active', name: 'Test User 1', webUrl: 'https://gitlab.com/testuser1', @@ -571,7 +571,7 @@ describe('GitlabOrgDiscoveryEntityProvider', () => { user: { id: 'gid://gitlab/User/34', username: 'testuser2', - publicEmail: 'testuser2@example.com', + commitEmail: 'testuser2@example.com', state: 'active', name: 'Test User 2', webUrl: 'https://gitlab.com/testuser2', From 13f3b3491ec3772c69c21dd20c7410665b544553 Mon Sep 17 00:00:00 2001 From: Oleg S <97077423+RobotSail@users.noreply.github.com> Date: Thu, 31 Aug 2023 16:07:55 -0400 Subject: [PATCH 044/131] chore: update props types to use PropsWithChildren Signed-off-by: Oleg S <97077423+RobotSail@users.noreply.github.com> --- .../core-app-api/src/app/AppContext.test.tsx | 2 +- .../src/routing/RoutingProvider.beta.test.tsx | 2 +- .../routing/RoutingProvider.stable.test.tsx | 2 +- packages/core-components/api-report.md | 4 +-- .../src/components/Link/Link.test.tsx | 4 ++- .../LogViewer/useLogViewerSelection.test.tsx | 2 +- .../layout/ErrorBoundary/ErrorBoundary.tsx | 4 +-- .../src/layout/HeaderLabel/HeaderLabel.tsx | 6 ++--- .../src/layout/HeaderTabs/HeaderTabs.test.tsx | 26 ++++++++++--------- .../src/extensions/PluginErrorBoundary.tsx | 4 +-- .../plugin-options/usePluginOptions.test.tsx | 2 +- .../src/routing/useRouteRef.test.tsx | 10 +++---- packages/test-utils/api-report.md | 3 ++- .../test-utils/src/testUtils/appWrappers.tsx | 9 +++++-- .../src/lib/VersionedContext.test.tsx | 4 +-- .../EntityRelationsGraph.test.tsx | 2 +- .../src/hooks/useEntity.test.tsx | 20 +++++++------- .../src/hooks/useRelatedEntities.test.tsx | 4 +-- .../src/components/Card/Card.tsx | 4 ++- .../InfoCardHeader/InfoCardHeader.tsx | 2 +- .../src/components/Wrapper/Wrapper.tsx | 4 ++- .../src/next/hooks/useTemplateSchema.test.tsx | 10 +++---- .../hooks/useTransformSchemaToProps.test.tsx | 4 +-- .../src/secrets/SecretsContext.test.tsx | 2 +- .../SearchModal/useSearchModal.test.tsx | 2 +- 25 files changed, 77 insertions(+), 61 deletions(-) diff --git a/packages/core-app-api/src/app/AppContext.test.tsx b/packages/core-app-api/src/app/AppContext.test.tsx index a27b7c6f35..ee7b90533f 100644 --- a/packages/core-app-api/src/app/AppContext.test.tsx +++ b/packages/core-app-api/src/app/AppContext.test.tsx @@ -40,7 +40,7 @@ describe('v1 consumer', () => { }; const renderedHook = renderHook(() => useMockAppV1(), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( ), }); diff --git a/packages/core-app-api/src/routing/RoutingProvider.beta.test.tsx b/packages/core-app-api/src/routing/RoutingProvider.beta.test.tsx index 532be8c09d..c3a5bab89e 100644 --- a/packages/core-app-api/src/routing/RoutingProvider.beta.test.tsx +++ b/packages/core-app-api/src/routing/RoutingProvider.beta.test.tsx @@ -353,7 +353,7 @@ describe('v1 consumer', () => { initialProps: { routeRef: routeRef1 as AnyRouteRef, }, - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( , string>([ diff --git a/packages/core-app-api/src/routing/RoutingProvider.stable.test.tsx b/packages/core-app-api/src/routing/RoutingProvider.stable.test.tsx index 24101dcf06..84a33842c9 100644 --- a/packages/core-app-api/src/routing/RoutingProvider.stable.test.tsx +++ b/packages/core-app-api/src/routing/RoutingProvider.stable.test.tsx @@ -385,7 +385,7 @@ describe('v1 consumer', () => { initialProps: { routeRef: routeRef1 as AnyRouteRef, }, - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( , string>([ diff --git a/packages/core-components/api-report.md b/packages/core-components/api-report.md index 54c26039a3..0aade4549e 100644 --- a/packages/core-components/api-report.md +++ b/packages/core-components/api-report.md @@ -347,10 +347,10 @@ export type EmptyStateImageClassKey = 'generalImg'; export const ErrorBoundary: ComponentClass; // @public (undocumented) -export type ErrorBoundaryProps = { +export type ErrorBoundaryProps = React_2.PropsWithChildren<{ slackChannel?: string | SlackChannel; onError?: (error: Error, errorInfo: string) => null; -}; +}>; // Warning: (ae-forgotten-export) The symbol "IErrorPageProps" needs to be exported by the entry point index.d.ts // diff --git a/packages/core-components/src/components/Link/Link.test.tsx b/packages/core-components/src/components/Link/Link.test.tsx index 682c4bea8a..1e39d14009 100644 --- a/packages/core-components/src/components/Link/Link.test.tsx +++ b/packages/core-components/src/components/Link/Link.test.tsx @@ -134,7 +134,9 @@ describe('', () => { }); describe('useResolvedPath', () => { - const wrapper: WrapperComponent<{}> = ({ children }) => { + const wrapper: WrapperComponent> = ({ + children, + }) => { const configApi = new ConfigReader({ app: { baseUrl: 'http://localhost:3000/example' }, }); diff --git a/packages/core-components/src/components/LogViewer/useLogViewerSelection.test.tsx b/packages/core-components/src/components/LogViewer/useLogViewerSelection.test.tsx index 93a6dfd7e9..a828818ed5 100644 --- a/packages/core-components/src/components/LogViewer/useLogViewerSelection.test.tsx +++ b/packages/core-components/src/components/LogViewer/useLogViewerSelection.test.tsx @@ -40,7 +40,7 @@ const lines = [ describe('useLogViewerSelection', () => { it('should manage a selection', () => { const rendered = renderHook(() => useLogViewerSelection(lines), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( {children} diff --git a/packages/core-components/src/layout/ErrorBoundary/ErrorBoundary.tsx b/packages/core-components/src/layout/ErrorBoundary/ErrorBoundary.tsx index 037e170640..0ecaedfc19 100644 --- a/packages/core-components/src/layout/ErrorBoundary/ErrorBoundary.tsx +++ b/packages/core-components/src/layout/ErrorBoundary/ErrorBoundary.tsx @@ -25,10 +25,10 @@ type SlackChannel = { }; /** @public */ -export type ErrorBoundaryProps = { +export type ErrorBoundaryProps = React.PropsWithChildren<{ slackChannel?: string | SlackChannel; onError?: (error: Error, errorInfo: string) => null; -}; +}>; type State = { error?: Error; diff --git a/packages/core-components/src/layout/HeaderLabel/HeaderLabel.tsx b/packages/core-components/src/layout/HeaderLabel/HeaderLabel.tsx index 0ea2e163a7..e6ca9b720e 100644 --- a/packages/core-components/src/layout/HeaderLabel/HeaderLabel.tsx +++ b/packages/core-components/src/layout/HeaderLabel/HeaderLabel.tsx @@ -18,7 +18,7 @@ import { BackstageTheme } from '@backstage/theme'; import Grid from '@material-ui/core/Grid'; import { alpha, makeStyles } from '@material-ui/core/styles'; import Typography from '@material-ui/core/Typography'; -import React from 'react'; +import React, { PropsWithChildren } from 'react'; import { Link } from '../../components/Link'; /** @public */ @@ -46,10 +46,10 @@ const useStyles = makeStyles( { name: 'BackstageHeaderLabel' }, ); -type HeaderLabelContentProps = { +type HeaderLabelContentProps = PropsWithChildren<{ value: React.ReactNode; className: string; -}; +}>; const HeaderLabelContent = ({ value, className }: HeaderLabelContentProps) => { return ( diff --git a/packages/core-components/src/layout/HeaderTabs/HeaderTabs.test.tsx b/packages/core-components/src/layout/HeaderTabs/HeaderTabs.test.tsx index a6bfc79040..ce5c572eef 100644 --- a/packages/core-components/src/layout/HeaderTabs/HeaderTabs.test.tsx +++ b/packages/core-components/src/layout/HeaderTabs/HeaderTabs.test.tsx @@ -55,18 +55,20 @@ describe('', () => { }, })); - const TextualBadge = React.forwardRef((props, ref) => ( - - - {props.children} - - - )); + const TextualBadge = React.forwardRef( + (props: React.PropsWithChildren<{}>, ref) => ( + + + {props.children} + + + ), + ); const iconTab = [ { id: 'icon-tab', diff --git a/packages/core-plugin-api/src/extensions/PluginErrorBoundary.tsx b/packages/core-plugin-api/src/extensions/PluginErrorBoundary.tsx index 215ebc8470..51ddecdf71 100644 --- a/packages/core-plugin-api/src/extensions/PluginErrorBoundary.tsx +++ b/packages/core-plugin-api/src/extensions/PluginErrorBoundary.tsx @@ -18,10 +18,10 @@ import React from 'react'; import { AppContext } from '../app/types'; import { BackstagePlugin } from '../plugin'; -type Props = { +type Props = React.PropsWithChildren<{ app: AppContext; plugin: BackstagePlugin; -}; +}>; type State = { error: Error | undefined }; diff --git a/packages/core-plugin-api/src/plugin-options/usePluginOptions.test.tsx b/packages/core-plugin-api/src/plugin-options/usePluginOptions.test.tsx index 11bf85aafa..96691d503b 100644 --- a/packages/core-plugin-api/src/plugin-options/usePluginOptions.test.tsx +++ b/packages/core-plugin-api/src/plugin-options/usePluginOptions.test.tsx @@ -38,7 +38,7 @@ describe('usePluginOptions', () => { }); const rendered = renderHook(() => usePluginOptions(), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( {children} ), }); diff --git a/packages/core-plugin-api/src/routing/useRouteRef.test.tsx b/packages/core-plugin-api/src/routing/useRouteRef.test.tsx index 618f40d281..7a03a1cdc4 100644 --- a/packages/core-plugin-api/src/routing/useRouteRef.test.tsx +++ b/packages/core-plugin-api/src/routing/useRouteRef.test.tsx @@ -36,7 +36,7 @@ describe('v1 consumer', () => { const routeRef = createRouteRef({ id: 'ref1' }); const renderedHook = renderHook(() => useRouteRef(routeRef), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( ), }); @@ -60,7 +60,7 @@ describe('v1 consumer', () => { history.push('/my-page'); const { rerender } = renderHook(() => useRouteRef(routeRef), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( { history.push('/my-page'); const { rerender } = renderHook(() => useRouteRef(routeRef), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( { history.push('/my-page'); const { rerender } = renderHook(() => useRouteRef(routeRef), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( { history.push('/my-page'); const { rerender } = renderHook(() => useRouteRef(routeRef), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( > | ReactNode, options?: TestAppOptions, ): Promise; diff --git a/packages/test-utils/src/testUtils/appWrappers.tsx b/packages/test-utils/src/testUtils/appWrappers.tsx index 6545cc312a..1628900b73 100644 --- a/packages/test-utils/src/testUtils/appWrappers.tsx +++ b/packages/test-utils/src/testUtils/appWrappers.tsx @@ -14,7 +14,12 @@ * limitations under the License. */ -import React, { ComponentType, ReactNode, ReactElement } from 'react'; +import React, { + ComponentType, + ReactNode, + ReactElement, + PropsWithChildren, +} from 'react'; import { MemoryRouter } from 'react-router-dom'; import { Route } from 'react-router-dom'; import { UnifiedThemeProvider, themes } from '@backstage/theme'; @@ -226,7 +231,7 @@ export function wrapInTestApp( * @public */ export async function renderInTestApp( - Component: ComponentType | ReactNode, + Component: ComponentType> | ReactNode, options: TestAppOptions = {}, ): Promise { let wrappedElement: React.ReactElement; diff --git a/packages/version-bridge/src/lib/VersionedContext.test.tsx b/packages/version-bridge/src/lib/VersionedContext.test.tsx index 659fd9b5b9..275e491324 100644 --- a/packages/version-bridge/src/lib/VersionedContext.test.tsx +++ b/packages/version-bridge/src/lib/VersionedContext.test.tsx @@ -30,7 +30,7 @@ describe('VersionedContext', () => { const Context = createVersionedContext('test-context-1'); const rendered = renderHook(() => useContext(Context), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( @@ -47,7 +47,7 @@ describe('VersionedContext', () => { const Context = createVersionedContext('test-context-2'); const rendered = renderHook(() => useVersionedContext('test-context-2'), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx index 8a3352abb4..2556cd06d7 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx @@ -30,7 +30,7 @@ import React, { FunctionComponent } from 'react'; import { EntityRelationsGraph } from './EntityRelationsGraph'; describe('', () => { - let Wrapper: FunctionComponent; + let Wrapper: FunctionComponent>; const entities: { [ref: string]: Entity } = { 'b:d/c': { apiVersion: 'a', diff --git a/plugins/catalog-react/src/hooks/useEntity.test.tsx b/plugins/catalog-react/src/hooks/useEntity.test.tsx index 3a1e5010d3..8fc388654a 100644 --- a/plugins/catalog-react/src/hooks/useEntity.test.tsx +++ b/plugins/catalog-react/src/hooks/useEntity.test.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React from 'react'; +import React, { PropsWithChildren } from 'react'; import { renderHook } from '@testing-library/react-hooks'; import { useEntity, @@ -32,7 +32,9 @@ const entity = { metadata: { name: 'my-entity' }, kind: 'MyKind' } as Entity; describe('useEntity', () => { it('should throw if no entity is provided', async () => { const { result } = renderHook(() => useEntity(), { - wrapper: ({ children }) => , + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( + + ), }); expect(result.error?.message).toMatch(/entity has not been loaded/); @@ -40,7 +42,7 @@ describe('useEntity', () => { it('should provide an entity', async () => { const { result } = renderHook(() => useEntity(), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( ), }); @@ -52,7 +54,7 @@ describe('useEntity', () => { const analyticsSpy = new MockAnalyticsApi(); const apis = TestApiRegistry.from([analyticsApiRef, analyticsSpy]); const { result } = renderHook(() => useAnalytics(), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( @@ -70,7 +72,7 @@ describe('useEntity', () => { describe('useAsyncEntity', () => { it('should provide no entity', async () => { const { result } = renderHook(() => useAsyncEntity(), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( ), }); @@ -84,7 +86,7 @@ describe('useAsyncEntity', () => { it('should provide an entity', async () => { const refresh = () => {}; const { result } = renderHook(() => useAsyncEntity(), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( { it('should provide an error', async () => { const error = new Error('oh no'); const { result } = renderHook(() => useAsyncEntity(), { - wrapper: ({ children }) => ( + wrapper: ({ children }: PropsWithChildren<{}>) => ( { const analyticsSpy = new MockAnalyticsApi(); const apis = TestApiRegistry.from([analyticsApiRef, analyticsSpy]); const { result } = renderHook(() => useAnalytics(), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( { const analyticsSpy = new MockAnalyticsApi(); const apis = TestApiRegistry.from([analyticsApiRef, analyticsSpy]); const { result } = renderHook(() => useAnalytics(), { - wrapper: ({ children }) => ( + wrapper: ({ children }: PropsWithChildren<{}>) => ( diff --git a/plugins/catalog-react/src/hooks/useRelatedEntities.test.tsx b/plugins/catalog-react/src/hooks/useRelatedEntities.test.tsx index 0d415274e0..8de7822d59 100644 --- a/plugins/catalog-react/src/hooks/useRelatedEntities.test.tsx +++ b/plugins/catalog-react/src/hooks/useRelatedEntities.test.tsx @@ -17,7 +17,7 @@ import { Entity } from '@backstage/catalog-model'; import { TestApiProvider } from '@backstage/test-utils'; import { WrapperComponent, renderHook } from '@testing-library/react-hooks'; -import React from 'react'; +import React, { PropsWithChildren } from 'react'; import { catalogApiRef } from '../api'; import { useRelatedEntities } from './useRelatedEntities'; @@ -50,7 +50,7 @@ describe('useRelatedEntities', () => { getEntitiesByRefs: jest.fn(), }; - const wrapper: WrapperComponent<{}> = ({ children }) => { + const wrapper: WrapperComponent> = ({ children }) => { return ( {children} diff --git a/plugins/github-pull-requests-board/src/components/Card/Card.tsx b/plugins/github-pull-requests-board/src/components/Card/Card.tsx index 320e166358..4fe833fa67 100644 --- a/plugins/github-pull-requests-board/src/components/Card/Card.tsx +++ b/plugins/github-pull-requests-board/src/components/Card/Card.tsx @@ -31,7 +31,9 @@ type Props = { labels?: Label[]; }; -const Card: FunctionComponent = (props: PropsWithChildren) => { +const Card: FunctionComponent> = ( + props: React.PropsWithChildren, +) => { const { title, createdAt, diff --git a/plugins/github-pull-requests-board/src/components/InfoCardHeader/InfoCardHeader.tsx b/plugins/github-pull-requests-board/src/components/InfoCardHeader/InfoCardHeader.tsx index 8c126bcaa4..004cf7c5ae 100644 --- a/plugins/github-pull-requests-board/src/components/InfoCardHeader/InfoCardHeader.tsx +++ b/plugins/github-pull-requests-board/src/components/InfoCardHeader/InfoCardHeader.tsx @@ -21,7 +21,7 @@ type Props = { onRefresh: () => void; }; -const InfoCardHeader: FunctionComponent = ( +const InfoCardHeader: FunctionComponent> = ( props: PropsWithChildren, ) => { const { children, onRefresh } = props; diff --git a/plugins/github-pull-requests-board/src/components/Wrapper/Wrapper.tsx b/plugins/github-pull-requests-board/src/components/Wrapper/Wrapper.tsx index 6d8f17cb72..bf5fa157fc 100644 --- a/plugins/github-pull-requests-board/src/components/Wrapper/Wrapper.tsx +++ b/plugins/github-pull-requests-board/src/components/Wrapper/Wrapper.tsx @@ -20,7 +20,9 @@ type Props = { fullscreen: boolean; }; -const Wrapper: FunctionComponent = (props: PropsWithChildren) => { +const Wrapper: FunctionComponent> = ( + props: PropsWithChildren, +) => { const { children, fullscreen } = props; return ( diff --git a/plugins/scaffolder-react/src/next/hooks/useTemplateSchema.test.tsx b/plugins/scaffolder-react/src/next/hooks/useTemplateSchema.test.tsx index d257d69c59..9a0732442c 100644 --- a/plugins/scaffolder-react/src/next/hooks/useTemplateSchema.test.tsx +++ b/plugins/scaffolder-react/src/next/hooks/useTemplateSchema.test.tsx @@ -50,7 +50,7 @@ describe('useTemplateSchema', () => { }; const { result } = renderHook(() => useTemplateSchema(manifest), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( false }]]} > @@ -117,7 +117,7 @@ describe('useTemplateSchema', () => { }; const { result } = renderHook(() => useTemplateSchema(manifest), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( false }]]} > @@ -161,7 +161,7 @@ describe('useTemplateSchema', () => { }; const { result } = renderHook(() => useTemplateSchema(manifest), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( true }]]} > @@ -212,7 +212,7 @@ describe('useTemplateSchema', () => { }; const { result } = renderHook(() => useTemplateSchema(manifest), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( false }]]} > @@ -250,7 +250,7 @@ describe('useTemplateSchema', () => { }; const { result } = renderHook(() => useTemplateSchema(manifest), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( false }]]} > diff --git a/plugins/scaffolder-react/src/next/hooks/useTransformSchemaToProps.test.tsx b/plugins/scaffolder-react/src/next/hooks/useTransformSchemaToProps.test.tsx index 6dba7b9f29..7ee5cbfdfd 100644 --- a/plugins/scaffolder-react/src/next/hooks/useTransformSchemaToProps.test.tsx +++ b/plugins/scaffolder-react/src/next/hooks/useTransformSchemaToProps.test.tsx @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React from 'react'; +import React, { PropsWithChildren } from 'react'; import { renderHook } from '@testing-library/react-hooks'; import { TestApiProvider } from '@backstage/test-utils'; import { type ParsedTemplateSchema } from './useTemplateSchema'; @@ -39,7 +39,7 @@ describe('useTransformSchemaToProps', () => { const { result } = renderHook( () => useTransformSchemaToProps(step, { layouts }), { - wrapper: ({ children }) => ( + wrapper: ({ children }: PropsWithChildren<{}>) => ( {children} ), }, diff --git a/plugins/scaffolder-react/src/secrets/SecretsContext.test.tsx b/plugins/scaffolder-react/src/secrets/SecretsContext.test.tsx index 593fbd908b..553f27d2a9 100644 --- a/plugins/scaffolder-react/src/secrets/SecretsContext.test.tsx +++ b/plugins/scaffolder-react/src/secrets/SecretsContext.test.tsx @@ -24,7 +24,7 @@ describe('SecretsContext', () => { hook: useTemplateSecrets(), }), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( {children} ), }, diff --git a/plugins/search/src/components/SearchModal/useSearchModal.test.tsx b/plugins/search/src/components/SearchModal/useSearchModal.test.tsx index 0de41e3589..e21f864ac2 100644 --- a/plugins/search/src/components/SearchModal/useSearchModal.test.tsx +++ b/plugins/search/src/components/SearchModal/useSearchModal.test.tsx @@ -85,7 +85,7 @@ describe('useSearchModal', () => { const history = createMemoryHistory({ initialEntries: ['/'] }); const rendered = renderHook(() => useSearchModal(true), { - wrapper: ({ children }) => ( + wrapper: ({ children }: React.PropsWithChildren<{}>) => ( {children} From fe25880fbc176cbd5fec6942dd3c2336a658eb96 Mon Sep 17 00:00:00 2001 From: fyyyyy Date: Tue, 15 Aug 2023 19:08:35 +0200 Subject: [PATCH 045/131] Feature: add zip download to dry run results page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See issue #19318 Signed-off-by: fyyyyy Signed-off-by: Fredrik Adelöw --- .../DryRunResults/DryRunResultsList.tsx | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx index 462c3b58fd..9163261985 100644 --- a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx +++ b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx @@ -25,6 +25,7 @@ import { makeStyles } from '@material-ui/core/styles'; import CancelIcon from '@material-ui/icons/Cancel'; import CheckIcon from '@material-ui/icons/Check'; import DeleteIcon from '@material-ui/icons/Delete'; +import DownloadIcon from '@material-ui/icons/GetApp'; import React from 'react'; import { useDryRun } from '../DryRunContext'; @@ -67,9 +68,18 @@ export function DryRunResultsList() { + dryRun.downloadResult(result.id)} + > + + dryRun.deleteResult(result.id)} > From d59fd8cf06c78c77c491acc72892b91358f96ba1 Mon Sep 17 00:00:00 2001 From: fyyyyy Date: Tue, 15 Aug 2023 19:10:14 +0200 Subject: [PATCH 046/131] Update DryRunContext.tsx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add zip download fn Signed-off-by: fyyyyy Signed-off-by: Fredrik Adelöw --- .../TemplateEditorPage/DryRunContext.tsx | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx index a617eebe1e..47d121b616 100644 --- a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx +++ b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx @@ -26,6 +26,7 @@ import React, { useRef, useState, } from 'react'; +import * as zip from "@zip.js/zip.js"; import { scaffolderApiRef, ScaffolderDryRunResponse, @@ -50,6 +51,7 @@ interface DryRun { selectResult(id: number): void; deleteResult(id: number): void; + downloadResult(id: number): void; execute(options: DryRunOptions): Promise; } @@ -122,6 +124,22 @@ export function DryRunProvider(props: DryRunProviderProps) { }); }, []); + const downloadResult = useCallback((id: number) => { + setState(prevState => { + const index = prevState.results.findIndex(r => r.id === id); + if (index === -1) { + return prevState; + } + const result = prevState.results[index]; + console.log('Result', result.id); + createZipDownload(result.directoryContents, 'result_' + result.id) + return { + results: prevState.results, + selectedResult: prevState.selectedResult, + }; + }); + }, []); + const execute = useCallback( async (options: DryRunOptions) => { if (!scaffolderApi.dryRun) { @@ -158,6 +176,7 @@ export function DryRunProvider(props: DryRunProviderProps) { ...state, selectResult, deleteResult, + downloadResult, execute, }), [state, selectResult, deleteResult, execute], @@ -177,3 +196,36 @@ export function useDryRun(): DryRun { } return value; } + +async function createZipDownload(directoryContents: { path: string; base64Content: string; executable: boolean; }[], name: string) { + // needs zip.js + + // Creates a BlobWriter object where the zip content will be written. + const zipFileWriter = new zip.BlobWriter(); + + // Creates a ZipWriter object writing data via `zipFileWriter`, adds the entry + const zipWriter = new zip.ZipWriter(zipFileWriter); + + for (const d of directoryContents) { + // Decode text content from base64 to ascii + const converted = atob(d.base64Content); + + // Creates a TextReader object storing the text of the entry to add in the zip (i.e. "Hello world!"). + const fileReader = new zip.TextReader(converted); + + await zipWriter.add(d.path, fileReader); + } + + // Closes the writer. + await zipWriter.close(); + + // Retrieves the Blob object containing the zip content into `zipFileBlob` + const zipFileBlob = await zipFileWriter.getData(); + + // Download zip + const a = document.createElement('a'); + a.href = URL.createObjectURL(zipFileBlob); + a.download = `dry-run-${name}.zip`; + a.click(); +} + From 740b5bb8d4aa6ece15573b5859cac6a757807831 Mon Sep 17 00:00:00 2001 From: fyyyyy Date: Tue, 15 Aug 2023 19:11:00 +0200 Subject: [PATCH 047/131] add zip.js to package.json for scaffolder plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: fyyyyy Signed-off-by: Fredrik Adelöw --- plugins/scaffolder/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 47e628bfcb..f5cf5eb6bc 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -73,6 +73,7 @@ "@rjsf/validator-ajv8": "5.7.3", "@types/react": "^16.13.1 || ^17.0.0", "@uiw/react-codemirror": "^4.9.3", + "@zip.js/zip.js": "^2.7.24", "classnames": "^2.2.6", "event-source-polyfill": "^1.0.31", "git-url-parse": "^13.0.0", From 0119c326394a0db03a1b09dc9f4d055071bebc96 Mon Sep 17 00:00:00 2001 From: fyyyyy Date: Tue, 15 Aug 2023 19:19:56 +0200 Subject: [PATCH 048/131] Add changeset three-hounds-wonder.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add changeset Signed-off-by: fyyyyy Signed-off-by: Fredrik Adelöw --- .changeset/three-hounds-wonder.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/three-hounds-wonder.md diff --git a/.changeset/three-hounds-wonder.md b/.changeset/three-hounds-wonder.md new file mode 100644 index 0000000000..fd4cb486bc --- /dev/null +++ b/.changeset/three-hounds-wonder.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': minor +--- + +adding a .zip download to dry run results page, including zip.js as dependency From 525d3c8b67e856408cb8949ed0a091fe5c19d75d Mon Sep 17 00:00:00 2001 From: fyyyyy Date: Tue, 15 Aug 2023 19:24:06 +0200 Subject: [PATCH 049/131] Update yarn.lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: fyyyyy Signed-off-by: Fredrik Adelöw --- yarn.lock | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/yarn.lock b/yarn.lock index 29953fc257..00e3730ac1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8796,6 +8796,7 @@ __metadata: "@types/json-schema": ^7.0.9 "@types/react": ^16.13.1 || ^17.0.0 "@uiw/react-codemirror": ^4.9.3 + "@zip.js/zip.js": ^2.7.24 classnames: ^2.2.6 event-source-polyfill: ^1.0.31 git-url-parse: ^13.0.0 @@ -19346,6 +19347,13 @@ __metadata: languageName: node linkType: hard +"@zip.js/zip.js@npm:^2.7.24": + version: 2.7.28 + resolution: "@zip.js/zip.js@npm:2.7.28" + checksum: 2447e7250daeb0a518c68347034075ad244e8e53e3ee83a2e3e078f919328edcc24a6c68f1814d42e231526230360164cce79d70f14cb9a16a07e91ae1e25257 + languageName: node + linkType: hard + "@zxing/text-encoding@npm:0.9.0": version: 0.9.0 resolution: "@zxing/text-encoding@npm:0.9.0" From 400beacb7072fdc529b4405235d877bfcca8283d Mon Sep 17 00:00:00 2001 From: TRIGUBF Date: Wed, 23 Aug 2023 16:50:35 +0200 Subject: [PATCH 050/131] revoke url, remove anchor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../src/components/TemplateEditorPage/DryRunContext.tsx | 2 ++ yarn.lock | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx index 47d121b616..530b9f5be1 100644 --- a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx +++ b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx @@ -227,5 +227,7 @@ async function createZipDownload(directoryContents: { path: string; base64Conten a.href = URL.createObjectURL(zipFileBlob); a.download = `dry-run-${name}.zip`; a.click(); + URL.revokeObjectURL(a.href); + a.remove(); } diff --git a/yarn.lock b/yarn.lock index 00e3730ac1..f7e0fb0f75 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19348,9 +19348,9 @@ __metadata: linkType: hard "@zip.js/zip.js@npm:^2.7.24": - version: 2.7.28 - resolution: "@zip.js/zip.js@npm:2.7.28" - checksum: 2447e7250daeb0a518c68347034075ad244e8e53e3ee83a2e3e078f919328edcc24a6c68f1814d42e231526230360164cce79d70f14cb9a16a07e91ae1e25257 + version: 2.7.24 + resolution: "@zip.js/zip.js@npm:2.7.24" + checksum: f56f973748d063158cbb6f07ccf4d4c64d35952bb2ecf3d9922db935b17f4e689b6d8568041083195ff4679a0d1a9378fa78adca7428819239d4a3fb4a1dd2d2 languageName: node linkType: hard From 149c6f03ca3707a49ffc74b06602b983ccb6600d Mon Sep 17 00:00:00 2001 From: TRIGUBF Date: Wed, 23 Aug 2023 17:06:22 +0200 Subject: [PATCH 051/131] replace zip.js with JSzip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- plugins/scaffolder/package.json | 2 +- .../TemplateEditorPage/DryRunContext.tsx | 31 ++++++-------- yarn.lock | 41 ++++++++++++++----- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index f5cf5eb6bc..cb73dcbe85 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -73,7 +73,6 @@ "@rjsf/validator-ajv8": "5.7.3", "@types/react": "^16.13.1 || ^17.0.0", "@uiw/react-codemirror": "^4.9.3", - "@zip.js/zip.js": "^2.7.24", "classnames": "^2.2.6", "event-source-polyfill": "^1.0.31", "git-url-parse": "^13.0.0", @@ -81,6 +80,7 @@ "immer": "^9.0.1", "json-schema": "^0.4.0", "json-schema-library": "^7.3.9", + "jszip": "^3.10.1", "lodash": "^4.17.21", "luxon": "^3.0.0", "qs": "^6.9.4", diff --git a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx index 530b9f5be1..d73b6ebc3f 100644 --- a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx +++ b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx @@ -26,7 +26,7 @@ import React, { useRef, useState, } from 'react'; -import * as zip from "@zip.js/zip.js"; +import JSZip from 'jszip'; import { scaffolderApiRef, ScaffolderDryRunResponse, @@ -198,33 +198,28 @@ export function useDryRun(): DryRun { } async function createZipDownload(directoryContents: { path: string; base64Content: string; executable: boolean; }[], name: string) { - // needs zip.js - - // Creates a BlobWriter object where the zip content will be written. - const zipFileWriter = new zip.BlobWriter(); - - // Creates a ZipWriter object writing data via `zipFileWriter`, adds the entry - const zipWriter = new zip.ZipWriter(zipFileWriter); + const zip = new JSZip(); for (const d of directoryContents) { // Decode text content from base64 to ascii const converted = atob(d.base64Content); - // Creates a TextReader object storing the text of the entry to add in the zip (i.e. "Hello world!"). - const fileReader = new zip.TextReader(converted); - - await zipWriter.add(d.path, fileReader); + // add folder/file to zip + await zip.file(d.path, converted); } - // Closes the writer. - await zipWriter.close(); + zip.generateAsync({type:"blob"}).then((blob) => { + saveAs(blob, name); + }, (err) => { + throw new Error(err); + }); +} - // Retrieves the Blob object containing the zip content into `zipFileBlob` - const zipFileBlob = await zipFileWriter.getData(); - // Download zip +// Download zip +function saveAs(blob: Blob, name: string) { const a = document.createElement('a'); - a.href = URL.createObjectURL(zipFileBlob); + a.href = URL.createObjectURL(blob); a.download = `dry-run-${name}.zip`; a.click(); URL.revokeObjectURL(a.href); diff --git a/yarn.lock b/yarn.lock index f7e0fb0f75..05f71c20df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8796,7 +8796,6 @@ __metadata: "@types/json-schema": ^7.0.9 "@types/react": ^16.13.1 || ^17.0.0 "@uiw/react-codemirror": ^4.9.3 - "@zip.js/zip.js": ^2.7.24 classnames: ^2.2.6 event-source-polyfill: ^1.0.31 git-url-parse: ^13.0.0 @@ -8804,6 +8803,7 @@ __metadata: immer: ^9.0.1 json-schema: ^0.4.0 json-schema-library: ^7.3.9 + jszip: ^3.10.1 lodash: ^4.17.21 luxon: ^3.0.0 msw: ^1.0.0 @@ -19347,13 +19347,6 @@ __metadata: languageName: node linkType: hard -"@zip.js/zip.js@npm:^2.7.24": - version: 2.7.24 - resolution: "@zip.js/zip.js@npm:2.7.24" - checksum: f56f973748d063158cbb6f07ccf4d4c64d35952bb2ecf3d9922db935b17f4e689b6d8568041083195ff4679a0d1a9378fa78adca7428819239d4a3fb4a1dd2d2 - languageName: node - linkType: hard - "@zxing/text-encoding@npm:0.9.0": version: 0.9.0 resolution: "@zxing/text-encoding@npm:0.9.0" @@ -28415,6 +28408,13 @@ __metadata: languageName: node linkType: hard +"immediate@npm:~3.0.5": + version: 3.0.6 + resolution: "immediate@npm:3.0.6" + checksum: f9b3486477555997657f70318cc8d3416159f208bec4cca3ff3442fd266bc23f50f0c9bd8547e1371a6b5e82b821ec9a7044a4f7b944798b25aa3cc6d5e63e62 + languageName: node + linkType: hard + "immer@npm:9.0.7": version: 9.0.7 resolution: "immer@npm:9.0.7" @@ -31066,6 +31066,18 @@ __metadata: languageName: node linkType: hard +"jszip@npm:^3.10.1": + version: 3.10.1 + resolution: "jszip@npm:3.10.1" + dependencies: + lie: ~3.3.0 + pako: ~1.0.2 + readable-stream: ~2.3.6 + setimmediate: ^1.0.5 + checksum: abc77bfbe33e691d4d1ac9c74c8851b5761fba6a6986630864f98d876f3fcc2d36817dfc183779f32c00157b5d53a016796677298272a714ae096dfe6b1c8b60 + languageName: node + linkType: hard + "just-diff-apply@npm:^4.0.1": version: 4.0.1 resolution: "just-diff-apply@npm:4.0.1" @@ -31402,6 +31414,15 @@ __metadata: languageName: node linkType: hard +"lie@npm:~3.3.0": + version: 3.3.0 + resolution: "lie@npm:3.3.0" + dependencies: + immediate: ~3.0.5 + checksum: 33102302cf19766f97919a6a98d481e01393288b17a6aa1f030a3542031df42736edde8dab29ffdbf90bebeffc48c761eb1d064dc77592ca3ba3556f9fe6d2a8 + languageName: node + linkType: hard + "lilconfig@npm:2.1.0, lilconfig@npm:^2.0.3": version: 2.1.0 resolution: "lilconfig@npm:2.1.0" @@ -34784,7 +34805,7 @@ __metadata: languageName: node linkType: hard -"pako@npm:^1.0.10, pako@npm:~1.0.5": +"pako@npm:^1.0.10, pako@npm:~1.0.2, pako@npm:~1.0.5": version: 1.0.11 resolution: "pako@npm:1.0.11" checksum: 1be2bfa1f807608c7538afa15d6f25baa523c30ec870a3228a89579e474a4d992f4293859524e46d5d87fd30fa17c5edf34dbef0671251d9749820b488660b16 @@ -37609,7 +37630,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.1, readable-stream@npm:^2.0.2, readable-stream@npm:^2.0.5, readable-stream@npm:^2.2.2, readable-stream@npm:^2.3.3, readable-stream@npm:^2.3.5, readable-stream@npm:^2.3.6": +"readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.1, readable-stream@npm:^2.0.2, readable-stream@npm:^2.0.5, readable-stream@npm:^2.2.2, readable-stream@npm:^2.3.3, readable-stream@npm:^2.3.5, readable-stream@npm:^2.3.6, readable-stream@npm:~2.3.6": version: 2.3.8 resolution: "readable-stream@npm:2.3.8" dependencies: From a40410b42c23cbaa1a877573958d06ac5e003944 Mon Sep 17 00:00:00 2001 From: TRIGUBF Date: Wed, 23 Aug 2023 17:13:54 +0200 Subject: [PATCH 052/131] move jszip to async import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../src/components/TemplateEditorPage/DryRunContext.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx index d73b6ebc3f..d9f853275f 100644 --- a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx +++ b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx @@ -26,7 +26,7 @@ import React, { useRef, useState, } from 'react'; -import JSZip from 'jszip'; +const {default: JSZip} = await import('jszip'); import { scaffolderApiRef, ScaffolderDryRunResponse, From 4d8aeacfa5fc7de643a4d1b094ce09f7c83955c8 Mon Sep 17 00:00:00 2001 From: TRIGUBF Date: Fri, 25 Aug 2023 11:33:54 +0200 Subject: [PATCH 053/131] move download logic to DryRunResultsList and helper, move jszip zo async import, add ownload helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../TemplateEditorPage/DryRunContext.tsx | 51 +------------------ .../DryRunResults/DryRunResultsList.tsx | 41 ++++++++++++++- .../scaffolder/src/lib/download/helpers.ts | 25 +++++++++ plugins/scaffolder/src/lib/download/index.ts | 17 +++++++ 4 files changed, 82 insertions(+), 52 deletions(-) create mode 100644 plugins/scaffolder/src/lib/download/helpers.ts create mode 100644 plugins/scaffolder/src/lib/download/index.ts diff --git a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx index d9f853275f..1765e8cb94 100644 --- a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx +++ b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunContext.tsx @@ -26,7 +26,6 @@ import React, { useRef, useState, } from 'react'; -const {default: JSZip} = await import('jszip'); import { scaffolderApiRef, ScaffolderDryRunResponse, @@ -41,7 +40,7 @@ interface DryRunOptions { files: Array<{ path: string; content: string }>; } -interface DryRunResult extends ScaffolderDryRunResponse { +export interface DryRunResult extends ScaffolderDryRunResponse { id: number; } @@ -51,7 +50,6 @@ interface DryRun { selectResult(id: number): void; deleteResult(id: number): void; - downloadResult(id: number): void; execute(options: DryRunOptions): Promise; } @@ -124,22 +122,6 @@ export function DryRunProvider(props: DryRunProviderProps) { }); }, []); - const downloadResult = useCallback((id: number) => { - setState(prevState => { - const index = prevState.results.findIndex(r => r.id === id); - if (index === -1) { - return prevState; - } - const result = prevState.results[index]; - console.log('Result', result.id); - createZipDownload(result.directoryContents, 'result_' + result.id) - return { - results: prevState.results, - selectedResult: prevState.selectedResult, - }; - }); - }, []); - const execute = useCallback( async (options: DryRunOptions) => { if (!scaffolderApi.dryRun) { @@ -176,7 +158,6 @@ export function DryRunProvider(props: DryRunProviderProps) { ...state, selectResult, deleteResult, - downloadResult, execute, }), [state, selectResult, deleteResult, execute], @@ -196,33 +177,3 @@ export function useDryRun(): DryRun { } return value; } - -async function createZipDownload(directoryContents: { path: string; base64Content: string; executable: boolean; }[], name: string) { - const zip = new JSZip(); - - for (const d of directoryContents) { - // Decode text content from base64 to ascii - const converted = atob(d.base64Content); - - // add folder/file to zip - await zip.file(d.path, converted); - } - - zip.generateAsync({type:"blob"}).then((blob) => { - saveAs(blob, name); - }, (err) => { - throw new Error(err); - }); -} - - -// Download zip -function saveAs(blob: Blob, name: string) { - const a = document.createElement('a'); - a.href = URL.createObjectURL(blob); - a.download = `dry-run-${name}.zip`; - a.click(); - URL.revokeObjectURL(a.href); - a.remove(); -} - diff --git a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx index 9163261985..384d48e871 100644 --- a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx +++ b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx @@ -27,7 +27,7 @@ import CheckIcon from '@material-ui/icons/Check'; import DeleteIcon from '@material-ui/icons/Delete'; import DownloadIcon from '@material-ui/icons/GetApp'; import React from 'react'; -import { useDryRun } from '../DryRunContext'; +import { DryRunResult, useDryRun } from '../DryRunContext'; const useStyles = makeStyles((theme: BackstageTheme) => ({ root: { @@ -72,7 +72,7 @@ export function DryRunResultsList() { edge="end" aria-label="download" title="Download as .zip" - onClick={() => dryRun.downloadResult(result.id)} + onClick={() => downloadResult(result)} > @@ -91,3 +91,40 @@ export function DryRunResultsList() { ); } + + +async function downloadResult(result: DryRunResult) { + await createZipDownload(result.directoryContents, `dry-run-result-${result.id}.zip`) +} + + +async function createZipDownload(directoryContents: { path: string; base64Content: string; executable: boolean; }[], name: string) { + const {default: JSZip} = await import('jszip'); + const zip = new JSZip(); + + for (const d of directoryContents) { + // Decode text content from base64 to ascii + const converted = atob(d.base64Content); + + // add folder/file to zip + await zip.file(d.path, converted); + } + + zip.generateAsync({type:"blob"}).then((blob) => { + // Download zip + downloadBlob(blob, name); + }, (err) => { + throw new Error(err); + }); +} + + + +function downloadBlob(blob: Blob, name: string) { + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = name; + a.click(); + URL.revokeObjectURL(a.href); + a.remove(); +} diff --git a/plugins/scaffolder/src/lib/download/helpers.ts b/plugins/scaffolder/src/lib/download/helpers.ts new file mode 100644 index 0000000000..2b596480d4 --- /dev/null +++ b/plugins/scaffolder/src/lib/download/helpers.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function downloadBlob(blob: Blob, name: string) { + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = name; + a.click(); + URL.revokeObjectURL(a.href); + a.remove(); +} + diff --git a/plugins/scaffolder/src/lib/download/index.ts b/plugins/scaffolder/src/lib/download/index.ts new file mode 100644 index 0000000000..0b51b8e61d --- /dev/null +++ b/plugins/scaffolder/src/lib/download/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { downloadBlob } from './helpers'; From eb8a79542ea68777b50b160c0bbf770573b8304a Mon Sep 17 00:00:00 2001 From: TRIGUBF Date: Fri, 25 Aug 2023 11:35:23 +0200 Subject: [PATCH 054/131] use download helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../DryRunResults/DryRunResultsList.tsx | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx index 384d48e871..0d9fcbc350 100644 --- a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx +++ b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx @@ -28,6 +28,7 @@ import DeleteIcon from '@material-ui/icons/Delete'; import DownloadIcon from '@material-ui/icons/GetApp'; import React from 'react'; import { DryRunResult, useDryRun } from '../DryRunContext'; +import { downloadBlob } from '../../../lib/download'; const useStyles = makeStyles((theme: BackstageTheme) => ({ root: { @@ -117,14 +118,3 @@ async function createZipDownload(directoryContents: { path: string; base64Conten throw new Error(err); }); } - - - -function downloadBlob(blob: Blob, name: string) { - const a = document.createElement('a'); - a.href = URL.createObjectURL(blob); - a.download = name; - a.click(); - URL.revokeObjectURL(a.href); - a.remove(); -} From e7f7ba34252098788e2ff60e7d97068620876b35 Mon Sep 17 00:00:00 2001 From: TRIGUBF Date: Fri, 25 Aug 2023 12:05:02 +0200 Subject: [PATCH 055/131] track async download state, disable button while downloading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../DryRunResults/DryRunResultsList.tsx | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx index 0d9fcbc350..2d7f78ac54 100644 --- a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx +++ b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx @@ -55,6 +55,14 @@ export function DryRunResultsList() { {dryRun.results.map(result => { const failed = result.log.some(l => l.body.status === 'failed'); + let isLoading = false; + + async function downloadResult() { + isLoading = true; + await downloadDirectoryContents(result.directoryContents, `dry-run-result-${result.id}.zip`) + isLoading = false; + } + return ( downloadResult(result)} + disabled={isLoading} + onClick={() => downloadResult()} > @@ -94,12 +103,8 @@ export function DryRunResultsList() { } -async function downloadResult(result: DryRunResult) { - await createZipDownload(result.directoryContents, `dry-run-result-${result.id}.zip`) -} - -async function createZipDownload(directoryContents: { path: string; base64Content: string; executable: boolean; }[], name: string) { +async function downloadDirectoryContents(directoryContents: { path: string; base64Content: string; executable: boolean; }[], name: string) { const {default: JSZip} = await import('jszip'); const zip = new JSZip(); From f3ba736e7e47087ab9b28072c8b40f5c60dd999e Mon Sep 17 00:00:00 2001 From: TRIGUBF Date: Fri, 25 Aug 2023 12:25:05 +0200 Subject: [PATCH 056/131] replace .then with async MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../DryRunResults/DryRunResultsList.tsx | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx index 2d7f78ac54..11031fc50c 100644 --- a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx +++ b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx @@ -116,10 +116,6 @@ async function downloadDirectoryContents(directoryContents: { path: string; base await zip.file(d.path, converted); } - zip.generateAsync({type:"blob"}).then((blob) => { - // Download zip - downloadBlob(blob, name); - }, (err) => { - throw new Error(err); - }); + const blob = await zip.generateAsync({type:"blob"}); + downloadBlob(blob, name); } From d031da5f0ef28f741a2adae05e0bd79ec81eabe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 4 Sep 2023 09:55:34 +0200 Subject: [PATCH 057/131] pretty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../DryRunResults/DryRunResultsList.tsx | 26 ++++++++++++------- .../scaffolder/src/lib/download/helpers.ts | 1 - 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx index 11031fc50c..df6e8ef5f2 100644 --- a/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx +++ b/plugins/scaffolder/src/components/TemplateEditorPage/DryRunResults/DryRunResultsList.tsx @@ -27,7 +27,7 @@ import CheckIcon from '@material-ui/icons/Check'; import DeleteIcon from '@material-ui/icons/Delete'; import DownloadIcon from '@material-ui/icons/GetApp'; import React from 'react'; -import { DryRunResult, useDryRun } from '../DryRunContext'; +import { useDryRun } from '../DryRunContext'; import { downloadBlob } from '../../../lib/download'; const useStyles = makeStyles((theme: BackstageTheme) => ({ @@ -59,10 +59,13 @@ export function DryRunResultsList() { async function downloadResult() { isLoading = true; - await downloadDirectoryContents(result.directoryContents, `dry-run-result-${result.id}.zip`) + await downloadDirectoryContents( + result.directoryContents, + `dry-run-result-${result.id}.zip`, + ); isLoading = false; } - + return ( Date: Mon, 4 Sep 2023 14:10:12 +0100 Subject: [PATCH 058/131] Fixed table sort bug in NewRelicComponent Signed-off-by: Joshua Grime --- .changeset/mean-queens-act.md | 5 +++++ .../NewRelicFetchComponent.tsx | 12 ++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 .changeset/mean-queens-act.md diff --git a/.changeset/mean-queens-act.md b/.changeset/mean-queens-act.md new file mode 100644 index 0000000000..a895ba5705 --- /dev/null +++ b/.changeset/mean-queens-act.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-newrelic': patch +--- + +Fixed bug in NewRelicComponent component where table would not sort correctly for numerical values. diff --git a/plugins/newrelic/src/components/NewRelicFetchComponent/NewRelicFetchComponent.tsx b/plugins/newrelic/src/components/NewRelicFetchComponent/NewRelicFetchComponent.tsx index e626e6a6d2..0cd1c3b14c 100644 --- a/plugins/newrelic/src/components/NewRelicFetchComponent/NewRelicFetchComponent.tsx +++ b/plugins/newrelic/src/components/NewRelicFetchComponent/NewRelicFetchComponent.tsx @@ -25,9 +25,9 @@ import { useApi } from '@backstage/core-plugin-api'; export const NewRelicAPMTable = ({ applications }: NewRelicApplications) => { const columns: TableColumn[] = [ { title: 'Application', field: 'name' }, - { title: 'Response Time', field: 'responseTime' }, - { title: 'Throughput', field: 'throughput' }, - { title: 'Error Rate', field: 'errorRate' }, + { title: 'Response Time (ms)', field: 'responseTime' }, + { title: 'Throughput (rpm)', field: 'throughput' }, + { title: 'Error Rate (%)', field: 'errorRate' }, { title: 'Instance Count', field: 'instanceCount' }, { title: 'Apdex', field: 'apdexScore' }, ]; @@ -43,9 +43,9 @@ export const NewRelicAPMTable = ({ applications }: NewRelicApplications) => { return { name, - responseTime: `${responseTime} ms`, - throughput: `${throughput} rpm`, - errorRate: `${errorRate}%`, + responseTime, + throughput, + errorRate, instanceCount, apdexScore, }; From 4c55c3b7c82e5c516abc91dad9fb27a3e2bebb2e Mon Sep 17 00:00:00 2001 From: Vladimir Masarik Date: Mon, 4 Sep 2023 18:15:10 +0200 Subject: [PATCH 059/131] Update the text Signed-off-by: Vladimir Masarik --- docs/auth/index.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/auth/index.md b/docs/auth/index.md index 6f3250559c..1d796d818b 100644 --- a/docs/auth/index.md +++ b/docs/auth/index.md @@ -10,7 +10,12 @@ configure Backstage to have any number of authentication providers, but only one of these will typically be used for sign-in, with the rest being used to provide access external resources. -> Note: Backstage backend APIs are by default unauthenticated. Thus, if your Backstage instance is exposed to the Internet, anyone can access information in the Backstage. If you would like to learn more, read about how to [authenticate API requests"](https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/authenticate-api-requests.md). +> NOTE: Identity management and the Sign-In page in Backstage is NOT a method for blocking +> access for unauthorized users. The identity system only serves to provide a personalized +> experience and access to a Backstage Identity Token, which can be passed to backend plugins. +> This also means that your Backstage backend APIs are by default unauthenticated. +> Thus, if your Backstage instance is exposed to the Internet, anyone can access +> information in the Backstage. You can lear more [here](../overview/threat-model#integrator-responsibilities). ## Built-in Authentication Providers @@ -61,13 +66,6 @@ the local `auth.environment` setting will be selected. ## Sign-In Configuration -> NOTE: Identity management and the `SignInPage` in Backstage is NOT a method -> for blocking access for unauthorized users, that either requires additional -> backend implementation or a separate service like Google's Identity-Aware -> Proxy. The identity system only serves to provide a personalized experience -> and access to a Backstage Identity Token, which can be passed to backend -> plugins. - Using an authentication provider for sign-in is something you need to configure both in the frontend app, as well as the `auth` backend plugin. For information on how to configure the backend app, see [Sign-in Identities and Resolvers](./identity-resolver.md). From 675c3766c6775d748f542dced0e3e296481813cd Mon Sep 17 00:00:00 2001 From: Vladimir Masarik Date: Mon, 4 Sep 2023 18:18:02 +0200 Subject: [PATCH 060/131] Fix typo Signed-off-by: Vladimir Masarik --- docs/auth/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/auth/index.md b/docs/auth/index.md index 1d796d818b..8aebe43cac 100644 --- a/docs/auth/index.md +++ b/docs/auth/index.md @@ -15,7 +15,7 @@ access external resources. > experience and access to a Backstage Identity Token, which can be passed to backend plugins. > This also means that your Backstage backend APIs are by default unauthenticated. > Thus, if your Backstage instance is exposed to the Internet, anyone can access -> information in the Backstage. You can lear more [here](../overview/threat-model#integrator-responsibilities). +> information in the Backstage. You can learn more [here](../overview/threat-model#integrator-responsibilities). ## Built-in Authentication Providers From 739b927df9cdb11bc8ae04dbc1b16cb3334afebf Mon Sep 17 00:00:00 2001 From: angom1 Date: Tue, 5 Sep 2023 10:50:37 +0100 Subject: [PATCH 061/131] Add check for null response Signed-off-by: angom1 --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 26d1b2a6fb..cf716ce521 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -150,7 +150,9 @@ export class GitLabClient { continue; } - for (const groupItem of response.data.group.descendantGroups.nodes) { + for (const groupItem of response.data.group.descendantGroups.nodes.filter( + group => group.id, + )) { const formattedGroupResponse = { id: Number(groupItem.id.replace(/^gid:\/\/gitlab\/Group\//, '')), name: groupItem.name, @@ -232,7 +234,9 @@ export class GitLabClient { continue; } - for (const userItem of response.data.group.groupMembers.nodes) { + for (const userItem of response.data.group.groupMembers.nodes.filter( + user => user.user.id, + )) { const formattedUserResponse = { id: Number(userItem.user.id.replace(/^gid:\/\/gitlab\/User\//, '')), username: userItem.user.username, From 02ba0a2efd2aea0d63e1ba219ba82cbdfc6f92d9 Mon Sep 17 00:00:00 2001 From: Christoph Jerolimov Date: Mon, 4 Sep 2023 11:00:50 +0200 Subject: [PATCH 062/131] Add info about which proxy-backend target isn't well configured. Signed-off-by: Christoph Jerolimov --- .changeset/dirty-boxes-give.md | 7 +++++++ plugins/proxy-backend/src/service/router.test.ts | 8 ++++++-- plugins/proxy-backend/src/service/router.ts | 7 +++++-- 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 .changeset/dirty-boxes-give.md diff --git a/.changeset/dirty-boxes-give.md b/.changeset/dirty-boxes-give.md new file mode 100644 index 0000000000..d339e49c98 --- /dev/null +++ b/.changeset/dirty-boxes-give.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-proxy-backend': patch +--- + +Add the route name to an error message that appears when the backend +proxy wasn't well configured. This will help users to understand the +issue and fix the right configuration. diff --git a/plugins/proxy-backend/src/service/router.test.ts b/plugins/proxy-backend/src/service/router.test.ts index 2b58e52e94..23b6e17ecc 100644 --- a/plugins/proxy-backend/src/service/router.test.ts +++ b/plugins/proxy-backend/src/service/router.test.ts @@ -157,7 +157,11 @@ describe('createRouter', () => { logger, discovery, }), - ).rejects.toThrow(new Error('Proxy target must be a string')); + ).rejects.toThrow( + new Error( + 'Proxy target for route "/test" must be a string, but is of type undefined', + ), + ); }); it('works if skip failures is set', async () => { @@ -189,7 +193,7 @@ describe('createRouter', () => { skipInvalidProxies: true, }); expect((logger.warn as jest.Mock).mock.calls[0][0]).toEqual( - 'skipped configuring /test due to Proxy target must be a string', + 'skipped configuring /test due to Proxy target for route "/test" must be a string, but is of type undefined', ); expect(router).toBeDefined(); }); diff --git a/plugins/proxy-backend/src/service/router.ts b/plugins/proxy-backend/src/service/router.ts index 61fddb60ed..96b9beceee 100644 --- a/plugins/proxy-backend/src/service/router.ts +++ b/plugins/proxy-backend/src/service/router.ts @@ -76,8 +76,11 @@ export function buildMiddleware( typeof config === 'string' ? { target: config } : { ...config }; // Validate that target is a valid URL. - if (typeof fullConfig.target !== 'string') { - throw new Error(`Proxy target must be a string`); + const targetType = typeof fullConfig.target; + if (targetType !== 'string') { + throw new Error( + `Proxy target for route "${route}" must be a string, but is of type ${targetType}`, + ); } try { // eslint-disable-next-line no-new From aa844e704a326cc0b546285839129a0997b3e519 Mon Sep 17 00:00:00 2001 From: Axel Koehler Date: Tue, 5 Sep 2023 15:27:59 +0200 Subject: [PATCH 063/131] feat(plugins/adr): configurable component header title Signed-off-by: Axel Koehler --- .changeset/plenty-lamps-pump.md | 5 +++++ .../adr/src/components/EntityAdrContent/EntityAdrContent.tsx | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changeset/plenty-lamps-pump.md diff --git a/.changeset/plenty-lamps-pump.md b/.changeset/plenty-lamps-pump.md new file mode 100644 index 0000000000..4ba45ce7ef --- /dev/null +++ b/.changeset/plenty-lamps-pump.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-adr': minor +--- + +configurable component header title diff --git a/plugins/adr/src/components/EntityAdrContent/EntityAdrContent.tsx b/plugins/adr/src/components/EntityAdrContent/EntityAdrContent.tsx index 32e353eba9..027da2aa67 100644 --- a/plugins/adr/src/components/EntityAdrContent/EntityAdrContent.tsx +++ b/plugins/adr/src/components/EntityAdrContent/EntityAdrContent.tsx @@ -160,8 +160,9 @@ const AdrListContainer = (props: { export const EntityAdrContent = (props: { contentDecorators?: AdrContentDecorator[]; filePathFilterFn?: AdrFilePathFilterFn; + headerTitle?: string; }) => { - const { contentDecorators, filePathFilterFn } = props; + const { contentDecorators, filePathFilterFn, headerTitle } = props; const classes = useStyles(); const { entity } = useEntity(); const [adrList, setAdrList] = useState([]); @@ -214,7 +215,7 @@ export const EntityAdrContent = (props: { return ( - + {appSupportConfigured && } From 5f33cae5f52c411aaef46d274e21bb7d56713900 Mon Sep 17 00:00:00 2001 From: npiyush97 Date: Tue, 5 Sep 2023 15:58:18 +0530 Subject: [PATCH 064/131] Tech radar timeline component formatting #19222 Signed-off-by: npiyush97 --- .../tech-radar/src/components/RadarTimeline/RadarTimeline.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/tech-radar/src/components/RadarTimeline/RadarTimeline.tsx b/plugins/tech-radar/src/components/RadarTimeline/RadarTimeline.tsx index 555873cf94..e54c4a499e 100644 --- a/plugins/tech-radar/src/components/RadarTimeline/RadarTimeline.tsx +++ b/plugins/tech-radar/src/components/RadarTimeline/RadarTimeline.tsx @@ -76,10 +76,10 @@ const RadarTimeline = (props: Props): JSX.Element => { '' )} - + {timeEntry.ring.name ? timeEntry.ring.name : ''} - + {timeEntry.date.toLocaleDateString() ? timeEntry.date.toLocaleDateString() : ''} From c357f62052af047bce6d095d992a35de693b08f9 Mon Sep 17 00:00:00 2001 From: npiyush97 Date: Tue, 5 Sep 2023 19:01:09 +0530 Subject: [PATCH 065/131] added changeset Signed-off-by: npiyush97 --- .changeset/stale-points-breathe.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/stale-points-breathe.md diff --git a/.changeset/stale-points-breathe.md b/.changeset/stale-points-breathe.md new file mode 100644 index 0000000000..95c0597414 --- /dev/null +++ b/.changeset/stale-points-breathe.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-tech-radar': patch +--- + +fixed Tech radar timeline component formatting. From 6279760b32037afc66cc6334e9529bf643b52a01 Mon Sep 17 00:00:00 2001 From: npiyush97 Date: Tue, 5 Sep 2023 19:03:15 +0530 Subject: [PATCH 066/131] added changeset with dco Signed-off-by: npiyush97 --- .changeset/stale-points-breathe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/stale-points-breathe.md b/.changeset/stale-points-breathe.md index 95c0597414..62c3196430 100644 --- a/.changeset/stale-points-breathe.md +++ b/.changeset/stale-points-breathe.md @@ -2,4 +2,4 @@ '@backstage/plugin-tech-radar': patch --- -fixed Tech radar timeline component formatting. +fixed related Tech radar timeline component formatting. From b880eb17caae8b2d28a637e8acbe065e05f61a5a Mon Sep 17 00:00:00 2001 From: PIYUSH NEGI <43876655+npiyush97@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:25:06 +0530 Subject: [PATCH 067/131] Update stale-points-breathe.md Signed-off-by: PIYUSH NEGI <43876655+npiyush97@users.noreply.github.com> --- .changeset/stale-points-breathe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/stale-points-breathe.md b/.changeset/stale-points-breathe.md index 62c3196430..36c01697d0 100644 --- a/.changeset/stale-points-breathe.md +++ b/.changeset/stale-points-breathe.md @@ -2,4 +2,4 @@ '@backstage/plugin-tech-radar': patch --- -fixed related Tech radar timeline component formatting. +Fixed `RadarTimeline` text formatting. From 2dc9cec3aaed6cf7f8ef6e1f911e68d680e4c0cf Mon Sep 17 00:00:00 2001 From: Vladimir Masarik Date: Wed, 6 Sep 2023 11:09:40 +0200 Subject: [PATCH 068/131] Fix missing file extention Signed-off-by: Vladimir Masarik --- docs/auth/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/auth/index.md b/docs/auth/index.md index 8aebe43cac..812d74ce0b 100644 --- a/docs/auth/index.md +++ b/docs/auth/index.md @@ -15,7 +15,7 @@ access external resources. > experience and access to a Backstage Identity Token, which can be passed to backend plugins. > This also means that your Backstage backend APIs are by default unauthenticated. > Thus, if your Backstage instance is exposed to the Internet, anyone can access -> information in the Backstage. You can learn more [here](../overview/threat-model#integrator-responsibilities). +> information in the Backstage. You can learn more [here](../overview/threat-model.md#integrator-responsibilities). ## Built-in Authentication Providers From 1110cf7bd10fc4f164c72483a9bf3e1d4f825436 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 20:15:15 +0000 Subject: [PATCH 069/131] chore(deps): update actions/upload-artifact action to v3.1.3 Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index d247312f89..62c693de6f 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -53,7 +53,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: 'Upload artifact' - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: name: SARIF file path: results.sarif From 2a64789f0cdb8a35440333ee5d9c0532b227ed30 Mon Sep 17 00:00:00 2001 From: Axel Koehler Date: Wed, 6 Sep 2023 08:53:57 +0200 Subject: [PATCH 070/131] support for i18n feature Signed-off-by: Axel Koehler --- .changeset/plenty-lamps-pump.md | 4 +-- .../EntityAdrContent/EntityAdrContent.tsx | 12 +++++---- plugins/adr/src/translations.ts | 26 +++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 plugins/adr/src/translations.ts diff --git a/.changeset/plenty-lamps-pump.md b/.changeset/plenty-lamps-pump.md index 4ba45ce7ef..04d0f2f4f0 100644 --- a/.changeset/plenty-lamps-pump.md +++ b/.changeset/plenty-lamps-pump.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-adr': minor +'@backstage/plugin-adr': patch --- -configurable component header title +support for i18n feature diff --git a/plugins/adr/src/components/EntityAdrContent/EntityAdrContent.tsx b/plugins/adr/src/components/EntityAdrContent/EntityAdrContent.tsx index 027da2aa67..8081043a9f 100644 --- a/plugins/adr/src/components/EntityAdrContent/EntityAdrContent.tsx +++ b/plugins/adr/src/components/EntityAdrContent/EntityAdrContent.tsx @@ -59,6 +59,8 @@ import FolderIcon from '@material-ui/icons/Folder'; import { adrApiRef, AdrFileInfo } from '../../api'; import { rootRouteRef } from '../../routes'; import { AdrContentDecorator, AdrReader } from '../AdrReader'; +import { useTranslationRef } from '@backstage/core-plugin-api/alpha'; +import { adrTranslationRef } from '../../translations'; const useStyles = makeStyles((theme: Theme) => ({ adrMenu: { @@ -160,9 +162,8 @@ const AdrListContainer = (props: { export const EntityAdrContent = (props: { contentDecorators?: AdrContentDecorator[]; filePathFilterFn?: AdrFilePathFilterFn; - headerTitle?: string; }) => { - const { contentDecorators, filePathFilterFn, headerTitle } = props; + const { contentDecorators, filePathFilterFn } = props; const classes = useStyles(); const { entity } = useEntity(); const [adrList, setAdrList] = useState([]); @@ -170,6 +171,7 @@ export const EntityAdrContent = (props: { const scmIntegrations = useApi(scmIntegrationsApiRef); const adrApi = useApi(adrApiRef); const entityHasAdrs = isAdrAvailable(entity); + const t = useTranslationRef(adrTranslationRef); const config = useApi(configApiRef); const appSupportConfigured = config?.getOptionalConfig('app.support'); @@ -215,7 +217,7 @@ export const EntityAdrContent = (props: { return ( - + {appSupportConfigured && } @@ -226,7 +228,7 @@ export const EntityAdrContent = (props: { {loading && } {entityHasAdrs && !loading && error && ( - + )} {entityHasAdrs && @@ -253,7 +255,7 @@ export const EntityAdrContent = (props: { ) : ( - No ADRs found + {t('no_adrs')} ))} ); diff --git a/plugins/adr/src/translations.ts b/plugins/adr/src/translations.ts new file mode 100644 index 0000000000..cc0995f597 --- /dev/null +++ b/plugins/adr/src/translations.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createTranslationRef } from '@backstage/core-plugin-api/alpha'; + +/** @alpha */ +export const adrTranslationRef = createTranslationRef({ + id: 'adr', + messages: { + content_header_title: 'Architecture Decision Records', + failed_to_fetch: 'Failed to fetch ADRs', + no_adrs: 'No ADRs found', + }, +}); From cd7331587eb319ef7c71fd72df5bec022ba7a7b0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 7 Sep 2023 10:49:38 +0200 Subject: [PATCH 071/131] cli: remove package fix command Signed-off-by: Patrik Oldsberg --- .changeset/tender-garlics-report.md | 5 + packages/cli/src/commands/fix.ts | 142 ---------------------------- packages/cli/src/commands/index.ts | 6 -- 3 files changed, 5 insertions(+), 148 deletions(-) create mode 100644 .changeset/tender-garlics-report.md delete mode 100644 packages/cli/src/commands/fix.ts diff --git a/.changeset/tender-garlics-report.md b/.changeset/tender-garlics-report.md new file mode 100644 index 0000000000..919af4d4f8 --- /dev/null +++ b/.changeset/tender-garlics-report.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Removed the experimental `package fix` command that was used to automatically add dependencies to `package.json`, but has since been replaced by the `no-undeclared-imports` rule from `@backstage/eslint-plugin`. diff --git a/packages/cli/src/commands/fix.ts b/packages/cli/src/commands/fix.ts deleted file mode 100644 index 263d171fd6..0000000000 --- a/packages/cli/src/commands/fix.ts +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { paths } from '../lib/paths'; -import { ESLint } from 'eslint'; -import { join as joinPath, basename } from 'path'; -import fs from 'fs-extra'; -import { isChildPath } from '@backstage/cli-common'; -import { PackageGraph } from '@backstage/cli-node'; - -function isTestPath(filePath: string) { - if (!isChildPath(joinPath(paths.targetDir, 'src'), filePath)) { - return true; - } - const name = basename(filePath); - return ( - name.startsWith('setupTests.') || - name.includes('.test.') || - name.includes('.stories.') - ); -} - -export async function command() { - const pkgJsonPath = paths.resolveTarget('package.json'); - const pkg = await fs.readJson(pkgJsonPath); - if (pkg.workspaces) { - throw new Error( - 'Adding dependencies to the workspace root is not supported', - ); - } - - const packages = await PackageGraph.listTargetPackages(); - const localPackageVersions = new Map( - packages.map(p => [p.packageJson.name, p.packageJson.version]), - ); - - const eslint = new ESLint({ - cwd: paths.targetDir, - overrideConfig: { - plugins: ['monorepo'], - rules: { - 'import/no-extraneous-dependencies': [ - 'error', - { - devDependencies: [ - `!${joinPath(paths.targetDir, 'src/**')}`, - joinPath(paths.targetDir, 'src/**/*.test.*'), - joinPath(paths.targetDir, 'src/**/*.stories.*'), - joinPath(paths.targetDir, 'src/setupTests.*'), - ], - optionalDependencies: true, - peerDependencies: true, - bundledDependencies: true, - }, - ], - }, - }, - extensions: ['jsx', 'ts', 'tsx', 'mjs', 'cjs'], - }); - - const results = await eslint.lintFiles(['.']); - - const addedDeps = new Set(); - const addedDevDeps = new Set(); - const removedDevDeps = new Set(); - - for (const result of results) { - for (const message of result.messages) { - // Just in case - if (message.ruleId !== 'import/no-extraneous-dependencies') { - continue; - } - - const match = message.message.match(/^'([^']*)' should be listed/); - if (!match) { - continue; - } - const packageName = match[1]; - if (!localPackageVersions.has(packageName)) { - continue; - } - - if (message.message.endsWith('not devDependencies.')) { - addedDeps.add(packageName); - removedDevDeps.add(packageName); - } else if (isTestPath(result.filePath)) { - addedDevDeps.add(packageName); - } else { - addedDeps.add(packageName); - } - } - } - - if (addedDeps.size || addedDevDeps.size || removedDevDeps.size) { - for (const name of addedDeps) { - if (!pkg.dependencies) { - pkg.dependencies = {}; - } - pkg.dependencies[name] = `^${localPackageVersions.get(name)}`; - } - for (const name of addedDevDeps) { - if (!pkg.devDependencies) { - pkg.devDependencies = {}; - } - pkg.devDependencies[name] = `^${localPackageVersions.get(name)}`; - } - for (const name of removedDevDeps) { - delete pkg.devDependencies[name]; - } - if (Object.keys(pkg.devDependencies).length === 0) { - delete pkg.devDependencies; - } - - if (pkg.dependencies) { - pkg.dependencies = Object.fromEntries( - Object.entries(pkg.dependencies).sort(([a], [b]) => a.localeCompare(b)), - ); - } - if (pkg.devDependencies) { - pkg.devDependencies = Object.fromEntries( - Object.entries(pkg.devDependencies).sort(([a], [b]) => - a.localeCompare(b), - ), - ); - } - - await fs.writeJson(pkgJsonPath, pkg, { spaces: 2 }); - } -} diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index a7f65588a5..0a461ba78c 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -159,12 +159,6 @@ export function registerScriptCommand(program: Command) { .description('Run tests, forwarding args to Jest, defaulting to watch mode') .action(lazy(() => import('./test').then(m => m.default))); - command - .command('fix', { hidden: true }) - .description('Applies automated fixes to the package. [EXPERIMENTAL]') - .option('--deps', 'Only fix monorepo dependencies in package.json') - .action(lazy(() => import('./fix').then(m => m.command))); - command .command('clean') .description('Delete cache directories') From 0ccb00afa0dca7393b1545861c838cd3051b9192 Mon Sep 17 00:00:00 2001 From: angom1 Date: Thu, 7 Sep 2023 10:57:17 +0100 Subject: [PATCH 072/131] Update query to commitEmail Signed-off-by: angom1 --- plugins/catalog-backend-module-gitlab/src/lib/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index cf716ce521..fe0da6c4b5 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -205,7 +205,7 @@ export class GitLabClient { user { id username - publicEmail + commitEmail name state webUrl From 4fa1c74cbadc5371d25cc466aa0a6d181eb694f5 Mon Sep 17 00:00:00 2001 From: raffitamizian Date: Thu, 7 Sep 2023 14:37:02 +0100 Subject: [PATCH 073/131] Enables dry-run for yeoman scaffolder action Signed-off-by: raffitamizian --- .changeset/big-spies-agree.md | 5 +++++ .../src/actions/run/yeoman.ts | 1 + 2 files changed, 6 insertions(+) create mode 100644 .changeset/big-spies-agree.md diff --git a/.changeset/big-spies-agree.md b/.changeset/big-spies-agree.md new file mode 100644 index 0000000000..1c544f4f36 --- /dev/null +++ b/.changeset/big-spies-agree.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend-module-yeoman': patch +--- + +Enables dry-run functionality for the run:yeoman scaffolder action diff --git a/plugins/scaffolder-backend-module-yeoman/src/actions/run/yeoman.ts b/plugins/scaffolder-backend-module-yeoman/src/actions/run/yeoman.ts index 89ea10b6a6..62d62807ae 100644 --- a/plugins/scaffolder-backend-module-yeoman/src/actions/run/yeoman.ts +++ b/plugins/scaffolder-backend-module-yeoman/src/actions/run/yeoman.ts @@ -61,6 +61,7 @@ export function createRunYeomanAction() { }, }, }, + supportsDryRun: true, async handler(ctx) { ctx.logger.info( `Templating using Yeoman generator: ${ctx.input.namespace}`, From e0088d9d1a4617295cbbaa94a36630318a3e7194 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Sep 2023 11:17:42 +0200 Subject: [PATCH 074/131] frontend-app-api: implement api factory discovery Co-authored-by: Camila Belo Signed-off-by: Patrik Oldsberg --- packages/frontend-app-api/package.json | 1 + packages/frontend-app-api/src/createApp.tsx | 66 +++++++++++++++---- .../frontend-app-api/src/extensions/Core.tsx | 34 ++++++++++ yarn.lock | 1 + 4 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 packages/frontend-app-api/src/extensions/Core.tsx diff --git a/packages/frontend-app-api/package.json b/packages/frontend-app-api/package.json index 33b6067858..17f4b1bef3 100644 --- a/packages/frontend-app-api/package.json +++ b/packages/frontend-app-api/package.json @@ -34,6 +34,7 @@ ], "dependencies": { "@backstage/config": "workspace:^", + "@backstage/core-app-api": "workspace:^", "@backstage/core-plugin-api": "workspace:^", "@backstage/frontend-plugin-api": "workspace:^", "@backstage/plugin-graphiql": "workspace:^", diff --git a/packages/frontend-app-api/src/createApp.tsx b/packages/frontend-app-api/src/createApp.tsx index c50b507642..77f0d1948a 100644 --- a/packages/frontend-app-api/src/createApp.tsx +++ b/packages/frontend-app-api/src/createApp.tsx @@ -21,6 +21,7 @@ import { coreExtensionData, } from '@backstage/frontend-plugin-api'; import { CoreRouter } from './extensions/CoreRouter'; +import { Core } from './extensions/Core'; import { createExtensionInstance, ExtensionInstance, @@ -31,8 +32,13 @@ import { readAppExtensionParameters, } from './wiring/parameters'; import { RoutingProvider } from './routing/RoutingContext'; -import { RouteRef } from '@backstage/core-plugin-api'; +import { ApiHolder, RouteRef } from '@backstage/core-plugin-api'; import { getAvailablePlugins } from './wiring/discovery'; +import { + ApiFactoryRegistry, + ApiProvider, + ApiResolver, +} from '@backstage/core-app-api'; /** @public */ export function createApp(options: { plugins: BackstagePlugin[] }): { @@ -40,7 +46,7 @@ export function createApp(options: { plugins: BackstagePlugin[] }): { } { const appConfig = ConfigReader.fromConfigs(process.env.APP_CONFIG as any); - const builtinExtensions = [CoreRouter]; + const builtinExtensions = [CoreRouter, Core]; const discoveredPlugins = getAvailablePlugins(); // pull in default extension instance from discovered packages @@ -115,25 +121,59 @@ export function createApp(options: { plugins: BackstagePlugin[] }): { const routePaths = extractRouteInfoFromInstanceTree(rootInstances); + const coreInstance = rootInstances.find(({ id }) => id === 'core'); + if (!coreInstance) { + throw Error('Unable to find core extension instance'); + } + + const apiHolder = createApiHolder(coreInstance); + return { createRoot() { - const rootComponents = rootInstances.map( - e => - e.data.get( - coreExtensionData.reactComponent.id, - ) as typeof coreExtensionData.reactComponent.T, - ); + const rootComponents = rootInstances + .map( + e => + e.data.get( + coreExtensionData.reactComponent.id, + ) as typeof coreExtensionData.reactComponent.T, + ) + .filter(Boolean); return ( - - {rootComponents.map((Component, i) => ( - - ))} - + + + {rootComponents.map((Component, i) => ( + + ))} + + ); }, }; } +function createApiHolder(coreExtension: ExtensionInstance): ApiHolder { + const factoryRegistry = new ApiFactoryRegistry(); + + const apiFactories = + coreExtension.attachments + .get('apis') + ?.map( + e => + e.data.get( + coreExtensionData.apiFactory.id, + ) as typeof coreExtensionData.apiFactory.T, + ) + .filter(Boolean) ?? []; + + for (const factory of apiFactories) { + factoryRegistry.register('default', factory); + } + + ApiResolver.validateFactories(factoryRegistry, factoryRegistry.getAllApis()); + + return new ApiResolver(factoryRegistry); +} + /** @internal */ export function extractRouteInfoFromInstanceTree( roots: ExtensionInstance[], diff --git a/packages/frontend-app-api/src/extensions/Core.tsx b/packages/frontend-app-api/src/extensions/Core.tsx new file mode 100644 index 0000000000..cb93b2888f --- /dev/null +++ b/packages/frontend-app-api/src/extensions/Core.tsx @@ -0,0 +1,34 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + coreExtensionData, + createExtension, +} from '@backstage/frontend-plugin-api'; + +export const Core = createExtension({ + id: 'core', + at: 'root', + inputs: { + apis: { + extensionData: { + api: coreExtensionData.apiFactory, + }, + }, + }, + output: {}, + factory() {}, +}); diff --git a/yarn.lock b/yarn.lock index 7075f315dc..4a9b30d532 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4317,6 +4317,7 @@ __metadata: dependencies: "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/core-app-api": "workspace:^" "@backstage/core-plugin-api": "workspace:^" "@backstage/frontend-plugin-api": "workspace:^" "@backstage/plugin-graphiql": "workspace:^" From b628d30a2b0303c66ba96fd46c0d6fc8a6d82505 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Sep 2023 11:43:30 +0200 Subject: [PATCH 075/131] frontend-app-api: add support for default app themes Co-authored-by: Camila Belo Co-authored-by: Philipp Hugenroth Signed-off-by: Patrik Oldsberg --- packages/frontend-app-api/src/createApp.tsx | 30 ++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/packages/frontend-app-api/src/createApp.tsx b/packages/frontend-app-api/src/createApp.tsx index 77f0d1948a..41d16a7a9a 100644 --- a/packages/frontend-app-api/src/createApp.tsx +++ b/packages/frontend-app-api/src/createApp.tsx @@ -32,13 +32,22 @@ import { readAppExtensionParameters, } from './wiring/parameters'; import { RoutingProvider } from './routing/RoutingContext'; -import { ApiHolder, RouteRef } from '@backstage/core-plugin-api'; +import { + ApiHolder, + appThemeApiRef, + RouteRef, +} from '@backstage/core-plugin-api'; import { getAvailablePlugins } from './wiring/discovery'; import { ApiFactoryRegistry, ApiProvider, ApiResolver, + AppThemeSelector, } from '@backstage/core-app-api'; +// eslint-disable-next-line @backstage/no-relative-monorepo-imports +import { AppThemeProvider } from '../../core-app-api/src/app/AppThemeProvider'; +// eslint-disable-next-line @backstage/no-relative-monorepo-imports +import { themes } from '../../app-defaults/src/defaults/themes'; /** @public */ export function createApp(options: { plugins: BackstagePlugin[] }): { @@ -140,11 +149,13 @@ export function createApp(options: { plugins: BackstagePlugin[] }): { .filter(Boolean); return ( - - {rootComponents.map((Component, i) => ( - - ))} - + + + {rootComponents.map((Component, i) => ( + + ))} + + ); }, @@ -169,6 +180,13 @@ function createApiHolder(coreExtension: ExtensionInstance): ApiHolder { factoryRegistry.register('default', factory); } + factoryRegistry.register('static', { + api: appThemeApiRef, + deps: {}, + // TODO: add extension for registering themes + factory: () => AppThemeSelector.createWithStorage(themes), + }); + ApiResolver.validateFactories(factoryRegistry, factoryRegistry.getAllApis()); return new ApiResolver(factoryRegistry); From e30df514e13f0254778207f038737fc32a6fc083 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Sep 2023 13:45:27 +0200 Subject: [PATCH 076/131] frontend-app-api: add config support Co-authored-by: Camila Belo Co-authored-by: Philipp Hugenroth Signed-off-by: Patrik Oldsberg --- .../src/app/defaultConfigLoader.ts | 12 ++++++--- packages/frontend-app-api/src/createApp.tsx | 27 ++++++++++++++++--- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/packages/core-app-api/src/app/defaultConfigLoader.ts b/packages/core-app-api/src/app/defaultConfigLoader.ts index 00aee2d367..2ea8673f39 100644 --- a/packages/core-app-api/src/app/defaultConfigLoader.ts +++ b/packages/core-app-api/src/app/defaultConfigLoader.ts @@ -16,7 +16,6 @@ import { AppConfig } from '@backstage/config'; import { JsonObject } from '@backstage/types'; -import { AppConfigLoader } from './types'; /** * The default config loader, which expects that config is available at compile-time @@ -30,12 +29,17 @@ import { AppConfigLoader } from './types'; * * @public */ -export const defaultConfigLoader: AppConfigLoader = async ( +export async function defaultConfigLoader(): Promise { + return defaultConfigLoaderSync(); +} + +/** @internal */ +export function defaultConfigLoaderSync( // This string may be replaced at runtime to provide additional config. // It should be replaced by a JSON-serialized config object. // It's a param so we can test it, but at runtime this will always fall back to default. runtimeConfigJson: string = '__APP_INJECTED_RUNTIME_CONFIG__', -) => { +) { const appConfig = process.env.APP_CONFIG; if (!appConfig) { throw new Error('No static configuration provided'); @@ -70,4 +74,4 @@ export const defaultConfigLoader: AppConfigLoader = async ( }); } return configs; -}; +} diff --git a/packages/frontend-app-api/src/createApp.tsx b/packages/frontend-app-api/src/createApp.tsx index 41d16a7a9a..7393740863 100644 --- a/packages/frontend-app-api/src/createApp.tsx +++ b/packages/frontend-app-api/src/createApp.tsx @@ -35,6 +35,8 @@ import { RoutingProvider } from './routing/RoutingContext'; import { ApiHolder, appThemeApiRef, + ConfigApi, + configApiRef, RouteRef, } from '@backstage/core-plugin-api'; import { getAvailablePlugins } from './wiring/discovery'; @@ -44,16 +46,24 @@ import { ApiResolver, AppThemeSelector, } from '@backstage/core-app-api'; + +// TODO: Get rid of all of these // eslint-disable-next-line @backstage/no-relative-monorepo-imports import { AppThemeProvider } from '../../core-app-api/src/app/AppThemeProvider'; // eslint-disable-next-line @backstage/no-relative-monorepo-imports +import { defaultConfigLoaderSync } from '../../core-app-api/src/app/defaultConfigLoader'; +// eslint-disable-next-line @backstage/no-relative-monorepo-imports import { themes } from '../../app-defaults/src/defaults/themes'; /** @public */ -export function createApp(options: { plugins: BackstagePlugin[] }): { +export function createApp(options: { + plugins: BackstagePlugin[]; + config?: ConfigApi; +}): { createRoot(): JSX.Element; } { - const appConfig = ConfigReader.fromConfigs(process.env.APP_CONFIG as any); + const appConfig = + options?.config ?? ConfigReader.fromConfigs(defaultConfigLoaderSync()); const builtinExtensions = [CoreRouter, Core]; const discoveredPlugins = getAvailablePlugins(); @@ -135,7 +145,7 @@ export function createApp(options: { plugins: BackstagePlugin[] }): { throw Error('Unable to find core extension instance'); } - const apiHolder = createApiHolder(coreInstance); + const apiHolder = createApiHolder(coreInstance, appConfig); return { createRoot() { @@ -162,7 +172,10 @@ export function createApp(options: { plugins: BackstagePlugin[] }): { }; } -function createApiHolder(coreExtension: ExtensionInstance): ApiHolder { +function createApiHolder( + coreExtension: ExtensionInstance, + configApi: ConfigApi, +): ApiHolder { const factoryRegistry = new ApiFactoryRegistry(); const apiFactories = @@ -187,6 +200,12 @@ function createApiHolder(coreExtension: ExtensionInstance): ApiHolder { factory: () => AppThemeSelector.createWithStorage(themes), }); + factoryRegistry.register('static', { + api: configApiRef, + deps: {}, + factory: () => configApi, + }); + ApiResolver.validateFactories(factoryRegistry, factoryRegistry.getAllApis()); return new ApiResolver(factoryRegistry); From 46b4845a7b46dda9aacce5ee51d0a87b15b3a27e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Sep 2023 13:46:03 +0200 Subject: [PATCH 077/131] graphiql: stop disabling endpoint extensions by default + configure in app-next Co-authored-by: Camila Belo Co-authored-by: Philipp Hugenroth Signed-off-by: Patrik Oldsberg --- packages/app-next/app-config.yaml | 2 ++ plugins/graphiql/src/alpha.tsx | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/app-next/app-config.yaml b/packages/app-next/app-config.yaml index c94d900b21..5a110b06e4 100644 --- a/packages/app-next/app-config.yaml +++ b/packages/app-next/app-config.yaml @@ -2,6 +2,8 @@ app: experimental: packages: 'all' # ✨ + extensions: + - apis.plugin.graphiql.browse.gitlab: true # scmAuthExtension: >- # createScmAuthExtension({ diff --git a/plugins/graphiql/src/alpha.tsx b/plugins/graphiql/src/alpha.tsx index ded517e3c7..f0333c8c6c 100644 --- a/plugins/graphiql/src/alpha.tsx +++ b/plugins/graphiql/src/alpha.tsx @@ -65,13 +65,14 @@ export const graphiqlBrowseApi = createApiExtension({ export function createEndpointExtension(options: { id: string; configSchema?: PortableSchema; + disabled?: boolean; factory: (options: { config: TConfig }) => { endpoint: GraphQLEndpoint }; }) { return createExtension({ id: `apis.plugin.graphiql.browse.${options.id}`, at: 'apis.plugin.graphiql.browse/endpoints', configSchema: options.configSchema, - disabled: true, + disabled: options.disabled ?? false, output: { endpoint: endpointDataRef, }, @@ -86,6 +87,7 @@ export function createEndpointExtension(options: { /** @alpha */ const gitlabGraphiQLBrowseExtension = createEndpointExtension({ id: 'gitlab', + disabled: true, configSchema: createSchemaFromZod(z => z .object({ From 089291c7e53008d55f7b387a02fae464d2797ff6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Sep 2023 14:29:45 +0200 Subject: [PATCH 078/131] core-app-api: fix defaultConfigLoader tests Co-authored-by: Camila Belo Co-authored-by: Philipp Hugenroth Signed-off-by: Patrik Oldsberg --- .../src/app/defaultConfigLoader.test.ts | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/packages/core-app-api/src/app/defaultConfigLoader.test.ts b/packages/core-app-api/src/app/defaultConfigLoader.test.ts index aebba92c43..7579e53dd2 100644 --- a/packages/core-app-api/src/app/defaultConfigLoader.test.ts +++ b/packages/core-app-api/src/app/defaultConfigLoader.test.ts @@ -14,40 +14,38 @@ * limitations under the License. */ -import { defaultConfigLoader } from './defaultConfigLoader'; +import { defaultConfigLoaderSync } from './defaultConfigLoader'; (process as any).env = { NODE_ENV: 'test' }; const anyEnv = process.env as any; const anyWindow = window as any; -describe('defaultConfigLoader', () => { +describe('defaultConfigLoaderSync', () => { afterEach(() => { delete anyEnv.APP_CONFIG; delete anyWindow.__APP_CONFIG__; }); - it('loads static config', async () => { + it('loads static config', () => { anyEnv.APP_CONFIG = [ { data: { my: 'config' }, context: 'a' }, { data: { my: 'override-config' }, context: 'b' }, ]; - const configs = await defaultConfigLoader(); + const configs = defaultConfigLoaderSync(); expect(configs).toEqual([ { data: { my: 'config' }, context: 'a' }, { data: { my: 'override-config' }, context: 'b' }, ]); }); - it('loads runtime config', async () => { + it('loads runtime config', () => { anyEnv.APP_CONFIG = [ { data: { my: 'override-config' }, context: 'a' }, { data: { my: 'config' }, context: 'b' }, ]; - const configs = await (defaultConfigLoader as any)( - '{"my":"runtime-config"}', - ); + const configs = (defaultConfigLoaderSync as any)('{"my":"runtime-config"}'); expect(configs).toEqual([ { data: { my: 'override-config' }, context: 'a' }, { data: { my: 'config' }, context: 'b' }, @@ -55,28 +53,28 @@ describe('defaultConfigLoader', () => { ]); }); - it('fails to load invalid missing config', async () => { - await expect(defaultConfigLoader()).rejects.toThrow( + it('fails to load invalid missing config', () => { + expect(() => defaultConfigLoaderSync()).toThrow( 'No static configuration provided', ); }); - it('fails to load invalid static config', async () => { + it('fails to load invalid static config', () => { anyEnv.APP_CONFIG = { my: 'invalid-config' }; - await expect(defaultConfigLoader()).rejects.toThrow( + expect(() => defaultConfigLoaderSync()).toThrow( 'Static configuration has invalid format', ); }); - it('fails to load bad runtime config', async () => { + it('fails to load bad runtime config', () => { anyEnv.APP_CONFIG = [{ data: { my: 'config' }, context: 'a' }]; - await expect((defaultConfigLoader as any)('}')).rejects.toThrow( + expect(() => defaultConfigLoaderSync('}')).toThrow( 'Failed to load runtime configuration, SyntaxError: Unexpected token } in JSON at position 0', ); }); - it('loads config from window.__APP_CONFIG__', async () => { + it('loads config from window.__APP_CONFIG__', () => { anyEnv.APP_CONFIG = [ { data: { my: 'config' }, context: 'a' }, { data: { my: 'override-config' }, context: 'b' }, @@ -84,7 +82,7 @@ describe('defaultConfigLoader', () => { const windowConfig = { app: { configKey: 'config-value' } }; anyWindow.__APP_CONFIG__ = windowConfig; - const configs = await defaultConfigLoader(); + const configs = defaultConfigLoaderSync(); expect(configs).toEqual([ ...anyEnv.APP_CONFIG, From 528b28ffc554b638f33f297f952ada85a1663f5f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Sep 2023 14:31:47 +0200 Subject: [PATCH 079/131] core-app-api: extract out overrideBaseUrlConfigs Co-authored-by: Camila Belo Co-authored-by: Philipp Hugenroth Signed-off-by: Patrik Oldsberg --- packages/core-app-api/src/app/AppManager.tsx | 63 ++------------ .../src/app/overrideBaseUrlConfigs.ts | 84 +++++++++++++++++++ 2 files changed, 89 insertions(+), 58 deletions(-) create mode 100644 packages/core-app-api/src/app/overrideBaseUrlConfigs.ts diff --git a/packages/core-app-api/src/app/AppManager.tsx b/packages/core-app-api/src/app/AppManager.tsx index de078c0ffc..0bd495c03b 100644 --- a/packages/core-app-api/src/app/AppManager.tsx +++ b/packages/core-app-api/src/app/AppManager.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { AppConfig, Config } from '@backstage/config'; +import { Config } from '@backstage/config'; import React, { ComponentType, PropsWithChildren, @@ -81,6 +81,7 @@ import { InternalAppContext } from './InternalAppContext'; import { AppRouter, getBasePath } from './AppRouter'; import { AppTranslationProvider } from './AppTranslationProvider'; import { AppTranslationApiImpl } from '../apis/implementations/AppTranslationApi'; +import { overrideBaseUrlConfigs } from './overrideBaseUrlConfigs'; type CompatiblePlugin = | BackstagePlugin @@ -88,17 +89,6 @@ type CompatiblePlugin = output(): Array<{ type: 'feature-flag'; name: string }>; }); -/** - * Creates a base URL that uses to the current document origin. - */ -function createLocalBaseUrl(fullUrl: string): string { - const url = new URL(fullUrl); - url.protocol = document.location.protocol; - url.hostname = document.location.hostname; - url.port = document.location.port; - return url.toString().replace(/\/$/, ''); -} - function useConfigLoader( configLoader: AppConfigLoader | undefined, components: AppComponents, @@ -131,52 +121,9 @@ function useConfigLoader( }; } - let configReader; - /** - * config.value can be undefined or empty. If it's either, don't bother overriding anything. - */ - if (config.value?.length) { - const urlConfigReader = ConfigReader.fromConfigs(config.value); - - /** - * Test configs may not define `app.baseUrl` or `backend.baseUrl` and we - * don't want to enforce here. - */ - const appBaseUrl = urlConfigReader.getOptionalString('app.baseUrl'); - const backendBaseUrl = urlConfigReader.getOptionalString('backend.baseUrl'); - - let configs = config.value; - const relativeResolverConfig: AppConfig = { - data: {}, - context: 'relative-resolver', - }; - if (appBaseUrl && backendBaseUrl) { - const appOrigin = new URL(appBaseUrl).origin; - const backendOrigin = new URL(backendBaseUrl).origin; - - if (appOrigin === backendOrigin) { - const newBackendBaseUrl = createLocalBaseUrl(backendBaseUrl); - if (backendBaseUrl !== newBackendBaseUrl) { - relativeResolverConfig.data.backend = { baseUrl: newBackendBaseUrl }; - } - } - } - if (appBaseUrl) { - const newAppBaseUrl = createLocalBaseUrl(appBaseUrl); - if (appBaseUrl !== newAppBaseUrl) { - relativeResolverConfig.data.app = { baseUrl: newAppBaseUrl }; - } - } - /** - * Only add the relative config if there is actually data to add. - */ - if (Object.keys(relativeResolverConfig.data).length) { - configs = configs.concat([relativeResolverConfig]); - } - configReader = ConfigReader.fromConfigs(configs); - } else { - configReader = ConfigReader.fromConfigs([]); - } + const configReader = ConfigReader.fromConfigs( + config.value?.length ? overrideBaseUrlConfigs(config.value) : [], + ); return { api: configReader }; } diff --git a/packages/core-app-api/src/app/overrideBaseUrlConfigs.ts b/packages/core-app-api/src/app/overrideBaseUrlConfigs.ts new file mode 100644 index 0000000000..5dab6db127 --- /dev/null +++ b/packages/core-app-api/src/app/overrideBaseUrlConfigs.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AppConfig, ConfigReader } from '@backstage/config'; + +/** + * Creates a base URL that uses to the current document origin. + */ +function createLocalBaseUrl(fullUrl: string): string { + const url = new URL(fullUrl); + url.protocol = document.location.protocol; + url.hostname = document.location.hostname; + url.port = document.location.port; + return url.toString().replace(/\/$/, ''); +} + +/** + * If we are able to override the app and backend base URLs to values that + * match the origin of the current location, then this function returns a + * new array of app configs that contain the overrides. + * + * @internal + */ +export function overrideBaseUrlConfigs(inputConfigs: AppConfig[]): AppConfig[] { + const urlConfigReader = ConfigReader.fromConfigs(inputConfigs); + + // In tests we may not have `app.baseUrl` or `backend.baseUrl`, to keep them optional + const appBaseUrl = urlConfigReader.getOptionalString('app.baseUrl'); + const backendBaseUrl = urlConfigReader.getOptionalString('backend.baseUrl'); + + let configs = inputConfigs; + + let newBackendBaseUrl: string | undefined = undefined; + let newAppBaseUrl: string | undefined = undefined; + + if (appBaseUrl && backendBaseUrl) { + const appOrigin = new URL(appBaseUrl).origin; + const backendOrigin = new URL(backendBaseUrl).origin; + + if (appOrigin === backendOrigin) { + const maybeNewBackendBaseUrl = createLocalBaseUrl(backendBaseUrl); + if (backendBaseUrl !== maybeNewBackendBaseUrl) { + newBackendBaseUrl = maybeNewBackendBaseUrl; + } + } + } + + if (appBaseUrl) { + const maybeNewAppBaseUrl = createLocalBaseUrl(appBaseUrl); + if (appBaseUrl !== maybeNewAppBaseUrl) { + newAppBaseUrl = maybeNewAppBaseUrl; + } + } + + // Only add the relative config if there is actually data to add. + if (newAppBaseUrl || newBackendBaseUrl) { + configs = configs.concat({ + data: { + app: newAppBaseUrl && { + baseUrl: newAppBaseUrl, + }, + backend: newBackendBaseUrl && { + baseUrl: newBackendBaseUrl, + }, + }, + context: 'relative-resolver', + }); + } + + return configs; +} From 7a77415a91f88e652dfffc9faf725baa41ff92f3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Sep 2023 14:34:26 +0200 Subject: [PATCH 080/131] frontend-app-api: use overrideBaseUrlConfigs Co-authored-by: Camila Belo Co-authored-by: Philipp Hugenroth Signed-off-by: Patrik Oldsberg --- packages/frontend-app-api/src/createApp.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/frontend-app-api/src/createApp.tsx b/packages/frontend-app-api/src/createApp.tsx index 7393740863..588ebc7f5e 100644 --- a/packages/frontend-app-api/src/createApp.tsx +++ b/packages/frontend-app-api/src/createApp.tsx @@ -53,6 +53,8 @@ import { AppThemeProvider } from '../../core-app-api/src/app/AppThemeProvider'; // eslint-disable-next-line @backstage/no-relative-monorepo-imports import { defaultConfigLoaderSync } from '../../core-app-api/src/app/defaultConfigLoader'; // eslint-disable-next-line @backstage/no-relative-monorepo-imports +import { overrideBaseUrlConfigs } from '../../core-app-api/src/app/overrideBaseUrlConfigs'; +// eslint-disable-next-line @backstage/no-relative-monorepo-imports import { themes } from '../../app-defaults/src/defaults/themes'; /** @public */ @@ -63,7 +65,8 @@ export function createApp(options: { createRoot(): JSX.Element; } { const appConfig = - options?.config ?? ConfigReader.fromConfigs(defaultConfigLoaderSync()); + options?.config ?? + ConfigReader.fromConfigs(overrideBaseUrlConfigs(defaultConfigLoaderSync())); const builtinExtensions = [CoreRouter, Core]; const discoveredPlugins = getAvailablePlugins(); From f0e3c10f4096095062fb1fb5c30471aa5f472db0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Sep 2023 15:32:22 +0200 Subject: [PATCH 081/131] frontend-app-api: add legacy app context and default APIs Co-authored-by: Camila Belo Co-authored-by: Philipp Hugenroth Signed-off-by: Patrik Oldsberg --- packages/frontend-app-api/src/createApp.tsx | 90 ++++++++++++++++++--- 1 file changed, 80 insertions(+), 10 deletions(-) diff --git a/packages/frontend-app-api/src/createApp.tsx b/packages/frontend-app-api/src/createApp.tsx index 588ebc7f5e..19ca7b367f 100644 --- a/packages/frontend-app-api/src/createApp.tsx +++ b/packages/frontend-app-api/src/createApp.tsx @@ -33,11 +33,16 @@ import { } from './wiring/parameters'; import { RoutingProvider } from './routing/RoutingContext'; import { + AnyApiFactory, ApiHolder, + AppComponents, + AppContext, appThemeApiRef, ConfigApi, configApiRef, + IconComponent, RouteRef, + BackstagePlugin as LegacyBackstagePlugin, } from '@backstage/core-plugin-api'; import { getAvailablePlugins } from './wiring/discovery'; import { @@ -51,11 +56,18 @@ import { // eslint-disable-next-line @backstage/no-relative-monorepo-imports import { AppThemeProvider } from '../../core-app-api/src/app/AppThemeProvider'; // eslint-disable-next-line @backstage/no-relative-monorepo-imports +import { AppContextProvider } from '../../core-app-api/src/app/AppContext'; +// eslint-disable-next-line @backstage/no-relative-monorepo-imports import { defaultConfigLoaderSync } from '../../core-app-api/src/app/defaultConfigLoader'; // eslint-disable-next-line @backstage/no-relative-monorepo-imports import { overrideBaseUrlConfigs } from '../../core-app-api/src/app/overrideBaseUrlConfigs'; // eslint-disable-next-line @backstage/no-relative-monorepo-imports -import { themes } from '../../app-defaults/src/defaults/themes'; +import { + apis as defaultApis, + components as defaultComponents, + icons as defaultIcons, + themes as defaultThemes, +} from '../../app-defaults/src/defaults'; /** @public */ export function createApp(options: { @@ -70,11 +82,12 @@ export function createApp(options: { const builtinExtensions = [CoreRouter, Core]; const discoveredPlugins = getAvailablePlugins(); + const allPlugins = [...discoveredPlugins, ...options.plugins]; // pull in default extension instance from discovered packages // apply config to adjust default extension instances and add more const extensionParams = mergeExtensionParameters({ - sources: [...options.plugins, ...discoveredPlugins], + sources: allPlugins, builtinExtensions, parameters: readAppExtensionParameters(appConfig), }); @@ -150,6 +163,8 @@ export function createApp(options: { const apiHolder = createApiHolder(coreInstance, appConfig); + const appContext = createLegacyAppContext(allPlugins); + return { createRoot() { const rootComponents = rootInstances @@ -162,19 +177,65 @@ export function createApp(options: { .filter(Boolean); return ( - - - {rootComponents.map((Component, i) => ( - - ))} - - + + + + {rootComponents.map((Component, i) => ( + + ))} + + + ); }, }; } +function toLegacyPlugin(plugin: BackstagePlugin): LegacyBackstagePlugin { + const errorMsg = 'Not implemented in legacy plugin compatibility layer'; + const notImplemented = () => { + throw new Error(errorMsg); + }; + return { + getId(): string { + return plugin.id; + }, + get routes(): never { + throw new Error(errorMsg); + }, + get externalRoutes(): never { + throw new Error(errorMsg); + }, + getApis: notImplemented, + getFeatureFlags: notImplemented, + provide: notImplemented, + __experimentalReconfigure: notImplemented, + }; +} + +function createLegacyAppContext(plugins: BackstagePlugin[]): AppContext { + return { + getPlugins(): LegacyBackstagePlugin[] { + return plugins.map(toLegacyPlugin); + }, + + getSystemIcon(key: string): IconComponent | undefined { + return key in defaultIcons + ? defaultIcons[key as keyof typeof defaultIcons] + : undefined; + }, + + getSystemIcons(): Record { + return defaultIcons; + }, + + getComponents(): AppComponents { + return defaultComponents; + }, + }; +} + function createApiHolder( coreExtension: ExtensionInstance, configApi: ConfigApi, @@ -200,7 +261,7 @@ function createApiHolder( api: appThemeApiRef, deps: {}, // TODO: add extension for registering themes - factory: () => AppThemeSelector.createWithStorage(themes), + factory: () => AppThemeSelector.createWithStorage(defaultThemes), }); factoryRegistry.register('static', { @@ -209,6 +270,15 @@ function createApiHolder( factory: () => configApi, }); + // TODO: ship these as default extensions instead + for (const factory of defaultApis as AnyApiFactory[]) { + if (!factoryRegistry.register('app', factory)) { + throw new Error( + `Duplicate or forbidden API factory for ${factory.api} in app`, + ); + } + } + ApiResolver.validateFactories(factoryRegistry, factoryRegistry.getAllApis()); return new ApiResolver(factoryRegistry); From 3b1d74770484a4a20e9b1629f880906967b911ed Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Sep 2023 15:42:13 +0200 Subject: [PATCH 082/131] app-next: remove legacy app wrapping Co-authored-by: Camila Belo Co-authored-by: Philipp Hugenroth Signed-off-by: Patrik Oldsberg --- packages/app-next/src/App.tsx | 6 ++---- packages/app-next/src/index.tsx | 5 ++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/app-next/src/App.tsx b/packages/app-next/src/App.tsx index 0452cc3709..aa9bb985c2 100644 --- a/packages/app-next/src/App.tsx +++ b/packages/app-next/src/App.tsx @@ -14,8 +14,6 @@ * limitations under the License. */ -import { graphiqlPlugin as legacyGraphiqlPlugin } from '@backstage/plugin-graphiql'; -import { createApp as createLegacyApp } from '@backstage/app-defaults'; import { createApp } from '@backstage/frontend-app-api'; import { pagesPlugin } from './examples/pagesPlugin'; import graphiqlPlugin from '@backstage/plugin-graphiql/alpha'; @@ -61,9 +59,9 @@ const app = createApp({ // }, }); -const legacyApp = createLegacyApp({ plugins: [legacyGraphiqlPlugin] }); +// const legacyApp = createLegacyApp({ plugins: [legacyGraphiqlPlugin] }); -export default legacyApp.createRoot(app.createRoot()); +export default app.createRoot(); // const routes = ( // diff --git a/packages/app-next/src/index.tsx b/packages/app-next/src/index.tsx index b15bc4c102..3c354b06d0 100644 --- a/packages/app-next/src/index.tsx +++ b/packages/app-next/src/index.tsx @@ -15,8 +15,7 @@ */ import '@backstage/cli/asset-types'; -import React from 'react'; import ReactDOM from 'react-dom'; -import App from './App'; +import app from './App'; -ReactDOM.render(, document.getElementById('root')); +ReactDOM.render(app, document.getElementById('root')); From 70169897f578f635e3d771edeec7be17b4045718 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Sep 2023 15:42:41 +0200 Subject: [PATCH 083/131] frontend-app-api: note on setting base path correctly Co-authored-by: Camila Belo Co-authored-by: Philipp Hugenroth Signed-off-by: Patrik Oldsberg --- packages/frontend-app-api/src/extensions/CoreRouter.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/frontend-app-api/src/extensions/CoreRouter.tsx b/packages/frontend-app-api/src/extensions/CoreRouter.tsx index 05f84aba6d..a148931b0f 100644 --- a/packages/frontend-app-api/src/extensions/CoreRouter.tsx +++ b/packages/frontend-app-api/src/extensions/CoreRouter.tsx @@ -48,6 +48,7 @@ export const CoreRouter = createExtension({ return element; }; bind({ + // TODO: set base path using the logic from AppRouter component: () => ( From c55a1a4a93125a576275daaeebc97b80ea4ff6a3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Sep 2023 16:35:59 +0200 Subject: [PATCH 084/131] frontend-app-api: implement core layout and initial sidebar Co-authored-by: Camila Belo Co-authored-by: Philipp Hugenroth Signed-off-by: Patrik Oldsberg --- packages/frontend-app-api/package.json | 2 + packages/frontend-app-api/src/createApp.tsx | 26 ++++-- .../src/extensions/CoreLayout.tsx | 68 +++++++++++++++ .../src/extensions/CoreNav.tsx | 84 +++++++++++++++++++ .../{CoreRouter.tsx => CoreRoutes.tsx} | 15 ++-- .../extensions/createPageExtension.test.tsx | 4 +- .../src/extensions/createPageExtension.tsx | 2 +- .../src/wiring/createPlugin.test.ts | 4 +- yarn.lock | 2 + 9 files changed, 187 insertions(+), 20 deletions(-) create mode 100644 packages/frontend-app-api/src/extensions/CoreLayout.tsx create mode 100644 packages/frontend-app-api/src/extensions/CoreNav.tsx rename packages/frontend-app-api/src/extensions/{CoreRouter.tsx => CoreRoutes.tsx} (80%) diff --git a/packages/frontend-app-api/package.json b/packages/frontend-app-api/package.json index 17f4b1bef3..f377be2122 100644 --- a/packages/frontend-app-api/package.json +++ b/packages/frontend-app-api/package.json @@ -35,10 +35,12 @@ "dependencies": { "@backstage/config": "workspace:^", "@backstage/core-app-api": "workspace:^", + "@backstage/core-components": "workspace:^", "@backstage/core-plugin-api": "workspace:^", "@backstage/frontend-plugin-api": "workspace:^", "@backstage/plugin-graphiql": "workspace:^", "@backstage/types": "workspace:^", + "@material-ui/core": "^4.12.4", "lodash": "^4.17.21" }, "peerDependencies": { diff --git a/packages/frontend-app-api/src/createApp.tsx b/packages/frontend-app-api/src/createApp.tsx index 19ca7b367f..4882191206 100644 --- a/packages/frontend-app-api/src/createApp.tsx +++ b/packages/frontend-app-api/src/createApp.tsx @@ -20,8 +20,10 @@ import { BackstagePlugin, coreExtensionData, } from '@backstage/frontend-plugin-api'; -import { CoreRouter } from './extensions/CoreRouter'; import { Core } from './extensions/Core'; +import { CoreRoutes } from './extensions/CoreRoutes'; +import { CoreLayout } from './extensions/CoreLayout'; +import { CoreNav } from './extensions/CoreNav'; import { createExtensionInstance, ExtensionInstance, @@ -43,6 +45,7 @@ import { IconComponent, RouteRef, BackstagePlugin as LegacyBackstagePlugin, + featureFlagsApiRef, } from '@backstage/core-plugin-api'; import { getAvailablePlugins } from './wiring/discovery'; import { @@ -58,6 +61,8 @@ import { AppThemeProvider } from '../../core-app-api/src/app/AppThemeProvider'; // eslint-disable-next-line @backstage/no-relative-monorepo-imports import { AppContextProvider } from '../../core-app-api/src/app/AppContext'; // eslint-disable-next-line @backstage/no-relative-monorepo-imports +import { LocalStorageFeatureFlags } from '../../core-app-api/src/apis/implementations/FeatureFlagsApi/LocalStorageFeatureFlags'; +// eslint-disable-next-line @backstage/no-relative-monorepo-imports import { defaultConfigLoaderSync } from '../../core-app-api/src/app/defaultConfigLoader'; // eslint-disable-next-line @backstage/no-relative-monorepo-imports import { overrideBaseUrlConfigs } from '../../core-app-api/src/app/overrideBaseUrlConfigs'; @@ -68,6 +73,7 @@ import { icons as defaultIcons, themes as defaultThemes, } from '../../app-defaults/src/defaults'; +import { BrowserRouter } from 'react-router-dom'; /** @public */ export function createApp(options: { @@ -80,7 +86,7 @@ export function createApp(options: { options?.config ?? ConfigReader.fromConfigs(overrideBaseUrlConfigs(defaultConfigLoaderSync())); - const builtinExtensions = [CoreRouter, Core]; + const builtinExtensions = [Core, CoreRoutes, CoreNav, CoreLayout]; const discoveredPlugins = getAvailablePlugins(); const allPlugins = [...discoveredPlugins, ...options.plugins]; @@ -180,9 +186,12 @@ export function createApp(options: { - {rootComponents.map((Component, i) => ( - - ))} + {/* TODO: set base path using the logic from AppRouter */} + + {rootComponents.map((Component, i) => ( + + ))} + @@ -257,6 +266,13 @@ function createApiHolder( factoryRegistry.register('default', factory); } + // TODO: properly discovery feature flags, maybe rework the whole thing + factoryRegistry.register('default', { + api: featureFlagsApiRef, + deps: {}, + factory: () => new LocalStorageFeatureFlags(), + }); + factoryRegistry.register('static', { api: appThemeApiRef, deps: {}, diff --git a/packages/frontend-app-api/src/extensions/CoreLayout.tsx b/packages/frontend-app-api/src/extensions/CoreLayout.tsx new file mode 100644 index 0000000000..845bc8cf5b --- /dev/null +++ b/packages/frontend-app-api/src/extensions/CoreLayout.tsx @@ -0,0 +1,68 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { + createExtension, + coreExtensionData, +} from '@backstage/frontend-plugin-api'; +import { SidebarPage } from '@backstage/core-components'; + +export const CoreLayout = createExtension({ + id: 'core.layout', + at: 'root', + inputs: { + nav: { + extensionData: { + component: coreExtensionData.reactComponent, + }, + }, + content: { + extensionData: { + component: coreExtensionData.reactComponent, + }, + }, + }, + output: { + component: coreExtensionData.reactComponent, + }, + factory({ bind, inputs }) { + // TODO: Support this as part of the core system + if (inputs.nav.length !== 1) { + throw Error( + `Extension 'core.layout' did not receive exactly one 'nav' input, got ${inputs.nav.length}`, + ); + } + const Nav = inputs.nav[0].component; + + if (inputs.content.length !== 1) { + throw Error( + `Extension 'core.layout' did not receive exactly one 'content' input, got ${inputs.content.length}`, + ); + } + const Content = inputs.content[0].component; + + bind({ + // TODO: set base path using the logic from AppRouter + component: () => ( + +