From 4611d59fdc4829a1c8a23df5536e76cee2cca1b2 Mon Sep 17 00:00:00 2001 From: Alex Krantz Date: Sat, 9 Oct 2021 17:29:13 -0700 Subject: [PATCH 001/103] fix: require https for Okta audience URL Signed-off-by: Alex Krantz --- plugins/auth-backend/src/providers/okta/provider.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/auth-backend/src/providers/okta/provider.ts b/plugins/auth-backend/src/providers/okta/provider.ts index 44e26bb7fe..7ea5b29e80 100644 --- a/plugins/auth-backend/src/providers/okta/provider.ts +++ b/plugins/auth-backend/src/providers/okta/provider.ts @@ -275,6 +275,10 @@ export const createOktaProvider = ( const audience = envConfig.getString('audience'); const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; + if (!audience.startsWith('https')) { + throw new Error("URL for 'audience' must start with 'https'."); + } + const catalogIdentityClient = new CatalogIdentityClient({ catalogApi, tokenIssuer, From 9322e632e9c7d9f5ebe46871ec371bb9f1e248be Mon Sep 17 00:00:00 2001 From: Alex Krantz Date: Sat, 9 Oct 2021 17:35:57 -0700 Subject: [PATCH 002/103] chore: add changeset Signed-off-by: Alex Krantz --- .changeset/blue-dingos-repeat.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/blue-dingos-repeat.md diff --git a/.changeset/blue-dingos-repeat.md b/.changeset/blue-dingos-repeat.md new file mode 100644 index 0000000000..111a94a00b --- /dev/null +++ b/.changeset/blue-dingos-repeat.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Require that audience URLs for Okta authentication start with https From bb8cea4a75bf350d03eb15eb072b85321e031954 Mon Sep 17 00:00:00 2001 From: Aldira Putra Raharja Date: Mon, 11 Oct 2021 21:51:01 +0700 Subject: [PATCH 003/103] Add `caData` support for kubernetes config Signed-off-by: Aldira Putra Raharja --- .../src/cluster-locator/ConfigClusterLocator.test.ts | 6 ++++++ .../src/cluster-locator/ConfigClusterLocator.ts | 1 + .../kubernetes-backend/src/cluster-locator/index.test.ts | 2 ++ .../src/service/KubernetesClientProvider.ts | 1 + plugins/kubernetes-backend/src/types/types.ts | 1 + 5 files changed, 11 insertions(+) diff --git a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts index 6d1506c4c7..62346ac6c3 100644 --- a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts +++ b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts @@ -53,6 +53,7 @@ describe('ConfigClusterLocator', () => { url: 'http://localhost:8080', authProvider: 'serviceAccount', skipTLSVerify: false, + caData: undefined, }, ]); }); @@ -89,6 +90,7 @@ describe('ConfigClusterLocator', () => { url: 'http://localhost:8080', authProvider: 'serviceAccount', skipTLSVerify: false, + caData: undefined, }, { name: 'cluster2', @@ -96,6 +98,7 @@ describe('ConfigClusterLocator', () => { url: 'http://localhost:8081', authProvider: 'google', skipTLSVerify: true, + caData: undefined, }, ]); }); @@ -141,6 +144,7 @@ describe('ConfigClusterLocator', () => { url: 'http://localhost:8080', authProvider: 'aws', skipTLSVerify: false, + caData: undefined, }, { assumeRole: 'SomeRole', @@ -150,6 +154,7 @@ describe('ConfigClusterLocator', () => { url: 'http://localhost:8081', authProvider: 'aws', skipTLSVerify: true, + caData: undefined, }, { assumeRole: 'SomeRole', @@ -159,6 +164,7 @@ describe('ConfigClusterLocator', () => { serviceAccountToken: undefined, authProvider: 'aws', skipTLSVerify: true, + caData: undefined, }, ]); }); diff --git a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts index 53d3a1d025..bb9dbfe13f 100644 --- a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts +++ b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts @@ -35,6 +35,7 @@ export class ConfigClusterLocator implements KubernetesClustersSupplier { url: c.getString('url'), serviceAccountToken: c.getOptionalString('serviceAccountToken'), skipTLSVerify: c.getOptionalBoolean('skipTLSVerify') ?? false, + caData: c.getOptionalString('caData'), authProvider: authProvider, }; const dashboardUrl = c.getOptionalString('dashboardUrl'); diff --git a/plugins/kubernetes-backend/src/cluster-locator/index.test.ts b/plugins/kubernetes-backend/src/cluster-locator/index.test.ts index 95be99a8a5..dde4dc1d93 100644 --- a/plugins/kubernetes-backend/src/cluster-locator/index.test.ts +++ b/plugins/kubernetes-backend/src/cluster-locator/index.test.ts @@ -54,6 +54,7 @@ describe('getCombinedClusterDetails', () => { url: 'http://localhost:8080', authProvider: 'serviceAccount', skipTLSVerify: false, + caData: undefined, }, { name: 'cluster2', @@ -61,6 +62,7 @@ describe('getCombinedClusterDetails', () => { url: 'http://localhost:8081', authProvider: 'google', skipTLSVerify: false, + caData: undefined, }, ]); }); diff --git a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts b/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts index 61b19854c2..4b2848508e 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts @@ -31,6 +31,7 @@ export class KubernetesClientProvider { name: clusterDetails.name, server: clusterDetails.url, skipTLSVerify: clusterDetails.skipTLSVerify, + caData: clusterDetails.caData, }; // TODO configure diff --git a/plugins/kubernetes-backend/src/types/types.ts b/plugins/kubernetes-backend/src/types/types.ts index cff87f265a..0e78f64f01 100644 --- a/plugins/kubernetes-backend/src/types/types.ts +++ b/plugins/kubernetes-backend/src/types/types.ts @@ -88,6 +88,7 @@ export interface ClusterDetails { authProvider: string; serviceAccountToken?: string | undefined; skipTLSVerify?: boolean; + caData?: string | undefined; /** * Specifies the link to the Kubernetes dashboard managing this cluster. * @remarks From c57b075d18fd39aee83d30c914e3a9aabac0175e Mon Sep 17 00:00:00 2001 From: Aldira Putra Raharja Date: Mon, 11 Oct 2021 21:54:40 +0700 Subject: [PATCH 004/103] include changeset Signed-off-by: Aldira Putra Raharja --- .changeset/polite-eyes-punch.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/polite-eyes-punch.md diff --git a/.changeset/polite-eyes-punch.md b/.changeset/polite-eyes-punch.md new file mode 100644 index 0000000000..fed1116bb2 --- /dev/null +++ b/.changeset/polite-eyes-punch.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-kubernetes-backend': patch +--- + +add caData support for kubernetes client config From 17fdd3aea4c1861381d78f527afd35a427540f9e Mon Sep 17 00:00:00 2001 From: Aldira Putra Raharja Date: Mon, 11 Oct 2021 22:05:04 +0700 Subject: [PATCH 005/103] Update docs regarding caData config Signed-off-by: Aldira Putra Raharja --- docs/features/kubernetes/configuration.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/features/kubernetes/configuration.md b/docs/features/kubernetes/configuration.md index f48faa202d..6c9bf43da4 100644 --- a/docs/features/kubernetes/configuration.md +++ b/docs/features/kubernetes/configuration.md @@ -29,6 +29,7 @@ kubernetes: serviceAccountToken: ${K8S_MINIKUBE_TOKEN} dashboardUrl: http://127.0.0.1:64713 # url copied from running the command: minikube service kubernetes-dashboard -n kubernetes-dashboard dashboardApp: standard + caData: ${K8S_CONFIG_CA_DATA} - url: http://127.0.0.2:9999 name: aws-cluster-1 authProvider: 'aws' @@ -135,6 +136,10 @@ See also https://github.com/backstage/backstage/tree/master/plugins/kubernetes/src/utils/clusterLinks/formatters for real examples. +##### `clusters.\*.caData` (optional) + +PEM-encoded bytes (typically read from a client certificate file). + #### `gke` This cluster locator is designed to work with Kubernetes clusters running in From 2bacec55312dc55a98a47e6a24a9015440c6e7e2 Mon Sep 17 00:00:00 2001 From: Aldira Putra Raharja Date: Mon, 11 Oct 2021 22:16:02 +0700 Subject: [PATCH 006/103] Rephrase docs to match descriptions from the official docs Signed-off-by: Aldira Putra Raharja --- docs/features/kubernetes/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/kubernetes/configuration.md b/docs/features/kubernetes/configuration.md index 6c9bf43da4..fc32f4da70 100644 --- a/docs/features/kubernetes/configuration.md +++ b/docs/features/kubernetes/configuration.md @@ -138,7 +138,7 @@ for real examples. ##### `clusters.\*.caData` (optional) -PEM-encoded bytes (typically read from a client certificate file). +PEM-encoded certificate authority certificates. #### `gke` From 037447efd3272693bf3a834c6342eb2de3cf2565 Mon Sep 17 00:00:00 2001 From: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> Date: Mon, 11 Oct 2021 15:04:59 -0400 Subject: [PATCH 007/103] adds atlassian auth provider Signed-off-by: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> --- packages/app/src/identityProviders.ts | 7 + .../auth/atlassian/AtlassianAuth.ts | 44 +++ .../implementations/auth/atlassian/index.ts | 17 ++ .../src/apis/implementations/auth/index.ts | 1 + packages/core-app-api/src/app/defaultApis.ts | 17 ++ .../src/apis/definitions/auth.ts | 12 + .../src/providers/atlassian/index.ts | 18 ++ .../src/providers/atlassian/provider.test.ts | 96 +++++++ .../src/providers/atlassian/provider.ts | 271 ++++++++++++++++++ .../src/providers/atlassian/strategy.ts | 111 +++++++ .../auth-backend/src/providers/factories.ts | 2 + plugins/auth-backend/src/providers/index.ts | 1 + .../AuthProviders/DefaultProviderSettings.tsx | 9 + 13 files changed, 606 insertions(+) create mode 100644 packages/core-app-api/src/apis/implementations/auth/atlassian/AtlassianAuth.ts create mode 100644 packages/core-app-api/src/apis/implementations/auth/atlassian/index.ts create mode 100644 plugins/auth-backend/src/providers/atlassian/index.ts create mode 100644 plugins/auth-backend/src/providers/atlassian/provider.test.ts create mode 100644 plugins/auth-backend/src/providers/atlassian/provider.ts create mode 100644 plugins/auth-backend/src/providers/atlassian/strategy.ts diff --git a/packages/app/src/identityProviders.ts b/packages/app/src/identityProviders.ts index 49204d2b5e..70630a3d0a 100644 --- a/packages/app/src/identityProviders.ts +++ b/packages/app/src/identityProviders.ts @@ -25,6 +25,7 @@ import { oauth2ApiRef, oidcAuthApiRef, bitbucketAuthApiRef, + atlassianAuthApiRef, } from '@backstage/core-plugin-api'; export const providers = [ @@ -88,4 +89,10 @@ export const providers = [ message: 'Sign In using Bitbucket', apiRef: bitbucketAuthApiRef, }, + { + id: 'atlassian-auth-provider', + title: 'Atlassian', + message: 'Sign In using Atlassian', + apiRef: atlassianAuthApiRef, + }, ]; diff --git a/packages/core-app-api/src/apis/implementations/auth/atlassian/AtlassianAuth.ts b/packages/core-app-api/src/apis/implementations/auth/atlassian/AtlassianAuth.ts new file mode 100644 index 0000000000..423c45bc6c --- /dev/null +++ b/packages/core-app-api/src/apis/implementations/auth/atlassian/AtlassianAuth.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AtlassianIcon from '@material-ui/icons/AcUnit'; +import { atlassianAuthApiRef } from '@backstage/core-plugin-api'; +import { OAuth2 } from '../oauth2'; +import { OAuthApiCreateOptions } from '../types'; + +const DEFAULT_PROVIDER = { + id: 'atlassian', + title: 'Atlassian', + icon: AtlassianIcon, +}; + +class AtlassianAuth { + static create({ + discoveryApi, + environment = 'development', + provider = DEFAULT_PROVIDER, + oauthRequestApi, + }: OAuthApiCreateOptions): typeof atlassianAuthApiRef.T { + return OAuth2.create({ + discoveryApi, + oauthRequestApi, + provider, + environment, + }); + } +} + +export default AtlassianAuth; diff --git a/packages/core-app-api/src/apis/implementations/auth/atlassian/index.ts b/packages/core-app-api/src/apis/implementations/auth/atlassian/index.ts new file mode 100644 index 0000000000..fb787be1ce --- /dev/null +++ b/packages/core-app-api/src/apis/implementations/auth/atlassian/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { default as AtlassianAuth } from './AtlassianAuth'; diff --git a/packages/core-app-api/src/apis/implementations/auth/index.ts b/packages/core-app-api/src/apis/implementations/auth/index.ts index b622b90b8c..bbc9d23ccc 100644 --- a/packages/core-app-api/src/apis/implementations/auth/index.ts +++ b/packages/core-app-api/src/apis/implementations/auth/index.ts @@ -24,3 +24,4 @@ export * from './auth0'; export * from './microsoft'; export * from './onelogin'; export * from './bitbucket'; +export * from './atlassian'; diff --git a/packages/core-app-api/src/app/defaultApis.ts b/packages/core-app-api/src/app/defaultApis.ts index 5e850f4cce..fb02274cf0 100644 --- a/packages/core-app-api/src/app/defaultApis.ts +++ b/packages/core-app-api/src/app/defaultApis.ts @@ -33,6 +33,7 @@ import { SamlAuth, OneLoginAuth, UnhandledErrorForwarder, + AtlassianAuth, } from '../apis'; import { @@ -55,6 +56,7 @@ import { oneloginAuthApiRef, oidcAuthApiRef, bitbucketAuthApiRef, + atlassianAuthApiRef, } from '@backstage/core-plugin-api'; import OAuth2Icon from '@material-ui/icons/AcUnit'; @@ -244,4 +246,19 @@ export const defaultApis = [ environment: configApi.getOptionalString('auth.environment'), }), }), + createApiFactory({ + api: atlassianAuthApiRef, + deps: { + discoveryApi: discoveryApiRef, + oauthRequestApi: oauthRequestApiRef, + configApi: configApiRef, + }, + factory: ({ discoveryApi, oauthRequestApi, configApi }) => { + return AtlassianAuth.create({ + discoveryApi, + oauthRequestApi, + environment: configApi.getOptionalString('auth.environment'), + }); + }, + }), ]; diff --git a/packages/core-plugin-api/src/apis/definitions/auth.ts b/packages/core-plugin-api/src/apis/definitions/auth.ts index a78414048c..f67614b758 100644 --- a/packages/core-plugin-api/src/apis/definitions/auth.ts +++ b/packages/core-plugin-api/src/apis/definitions/auth.ts @@ -352,3 +352,15 @@ export const bitbucketAuthApiRef: ApiRef< > = createApiRef({ id: 'core.auth.bitbucket', }); + +/** + * Provides authentication towards Atlassian APIs. + * + * See https://developer.atlassian.com/cloud/jira/platform/scopes-for-connect-and-oauth-2-3LO-apps/ + * for a full list of supported scopes. + */ +export const atlassianAuthApiRef: ApiRef< + OAuthApi & ProfileInfoApi & BackstageIdentityApi & SessionApi +> = createApiRef({ + id: 'core.auth.atlassian', +}); diff --git a/plugins/auth-backend/src/providers/atlassian/index.ts b/plugins/auth-backend/src/providers/atlassian/index.ts new file mode 100644 index 0000000000..b99b63d7bc --- /dev/null +++ b/plugins/auth-backend/src/providers/atlassian/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createAtlassianProvider } from './provider'; +export type { AtlassianAuthProvider } from './provider'; diff --git a/plugins/auth-backend/src/providers/atlassian/provider.test.ts b/plugins/auth-backend/src/providers/atlassian/provider.test.ts new file mode 100644 index 0000000000..e561c2e037 --- /dev/null +++ b/plugins/auth-backend/src/providers/atlassian/provider.test.ts @@ -0,0 +1,96 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AtlassianAuthProvider } from './provider'; +import * as helpers from '../../lib/passport/PassportStrategyHelper'; +import { getVoidLogger } from '@backstage/backend-common'; +import { TokenIssuer } from '../../identity'; +import { CatalogIdentityClient } from '../../lib/catalog'; +import { OAuthResult } from '../../lib/oauth'; + +const mockFrameHandler = jest.spyOn( + helpers, + 'executeFrameHandlerStrategy', +) as unknown as jest.MockedFunction<() => Promise<{ result: OAuthResult }>>; + +describe('createAtlassianProvider', () => { + it('should auth', async () => { + const tokenIssuer = { + issueToken: jest.fn(), + listPublicKeys: jest.fn(), + }; + const catalogIdentityClient = { + findUser: jest.fn(), + }; + + const provider = new AtlassianAuthProvider({ + logger: getVoidLogger(), + catalogIdentityClient: + catalogIdentityClient as unknown as CatalogIdentityClient, + tokenIssuer: tokenIssuer as unknown as TokenIssuer, + authHandler: async ({ fullProfile }) => ({ + profile: { + email: fullProfile.emails![0]!.value, + displayName: fullProfile.displayName, + picture: 'http://google.com/lols', + }, + }), + clientId: 'mock', + clientSecret: 'mock', + callbackUrl: 'mock', + scopes: [], + }); + + mockFrameHandler.mockResolvedValueOnce({ + result: { + fullProfile: { + photos: [ + { + value: + 'https://a1cf74336522e87f135f-2f21ace9a6cf0052456644b80fa06d4f.ssl.cf2.rackcdn.com/images/characters_opt/p-mystic-river-sean-penn.jpg', + }, + ], + emails: [{ value: 'conrad@example.com' }], + displayName: 'Conrad', + id: 'conrad', + provider: 'google', + }, + params: { + id_token: 'idToken', + scope: 'scope', + expires_in: 123, + }, + accessToken: 'accessToken', + refreshToken: 'wacka', + }, + }); + const { response } = await provider.handler({} as any); + expect(response).toEqual({ + providerInfo: { + accessToken: 'accessToken', + expiresInSeconds: 123, + idToken: 'idToken', + scope: 'scope', + refreshToken: 'wacka', + }, + profile: { + email: 'conrad@example.com', + displayName: 'Conrad', + picture: 'http://google.com/lols', + }, + }); + }); +}); diff --git a/plugins/auth-backend/src/providers/atlassian/provider.ts b/plugins/auth-backend/src/providers/atlassian/provider.ts new file mode 100644 index 0000000000..a4ab5600e1 --- /dev/null +++ b/plugins/auth-backend/src/providers/atlassian/provider.ts @@ -0,0 +1,271 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AtlassianStrategy from './strategy'; +import { + encodeState, + OAuthAdapter, + OAuthEnvironmentHandler, + OAuthHandlers, + OAuthProviderOptions, + OAuthRefreshRequest, + OAuthResponse, + OAuthResult, + OAuthStartRequest, +} from '../../lib/oauth'; +import passport from 'passport'; +import { + executeFetchUserProfileStrategy, + executeFrameHandlerStrategy, + executeRedirectStrategy, + executeRefreshTokenStrategy, + makeProfileInfo, + PassportDoneCallback, +} from '../../lib/passport'; +import { + AuthHandler, + AuthProviderFactory, + RedirectInfo, + SignInResolver, +} from '../types'; +import express from 'express'; +import { TokenIssuer } from '../../identity'; +import { CatalogIdentityClient } from '../../lib/catalog'; +import { Logger } from 'winston'; + +export type AtlassianAuthProviderOptions = OAuthProviderOptions & { + scopes: string[]; + signInResolver?: SignInResolver; + authHandler: AuthHandler; + tokenIssuer: TokenIssuer; + catalogIdentityClient: CatalogIdentityClient; + logger: Logger; +}; + +export const atlassianDefaultSignInResolver: SignInResolver = + async (info, ctx) => { + const { profile, result } = info; + + let id = result.fullProfile.id; + + if (profile.email) { + id = profile.email.split('@')[0]; + } + + const token = await ctx.tokenIssuer.issueToken({ + claims: { sub: id, ent: [`user:default/${id}`] }, + }); + + return { id, token }; + }; + +export const atlassianDefaultAuthHandler: AuthHandler = async ({ + fullProfile, + params, +}) => ({ + profile: makeProfileInfo(fullProfile, params.id_token), +}); + +export class AtlassianAuthProvider implements OAuthHandlers { + private readonly _strategy: AtlassianStrategy; + private readonly signInResolver?: SignInResolver; + private readonly authHandler: AuthHandler; + private readonly tokenIssuer: TokenIssuer; + private readonly catalogIdentityClient: CatalogIdentityClient; + private readonly logger: Logger; + + constructor(options: AtlassianAuthProviderOptions) { + this.catalogIdentityClient = options.catalogIdentityClient; + this.logger = options.logger; + this.tokenIssuer = options.tokenIssuer; + this.authHandler = options.authHandler; + this.signInResolver = options.signInResolver; + + this._strategy = new AtlassianStrategy( + { + clientID: options.clientId, + clientSecret: options.clientSecret, + callbackURL: options.callbackUrl, + scope: options.scopes, + }, + ( + accessToken: any, + refreshToken: any, + params: any, + fullProfile: passport.Profile, + done: PassportDoneCallback, + ) => { + done(undefined, { + fullProfile, + accessToken, + refreshToken, + params, + }); + }, + ); + } + + async start(req: OAuthStartRequest): Promise { + return await executeRedirectStrategy(req, this._strategy, { + state: encodeState(req.state), + }); + } + + async handler( + req: express.Request, + ): Promise<{ response: OAuthResponse; refreshToken: string }> { + const { result } = await executeFrameHandlerStrategy( + req, + this._strategy, + ); + + return { + response: await this.handleResult(result), + refreshToken: result.refreshToken ?? '', + }; + } + + private async handleResult(result: OAuthResult): Promise { + const { profile } = await this.authHandler(result); + + const response: OAuthResponse = { + providerInfo: { + idToken: result.params.id_token, + accessToken: result.accessToken, + refreshToken: result.refreshToken, // GitLab expires the old refresh token when used + scope: result.params.scope, + expiresInSeconds: result.params.expires_in, + }, + profile, + }; + + if (this.signInResolver) { + response.backstageIdentity = await this.signInResolver( + { + result, + profile, + }, + { + tokenIssuer: this.tokenIssuer, + catalogIdentityClient: this.catalogIdentityClient, + logger: this.logger, + }, + ); + } + + return response; + } + + async refresh(req: OAuthRefreshRequest): Promise { + const { + accessToken, + params, + refreshToken: newRefreshToken, + } = await executeRefreshTokenStrategy( + this._strategy, + req.refreshToken, + req.scope, + ); + + const fullProfile = await executeFetchUserProfileStrategy( + this._strategy, + accessToken, + ); + + return this.handleResult({ + fullProfile, + params, + accessToken, + refreshToken: newRefreshToken, + }); + } +} + +export type AtlassianProviderOptions = { + /** + * The profile transformation function used to verify and convert the auth response + * into the profile that will be presented to the user. + */ + authHandler?: AuthHandler; + + /** + * Configure sign-in for this provider, without it the provider can not be used to sign users in. + */ + /** + * Maps an auth result to a Backstage identity for the user. + * + * Set to `'email'` to use the default email-based sign in resolver, which will search + * the catalog for a single user entity that has a matching `microsoft.com/email` annotation. + */ + signIn?: { + resolver?: SignInResolver; + }; +}; + +export const createAtlassianProvider = ( + options?: AtlassianProviderOptions, +): AuthProviderFactory => { + return ({ + providerId, + globalConfig, + config, + tokenIssuer, + catalogApi, + logger, + }) => + OAuthEnvironmentHandler.mapConfig(config, envConfig => { + const clientId = envConfig.getString('clientId'); + const clientSecret = envConfig.getString('clientSecret'); + const scopes = envConfig.getStringArray('scopes'); + const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; + + const catalogIdentityClient = new CatalogIdentityClient({ + catalogApi, + tokenIssuer, + }); + + const authHandler: AuthHandler = + options?.authHandler ?? atlassianDefaultAuthHandler; + + const signInResolverFn = + options?.signIn?.resolver ?? atlassianDefaultSignInResolver; + + const signInResolver: SignInResolver = info => + signInResolverFn(info, { + catalogIdentityClient, + tokenIssuer, + logger, + }); + + const provider = new AtlassianAuthProvider({ + clientId, + clientSecret, + scopes, + callbackUrl, + authHandler, + signInResolver, + catalogIdentityClient, + logger, + tokenIssuer, + }); + + return OAuthAdapter.fromConfig(globalConfig, provider, { + disableRefresh: true, + providerId, + tokenIssuer, + }); + }); +}; diff --git a/plugins/auth-backend/src/providers/atlassian/strategy.ts b/plugins/auth-backend/src/providers/atlassian/strategy.ts new file mode 100644 index 0000000000..630348b16d --- /dev/null +++ b/plugins/auth-backend/src/providers/atlassian/strategy.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import OAuth2Strategy, { InternalOAuthError } from 'passport-oauth2'; +import { Profile } from 'passport'; + +interface ProfileResponse { + account_id: string; + email: string; + name: string; + picture: string; + nickname: string; +} + +interface AtlassianStrategyOptions { + clientID: string; + clientSecret: string; + callbackURL: string; + scope: string[]; +} + +const defaultScopes = ['offline_access', 'read:me']; + +export default class AtlassianStrategy extends OAuth2Strategy { + private readonly profileURL: string; + + constructor( + options: AtlassianStrategyOptions, + verify: OAuth2Strategy.VerifyFunction, + ) { + if (!options.scope) { + throw new TypeError('Atlassian requires a scope option'); + } + + const optionsWithURLs = { + ...options, + authorizationURL: `https://auth.atlassian.com/authorize`, + tokenURL: `https://auth.atlassian.com/oauth/token`, + scope: Array.from(new Set([...defaultScopes, ...options.scope])), + }; + + super(optionsWithURLs, verify); + this.profileURL = 'https://api.atlassian.com/me'; + this.name = 'atlassian'; + + this._oauth2.useAuthorizationHeaderforGET(true); + } + + authorizationParams() { + return { + audience: 'api.atlassian.com', + prompt: 'consent', + }; + } + + userProfile( + accessToken: string, + done: (err?: Error | null, profile?: any) => void, + ): void { + this._oauth2.get(this.profileURL, accessToken, (err, body) => { + if (err) { + return done( + new InternalOAuthError( + 'Failed to fetch user profile', + err.statusCode, + ), + ); + } + + if (!body) { + return done( + new Error('Failed to fetch user profile, body cannot be empty'), + ); + } + + try { + const json = typeof body !== 'string' ? body.toString() : body; + const profile = AtlassianStrategy.parse(json); + return done(null, profile); + } catch (e) { + return done(new Error('Failed to parse user profile')); + } + }); + } + + static parse(json: string): Profile { + const resp = JSON.parse(json) as ProfileResponse; + + return { + id: resp.account_id, + provider: 'atlassian', + username: resp.nickname, + displayName: resp.name, + emails: [{ value: resp.email }], + photos: [{ value: resp.picture }], + }; + } +} diff --git a/plugins/auth-backend/src/providers/factories.ts b/plugins/auth-backend/src/providers/factories.ts index c4894f2757..42d3569815 100644 --- a/plugins/auth-backend/src/providers/factories.ts +++ b/plugins/auth-backend/src/providers/factories.ts @@ -27,6 +27,7 @@ import { createOneLoginProvider } from './onelogin'; import { AuthProviderFactory } from './types'; import { createAwsAlbProvider } from './aws-alb'; import { createBitbucketProvider } from './bitbucket'; +import { createAtlassianProvider } from './atlassian'; export const factories: { [providerId: string]: AuthProviderFactory } = { google: createGoogleProvider(), @@ -41,4 +42,5 @@ export const factories: { [providerId: string]: AuthProviderFactory } = { onelogin: createOneLoginProvider(), awsalb: createAwsAlbProvider(), bitbucket: createBitbucketProvider(), + atlassian: createAtlassianProvider(), }; diff --git a/plugins/auth-backend/src/providers/index.ts b/plugins/auth-backend/src/providers/index.ts index 90466a8094..8c5f05fefd 100644 --- a/plugins/auth-backend/src/providers/index.ts +++ b/plugins/auth-backend/src/providers/index.ts @@ -21,6 +21,7 @@ export * from './microsoft'; export * from './oauth2'; export * from './okta'; export * from './bitbucket'; +export * from './atlassian'; export { factories as defaultAuthProviderFactories } from './factories'; diff --git a/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx b/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx index 4b9cf9c783..8cfb3a7257 100644 --- a/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx +++ b/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx @@ -25,6 +25,7 @@ import { oktaAuthApiRef, microsoftAuthApiRef, bitbucketAuthApiRef, + atlassianAuthApiRef, } from '@backstage/core-plugin-api'; type Props = { @@ -89,6 +90,14 @@ export const DefaultProviderSettings = ({ configuredProviders }: Props) => ( icon={Star} /> )} + {configuredProviders.includes('atlassian') && ( + + )} {configuredProviders.includes('oauth2') && ( Date: Mon, 11 Oct 2021 16:45:50 -0400 Subject: [PATCH 008/103] updates scope configuration Signed-off-by: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> --- app-config.yaml | 5 + .../src/providers/atlassian/provider.test.ts | 114 +++++++++++++----- .../src/providers/atlassian/provider.ts | 6 +- .../src/providers/atlassian/strategy.ts | 6 +- 4 files changed, 99 insertions(+), 32 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index c24ac1701b..e1b648c472 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -364,6 +364,11 @@ auth: development: clientId: ${AUTH_BITBUCKET_CLIENT_ID} clientSecret: ${AUTH_BITBUCKET_CLIENT_SECRET} + atlassian: + development: + clientId: ${AUTH_ATLASSIAN_CLIENT_ID} + clientSecret: ${AUTH_ATLASSIAN_CLIENT_SECRET} + scopes: ${AUTH_ATLASSIAN_SCOPES} costInsights: engineerCost: 200000 products: diff --git a/plugins/auth-backend/src/providers/atlassian/provider.test.ts b/plugins/auth-backend/src/providers/atlassian/provider.test.ts index e561c2e037..58c7468246 100644 --- a/plugins/auth-backend/src/providers/atlassian/provider.test.ts +++ b/plugins/auth-backend/src/providers/atlassian/provider.test.ts @@ -14,12 +14,16 @@ * limitations under the License. */ -import { AtlassianAuthProvider } from './provider'; +import { + AtlassianAuthProvider, + atlassianDefaultSignInResolver, +} from './provider'; import * as helpers from '../../lib/passport/PassportStrategyHelper'; import { getVoidLogger } from '@backstage/backend-common'; import { TokenIssuer } from '../../identity'; import { CatalogIdentityClient } from '../../lib/catalog'; import { OAuthResult } from '../../lib/oauth'; +import { PassportProfile } from '../../lib/passport/types'; const mockFrameHandler = jest.spyOn( helpers, @@ -27,33 +31,34 @@ const mockFrameHandler = jest.spyOn( ) as unknown as jest.MockedFunction<() => Promise<{ result: OAuthResult }>>; describe('createAtlassianProvider', () => { + const tokenIssuer = { + issueToken: jest.fn(), + listPublicKeys: jest.fn(), + }; + const catalogIdentityClient = { + findUser: jest.fn(), + }; + + const provider = new AtlassianAuthProvider({ + logger: getVoidLogger(), + catalogIdentityClient: + catalogIdentityClient as unknown as CatalogIdentityClient, + tokenIssuer: tokenIssuer as unknown as TokenIssuer, + authHandler: async ({ fullProfile }) => ({ + profile: { + email: fullProfile.emails![0]!.value, + displayName: fullProfile.displayName, + picture: 'http://google.com/lols', + }, + }), + clientId: 'mock', + clientSecret: 'mock', + callbackUrl: 'mock', + scopes: [], + signInResolver: atlassianDefaultSignInResolver, + }); + it('should auth', async () => { - const tokenIssuer = { - issueToken: jest.fn(), - listPublicKeys: jest.fn(), - }; - const catalogIdentityClient = { - findUser: jest.fn(), - }; - - const provider = new AtlassianAuthProvider({ - logger: getVoidLogger(), - catalogIdentityClient: - catalogIdentityClient as unknown as CatalogIdentityClient, - tokenIssuer: tokenIssuer as unknown as TokenIssuer, - authHandler: async ({ fullProfile }) => ({ - profile: { - email: fullProfile.emails![0]!.value, - displayName: fullProfile.displayName, - picture: 'http://google.com/lols', - }, - }), - clientId: 'mock', - clientSecret: 'mock', - callbackUrl: 'mock', - scopes: [], - }); - mockFrameHandler.mockResolvedValueOnce({ result: { fullProfile: { @@ -79,6 +84,9 @@ describe('createAtlassianProvider', () => { }); const { response } = await provider.handler({} as any); expect(response).toEqual({ + backstageIdentity: { + id: 'conrad', + }, providerInfo: { accessToken: 'accessToken', expiresInSeconds: 123, @@ -93,4 +101,56 @@ describe('createAtlassianProvider', () => { }, }); }); + + it('should forward a new refresh token on refresh', async () => { + const mockRefreshToken = jest.spyOn( + helpers, + 'executeRefreshTokenStrategy', + ) as unknown as jest.MockedFunction<() => Promise<{}>>; + + mockRefreshToken.mockResolvedValueOnce({ + accessToken: 'a.b.c', + refreshToken: 'dont-forget-to-send-refresh', + params: { + id_token: 'my-id', + scope: 'read_user', + }, + }); + + const mockUserProfile = jest.spyOn( + helpers, + 'executeFetchUserProfileStrategy', + ) as unknown as jest.MockedFunction<() => Promise>; + + mockUserProfile.mockResolvedValueOnce({ + id: 'uid-my-id', + username: 'mockuser', + provider: 'atlassian', + displayName: 'Mocked User', + emails: [ + { + value: 'mockuser@gmail.com', + }, + ], + }); + + const response = await provider.refresh({} as any); + + expect(response).toEqual({ + backstageIdentity: { + id: 'mockuser', + }, + profile: { + displayName: 'Mocked User', + email: 'mockuser@gmail.com', + picture: 'http://google.com/lols', + }, + providerInfo: { + accessToken: 'a.b.c', + idToken: 'my-id', + refreshToken: 'dont-forget-to-send-refresh', + scope: 'read_user', + }, + }); + }); }); diff --git a/plugins/auth-backend/src/providers/atlassian/provider.ts b/plugins/auth-backend/src/providers/atlassian/provider.ts index a4ab5600e1..c7d1a9590c 100644 --- a/plugins/auth-backend/src/providers/atlassian/provider.ts +++ b/plugins/auth-backend/src/providers/atlassian/provider.ts @@ -145,7 +145,7 @@ export class AtlassianAuthProvider implements OAuthHandlers { providerInfo: { idToken: result.params.id_token, accessToken: result.accessToken, - refreshToken: result.refreshToken, // GitLab expires the old refresh token when used + refreshToken: result.refreshToken, scope: result.params.scope, expiresInSeconds: result.params.expires_in, }, @@ -229,7 +229,7 @@ export const createAtlassianProvider = ( OAuthEnvironmentHandler.mapConfig(config, envConfig => { const clientId = envConfig.getString('clientId'); const clientSecret = envConfig.getString('clientSecret'); - const scopes = envConfig.getStringArray('scopes'); + const scopes = envConfig.getString('scopes'); const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; const catalogIdentityClient = new CatalogIdentityClient({ @@ -253,7 +253,7 @@ export const createAtlassianProvider = ( const provider = new AtlassianAuthProvider({ clientId, clientSecret, - scopes, + scopes: [scopes], callbackUrl, authHandler, signInResolver, diff --git a/plugins/auth-backend/src/providers/atlassian/strategy.ts b/plugins/auth-backend/src/providers/atlassian/strategy.ts index 630348b16d..d07b09c5f8 100644 --- a/plugins/auth-backend/src/providers/atlassian/strategy.ts +++ b/plugins/auth-backend/src/providers/atlassian/strategy.ts @@ -29,7 +29,7 @@ interface AtlassianStrategyOptions { clientID: string; clientSecret: string; callbackURL: string; - scope: string[]; + scope: string; } const defaultScopes = ['offline_access', 'read:me']; @@ -45,11 +45,13 @@ export default class AtlassianStrategy extends OAuth2Strategy { throw new TypeError('Atlassian requires a scope option'); } + const scopes = options.scope.split(' '); + const optionsWithURLs = { ...options, authorizationURL: `https://auth.atlassian.com/authorize`, tokenURL: `https://auth.atlassian.com/oauth/token`, - scope: Array.from(new Set([...defaultScopes, ...options.scope])), + scope: Array.from(new Set([...defaultScopes, ...scopes])), }; super(optionsWithURLs, verify); From 202f3229277f51adfab919db94cdf1bc1351f40f Mon Sep 17 00:00:00 2001 From: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> Date: Mon, 11 Oct 2021 17:06:50 -0400 Subject: [PATCH 009/103] chore: adds changeset Signed-off-by: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> --- .changeset/swift-pugs-serve.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .changeset/swift-pugs-serve.md diff --git a/.changeset/swift-pugs-serve.md b/.changeset/swift-pugs-serve.md new file mode 100644 index 0000000000..e6f1945afb --- /dev/null +++ b/.changeset/swift-pugs-serve.md @@ -0,0 +1,13 @@ +--- +'example-app': patch +'@backstage/core-app-api': patch +'@backstage/core-plugin-api': patch +'@backstage/plugin-auth-backend': patch +'@backstage/plugin-user-settings': patch +--- + +Atlassian Cloud authentication + +- AtlassianAuth added to core-app-api +- Atlassian provider added to plugin-auth-backend +- Updated user-settings with Atlassian connection From 044a4511eb1dd1c7dc646bb0dd6650c62d6bb812 Mon Sep 17 00:00:00 2001 From: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> Date: Tue, 12 Oct 2021 11:30:02 -0400 Subject: [PATCH 010/103] adds docs, fixes plugin Signed-off-by: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> --- .changeset/swift-pugs-serve.md | 3 +- docs/auth/atlassian/provider.md | 64 +++++++++++++++++++ .../src/providers/atlassian/provider.ts | 4 +- 3 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 docs/auth/atlassian/provider.md diff --git a/.changeset/swift-pugs-serve.md b/.changeset/swift-pugs-serve.md index e6f1945afb..8262a66d7f 100644 --- a/.changeset/swift-pugs-serve.md +++ b/.changeset/swift-pugs-serve.md @@ -1,12 +1,11 @@ --- -'example-app': patch '@backstage/core-app-api': patch '@backstage/core-plugin-api': patch '@backstage/plugin-auth-backend': patch '@backstage/plugin-user-settings': patch --- -Atlassian Cloud authentication +Atlassian auth provider - AtlassianAuth added to core-app-api - Atlassian provider added to plugin-auth-backend diff --git a/docs/auth/atlassian/provider.md b/docs/auth/atlassian/provider.md new file mode 100644 index 0000000000..38774ca3cc --- /dev/null +++ b/docs/auth/atlassian/provider.md @@ -0,0 +1,64 @@ +--- +id: provider +title: Atlassian Authentication Provider +sidebar_label: Atlassian +description: Adding Atlassian as an authentication provider in Backstage +--- + +The Backstage `core-plugin-api` package comes with an Atlassian authentication +provider that can authenticate users using Atlassian products. This auth +**only** provides scopes for the following APIs: + +- Confluence API +- User REST API +- Jira platform REST API +- Jira Service Desk API +- Personal data reporting API +- User identity API + +## Create an OAuth 2.0 (3LO) app in the Atlassian developer console + +To add Atlassian authentication, you must create an OAuth 2.0 (3LO) app. + +Go to `https://developer.atlassian.com/console/myapps/`. + +Click on the drop down `Create`, and choose `OAuth 2.0 integration`. + +Name your integration and click on the `Create` button. + +Settings for local development: + +- Callback URL: `http://localhost:7000/api/auth/atlassian` +- Use rotating refresh tokens +- For permissions, you **must** enable `View user profile` for the currently + logged-in user, under `User identity API` + +## Configuration + +The provider configuration can then be added to your `app-config.yaml` under the +root `auth` configuration: + +```yaml +auth: + environment: development + providers: + atlassian: + development: + clientId: ${AUTH_ATLASSIAN_CLIENT_ID} + clientSecret: ${AUTH_ATLASSIAN_CLIENT_SECRET} + scopes: ${AUTH_ATLASSIAN_SCOPES} +``` + +The Atlassian provider is a structure with three configuration keys: + +- `clientId`: The Key you generated in the developer console. +- `clientSecret`: The Secret tied to the generated Key. +- `scopes`: List of scopes the app has permissions for, separated by spaces. + +**NOTE:** the scopes `offline_access` and `read:me` are provided by default. + +## Adding the provider to the Backstage frontend + +To add the provider to the frontend, add the `atlassianAuthApi` reference and +`SignInPage` component as shown in +[Adding the provider to the sign-in page](../index.md#adding-the-provider-to-the-sign-in-page). diff --git a/plugins/auth-backend/src/providers/atlassian/provider.ts b/plugins/auth-backend/src/providers/atlassian/provider.ts index c7d1a9590c..4c031cedc9 100644 --- a/plugins/auth-backend/src/providers/atlassian/provider.ts +++ b/plugins/auth-backend/src/providers/atlassian/provider.ts @@ -47,7 +47,7 @@ import { CatalogIdentityClient } from '../../lib/catalog'; import { Logger } from 'winston'; export type AtlassianAuthProviderOptions = OAuthProviderOptions & { - scopes: string[]; + scopes: string; signInResolver?: SignInResolver; authHandler: AuthHandler; tokenIssuer: TokenIssuer; @@ -253,7 +253,7 @@ export const createAtlassianProvider = ( const provider = new AtlassianAuthProvider({ clientId, clientSecret, - scopes: [scopes], + scopes, callbackUrl, authHandler, signInResolver, From 1f58e63bcc089788c9123f38ee144a909ecb610f Mon Sep 17 00:00:00 2001 From: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> Date: Tue, 12 Oct 2021 11:35:31 -0400 Subject: [PATCH 011/103] updates test Signed-off-by: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> --- plugins/auth-backend/src/providers/atlassian/provider.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/auth-backend/src/providers/atlassian/provider.test.ts b/plugins/auth-backend/src/providers/atlassian/provider.test.ts index 58c7468246..86ce21265e 100644 --- a/plugins/auth-backend/src/providers/atlassian/provider.test.ts +++ b/plugins/auth-backend/src/providers/atlassian/provider.test.ts @@ -54,7 +54,7 @@ describe('createAtlassianProvider', () => { clientId: 'mock', clientSecret: 'mock', callbackUrl: 'mock', - scopes: [], + scopes: 'scope', signInResolver: atlassianDefaultSignInResolver, }); From e2039ef99a0391bf01b9fd5be9fdccc1a04a9fa5 Mon Sep 17 00:00:00 2001 From: Alex Krantz Date: Tue, 12 Oct 2021 10:28:42 -0700 Subject: [PATCH 012/103] fix: add comment for clarification and require full scheme Signed-off-by: Alex Krantz --- plugins/auth-backend/src/providers/okta/provider.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/auth-backend/src/providers/okta/provider.ts b/plugins/auth-backend/src/providers/okta/provider.ts index 7ea5b29e80..dfffca9da7 100644 --- a/plugins/auth-backend/src/providers/okta/provider.ts +++ b/plugins/auth-backend/src/providers/okta/provider.ts @@ -275,8 +275,11 @@ export const createOktaProvider = ( const audience = envConfig.getString('audience'); const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; - if (!audience.startsWith('https')) { - throw new Error("URL for 'audience' must start with 'https'."); + // This is a safe assumption as `passport-okta-oauth` uses the audience + // as the base for building the authorization, token, and user info URLs. + // https://github.com/fischerdan/passport-okta-oauth/blob/ea9ac42d/lib/passport-okta-oauth/oauth2.js#L12-L14 + if (!audience.startsWith('https://')) { + throw new Error("URL for 'audience' must start with 'https://'."); } const catalogIdentityClient = new CatalogIdentityClient({ From 93f6d19f7fd5db370294668f2cd86dc6fea0ae27 Mon Sep 17 00:00:00 2001 From: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> Date: Wed, 13 Oct 2021 08:23:29 -0400 Subject: [PATCH 013/103] updates api reports Signed-off-by: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> --- packages/core-app-api/api-report.md | 18 +++++++++++++++-- packages/core-plugin-api/api-report.md | 7 +++++++ plugins/auth-backend/api-report.md | 28 +++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/packages/core-app-api/api-report.md b/packages/core-app-api/api-report.md index 22d5092aa6..869f9a6eb0 100644 --- a/packages/core-app-api/api-report.md +++ b/packages/core-app-api/api-report.md @@ -15,6 +15,7 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { AppConfig } from '@backstage/config'; import { AppTheme } from '@backstage/core-plugin-api'; import { AppThemeApi } from '@backstage/core-plugin-api'; +import { atlassianAuthApiRef } from '@backstage/core-plugin-api'; import { auth0AuthApiRef } from '@backstage/core-plugin-api'; import { AuthProvider } from '@backstage/core-plugin-api'; import { AuthRequester } from '@backstage/core-plugin-api'; @@ -236,12 +237,25 @@ export class AppThemeSelector implements AppThemeApi { setActiveThemeId(themeId?: string): void; } +// Warning: (ae-missing-release-tag) "AtlassianAuth" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class AtlassianAuth { + // Warning: (ae-forgotten-export) The symbol "OAuthApiCreateOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static create({ + discoveryApi, + environment, + provider, + oauthRequestApi, + }: OAuthApiCreateOptions): typeof atlassianAuthApiRef.T; +} + // Warning: (ae-missing-release-tag) "Auth0Auth" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) export class Auth0Auth { - // Warning: (ae-forgotten-export) The symbol "OAuthApiCreateOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) static create({ discoveryApi, diff --git a/packages/core-plugin-api/api-report.md b/packages/core-plugin-api/api-report.md index 4d1fe4fffd..808131b026 100644 --- a/packages/core-plugin-api/api-report.md +++ b/packages/core-plugin-api/api-report.md @@ -211,6 +211,13 @@ export type AppThemeApi = { // @public (undocumented) export const appThemeApiRef: ApiRef; +// Warning: (ae-missing-release-tag) "atlassianAuthApiRef" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export const atlassianAuthApiRef: ApiRef< + OAuthApi & ProfileInfoApi & BackstageIdentityApi & SessionApi +>; + // Warning: (ae-missing-release-tag) "attachComponentData" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/plugins/auth-backend/api-report.md b/plugins/auth-backend/api-report.md index 0bdd73f3e8..ed0647ecda 100644 --- a/plugins/auth-backend/api-report.md +++ b/plugins/auth-backend/api-report.md @@ -16,6 +16,25 @@ import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { Profile } from 'passport'; import { UserEntity } from '@backstage/catalog-model'; +// Warning: (ae-missing-release-tag) "AtlassianAuthProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class AtlassianAuthProvider implements OAuthHandlers { + // Warning: (ae-forgotten-export) The symbol "AtlassianAuthProviderOptions" needs to be exported by the entry point index.d.ts + constructor(options: AtlassianAuthProviderOptions); + // (undocumented) + handler(req: express.Request): Promise<{ + response: OAuthResponse; + refreshToken: string; + }>; + // (undocumented) + refresh(req: OAuthRefreshRequest): Promise; + // Warning: (ae-forgotten-export) The symbol "RedirectInfo" needs to be exported by the entry point index.d.ts + // + // (undocumented) + start(req: OAuthStartRequest): Promise; +} + // Warning: (ae-missing-release-tag) "AuthProviderFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -146,6 +165,14 @@ export const bitbucketUserIdSignInResolver: SignInResolver // @public (undocumented) export const bitbucketUsernameSignInResolver: SignInResolver; +// Warning: (ae-forgotten-export) The symbol "AtlassianProviderOptions" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "createAtlassianProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export const createAtlassianProvider: ( + options?: AtlassianProviderOptions | undefined, +) => AuthProviderFactory; + // Warning: (ae-missing-release-tag) "createAwsAlbProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -388,7 +415,6 @@ export interface OAuthHandlers { // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (tsdoc-param-tag-with-invalid-type) The @param block should not include a JSDoc-style '{type}' // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (ae-forgotten-export) The symbol "RedirectInfo" needs to be exported by the entry point index.d.ts start(req: OAuthStartRequest): Promise; } From 6c03f1b1378510b9de5d6f6a22fd318d1346c4b0 Mon Sep 17 00:00:00 2001 From: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> Date: Wed, 13 Oct 2021 08:57:27 -0400 Subject: [PATCH 014/103] adds sidebar link Signed-off-by: Daniel Deloff <44780793+rv-ddeloff@users.noreply.github.com> --- microsite/sidebars.json | 1 + 1 file changed, 1 insertion(+) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index c269f8ac8f..541792eca6 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -214,6 +214,7 @@ "label": "Included providers", "ids": [ "auth/auth0/provider", + "auth/atlassian/provider", "auth/bitbucket/provider", "auth/microsoft/provider", "auth/github/provider", From 26b7943660c11b4cd5724e9a4434f1142986ba2b Mon Sep 17 00:00:00 2001 From: Aldira Putra Raharja Date: Wed, 13 Oct 2021 20:16:55 +0700 Subject: [PATCH 015/103] update api-report and update docs around caData Signed-off-by: Aldira Putra Raharja --- docs/features/kubernetes/configuration.md | 14 ++++++++++++++ plugins/kubernetes-backend/api-report.md | 2 ++ 2 files changed, 16 insertions(+) diff --git a/docs/features/kubernetes/configuration.md b/docs/features/kubernetes/configuration.md index fc32f4da70..1be1b3471a 100644 --- a/docs/features/kubernetes/configuration.md +++ b/docs/features/kubernetes/configuration.md @@ -140,6 +140,20 @@ for real examples. PEM-encoded certificate authority certificates. +This values could be obtained via inspecting the kube config file under +`clusters.cluster.certificate-authority-data`. For GKE, execute the following +command to obtain the value + +``` +gcloud container clusters describe \ + --zone= \ + --format="value(masterAuth.clusterCaCertificate)" +``` + +See also +https://cloud.google.com/kubernetes-engine/docs/how-to/api-server-authentication#environments-without-gcloud +for complete docs about GKE without gcloud. + #### `gke` This cluster locator is designed to work with Kubernetes clusters running in diff --git a/plugins/kubernetes-backend/api-report.md b/plugins/kubernetes-backend/api-report.md index 3d52c4ff43..34ab7d4db1 100644 --- a/plugins/kubernetes-backend/api-report.md +++ b/plugins/kubernetes-backend/api-report.md @@ -27,6 +27,8 @@ export interface AWSClusterDetails extends ClusterDetails { export interface ClusterDetails { // (undocumented) authProvider: string; + // (undocumented) + caData?: string | undefined; dashboardApp?: string; dashboardUrl?: string; name: string; From 406dcf06e5fe3ce5d2c4abd74c6a7359605f55f3 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 14 Oct 2021 12:31:12 +0200 Subject: [PATCH 016/103] Add `MicrosoftGraphOrgEntityProvider` Signed-off-by: Oliver Sand --- .changeset/wise-hats-remember.md | 5 + .../MicrosoftGraphOrgEntityProvider.ts | 187 ++++++++++++++++++ .../src/processors/index.ts | 1 + 3 files changed, 193 insertions(+) create mode 100644 .changeset/wise-hats-remember.md create mode 100644 plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts diff --git a/.changeset/wise-hats-remember.md b/.changeset/wise-hats-remember.md new file mode 100644 index 0000000000..6666502bee --- /dev/null +++ b/.changeset/wise-hats-remember.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +Add `MicrosoftGraphOrgEntityProvider` as an alternative to `MicrosoftGraphOrgReaderProcessor` that automatically handles user and group deletions. diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts new file mode 100644 index 0000000000..47f9167fbc --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -0,0 +1,187 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Entity, + LOCATION_ANNOTATION, + ORIGIN_LOCATION_ANNOTATION, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; +import { merge } from 'lodash'; +import { Logger } from 'winston'; +import { + GroupTransformer, + MicrosoftGraphClient, + MicrosoftGraphProviderConfig, + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, + MICROSOFT_GRAPH_TENANT_ID_ANNOTATION, + MICROSOFT_GRAPH_USER_ID_ANNOTATION, + OrganizationTransformer, + readMicrosoftGraphConfig, + readMicrosoftGraphOrg, + UserTransformer, +} from '../microsoftGraph'; + +/** + * Reads user and group entries out of Microsoft Graph, and provides them as + * User and Group entities for the catalog. + */ +export class MicrosoftGraphOrgEntityProvider implements EntityProvider { + private connection?: EntityProviderConnection; + + static fromConfig( + config: Config, + options: { + id: string; + target: string; + logger: Logger; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; + }, + ) { + const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg'); + const providers = c ? readMicrosoftGraphConfig(c) : []; + const provider = providers.find(p => options.target.startsWith(p.target)); + + if (!provider) { + throw new Error( + `There is no Microsoft Graph Org provider that matches ${options.target}. Please add a configuration entry for it under catalog.processors.microsoftGraphOrg.providers.`, + ); + } + + if (provider.userFilter && provider.userGroupMemberFilter) { + throw new Error( + `userFilter and userGroupMemberFilter are mutually exclusive, only one can be specified.`, + ); + } + + const logger = options.logger.child({ + target: options.target, + }); + + return new MicrosoftGraphOrgEntityProvider({ + id: options.id, + userTransformer: options.userTransformer, + groupTransformer: options.groupTransformer, + organizationTransformer: options.organizationTransformer, + logger, + provider, + }); + } + + constructor( + private options: { + id: string; + provider: MicrosoftGraphProviderConfig; + logger: Logger; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; + }, + ) {} + + getProviderName() { + return `MicrosoftGraphOrgEntityProvider:${this.options.id}`; + } + + async connect(connection: EntityProviderConnection) { + this.connection = connection; + } + + async read() { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const provider = this.options.provider; + const { markReadComplete } = trackProgress(this.options.logger); + const client = MicrosoftGraphClient.create(this.options.provider); + + const { users, groups } = await readMicrosoftGraphOrg( + client, + provider.tenantId, + { + userFilter: provider.userFilter, + userGroupMemberFilter: provider.userGroupMemberFilter, + groupFilter: provider.groupFilter, + groupTransformer: this.options.groupTransformer, + userTransformer: this.options.userTransformer, + organizationTransformer: this.options.organizationTransformer, + logger: this.options.logger, + }, + ); + + const { markCommitComplete } = markReadComplete({ users, groups }); + + await this.connection.applyMutation({ + type: 'full', + entities: [...users, ...groups].map(entity => ({ + locationKey: `msgraph-org-provider:${this.options.id}`, + entity: withLocations(this.options.id, entity), + })), + }); + + markCommitComplete(); + } +} + +// Helps wrap the timing and logging behaviors +function trackProgress(logger: Logger) { + let timestamp = Date.now(); + let summary: string; + + logger.info('Reading msgraph users and groups'); + + function markReadComplete(read: { users: unknown[]; groups: unknown[] }) { + summary = `${read.users.length} msgraph users and ${read.groups.length} msgraph groups`; + const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + timestamp = Date.now(); + logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`); + return { markCommitComplete }; + } + + function markCommitComplete() { + const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + logger.info(`Committed ${summary} in ${commitDuration} seconds.`); + } + + return { markReadComplete }; +} + +// Makes sure that emitted entities have a proper location based on their uuid +function withLocations(providerId: string, entity: Entity): Entity { + const dn = + entity.metadata.annotations?.[MICROSOFT_GRAPH_USER_ID_ANNOTATION] || + entity.metadata.annotations?.[MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] || + entity.metadata.annotations?.[MICROSOFT_GRAPH_TENANT_ID_ANNOTATION] || + entity.metadata.name; + const location = `msgraph:${providerId}/${encodeURIComponent(dn)}`; + return merge( + { + metadata: { + annotations: { + [LOCATION_ANNOTATION]: location, + [ORIGIN_LOCATION_ANNOTATION]: location, + }, + }, + }, + entity, + ) as Entity; +} diff --git a/plugins/catalog-backend-module-msgraph/src/processors/index.ts b/plugins/catalog-backend-module-msgraph/src/processors/index.ts index f5dc101563..7f70bde994 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/index.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/index.ts @@ -14,4 +14,5 @@ * limitations under the License. */ +export { MicrosoftGraphOrgEntityProvider } from './MicrosoftGraphOrgEntityProvider'; export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor'; From 4630d393813af22884d0c48a29ebb7ae63ed4f3b Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 14 Oct 2021 12:56:54 +0200 Subject: [PATCH 017/103] Add docs Signed-off-by: Oliver Sand --- .../catalog-backend-module-msgraph/README.md | 92 +++++++++++++------ .../api-report.md | 34 +++++++ 2 files changed, 97 insertions(+), 29 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index 4c41d42915..b89f3af460 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -1,41 +1,24 @@ # Catalog Backend Module for Microsoft Graph This is an extension module to the `plugin-catalog-backend` plugin, providing a -`MicrosoftGraphOrgReaderProcessor` that can be used to ingest organization data -from the Microsoft Graph API. This processor is useful if you want to import -users and groups from Azure Active Directory or Office 365. +`MicrosoftGraphOrgReaderProcessor` and a `MicrosoftGraphOrgEntityProvider` that +can be used to ingest organization data from the Microsoft Graph API. This +processor is useful if you want to import users and groups from Azure Active +Directory or Office 365. ## Getting Started -1. The processor is not installed by default, therefore you have to add a - dependency to `@backstage/plugin-catalog-backend-module-msgraph` to your - backend package. +First you need to decide whether you want to use an [entity provider or a processor](https://backstage.io/docs/features/software-catalog/life-of-an-entity#stitching) to ingest the organization data. +If you want groups and users deleted from the source to be automatically deleted +from Backstage, choose the entity provider. -```bash -# From your Backstage root directory -cd packages/backend -yarn add @backstage/plugin-catalog-backend-module-msgraph -``` - -2. The `MicrosoftGraphOrgReaderProcessor` is not registered by default, so you - have to register it in the catalog plugin: - -```typescript -// packages/backend/src/plugins/catalog.ts -builder.addProcessor( - MicrosoftGraphOrgReaderProcessor.fromConfig(config, { - logger, - }), -); -``` - -3. Create or use an existing App registration in the [Microsoft Azure Portal](https://portal.azure.com/). +1. Create or use an existing App registration in the [Microsoft Azure Portal](https://portal.azure.com/). The App registration requires at least the API permissions `Group.Read.All`, `GroupMember.Read.All`, `User.Read` and `User.Read.All` for Microsoft Graph (if you still run into errors about insufficient privileges, add `Team.ReadBasic.All` and `TeamMember.Read.All` too). -4. Configure the processor: +2. Configure the processor or entity provider: ```yaml # app-config.yaml @@ -69,6 +52,56 @@ catalog: By default, all users are loaded. If you want to filter users based on their attributes, use `userFilter`. `userGroupMemberFilter` can be used if you want to load users based on their group membership. +3. The package is not installed by default, therefore you have to add a + dependency to `@backstage/plugin-catalog-backend-module-msgraph` to your + backend package. + +```bash +# From your Backstage root directory +cd packages/backend +yarn add @backstage/plugin-catalog-backend-module-msgraph +``` + +### Using the Entity Provider + +4. The `MicrosoftGraphOrgEntityProvider` is not registered by default, so you + have to register it in the catalog plugin. Pass the target to reference a + provider from the configuration. As entity providers are not part of the + entity refresh loop, you have to run them manually. + +```typescript +// packages/backend/src/plugins/catalog.ts +const msGraphOrgEntityProvider = MicrosoftGraphOrgEntityProvider.fromConfig( + env.config, + { + id: 'https://graph.microsoft.com/v1.0', + target: 'https://graph.microsoft.com/v1.0', + logger: env.logger, + }, +); +builder.addEntityProvider(msGraphOrgEntityProvider); + +// Trigger a read every 5 minutes +useHotCleanup( + module, + runPeriodically(async () => msGraphOrgEntityProvider.read(), 5 * 60 * 1000), +); +``` + +### Using the Processor + +4. The `MicrosoftGraphOrgReaderProcessor` is not registered by default, so you + have to register it in the catalog plugin: + +```typescript +// packages/backend/src/plugins/catalog.ts +builder.addProcessor( + MicrosoftGraphOrgReaderProcessor.fromConfig(config, { + logger, + }), +); +``` + 5. Add a location that ingests from Microsoft Graph: ```yaml @@ -84,10 +117,11 @@ catalog: … ``` -## Customize the Processor +## Customize the Processor or Entity Provider -In case you want to customize the ingested entities, the `MicrosoftGraphOrgReaderProcessor` -allows to pass transformers for users, groups and the organization. +In case you want to customize the ingested entities, both the `MicrosoftGraphOrgReaderProcessor` +and the `MicrosoftGraphOrgEntityProvider` allows to pass transformers for users, +groups and the organization. 1. Create a transformer: diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index d290fcf9b6..1357659739 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -6,6 +6,8 @@ import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; +import { EntityProvider } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { GroupEntity } from '@backstage/catalog-model'; import { LocationSpec } from '@backstage/catalog-model'; import { Logger as Logger_2 } from 'winston'; @@ -93,6 +95,38 @@ export class MicrosoftGraphClient { requestRaw(url: string): Promise; } +// Warning: (ae-missing-release-tag) "MicrosoftGraphOrgEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export class MicrosoftGraphOrgEntityProvider implements EntityProvider { + constructor(options: { + id: string; + provider: MicrosoftGraphProviderConfig; + logger: Logger_2; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; + }); + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + config: Config, + options: { + id: string; + target: string; + logger: Logger_2; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; + }, + ): MicrosoftGraphOrgEntityProvider; + // (undocumented) + getProviderName(): string; + // (undocumented) + read(): Promise; +} + // Warning: (ae-missing-release-tag) "MicrosoftGraphOrgReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public From ab2df3be333952daf9c91085f976597f13281733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarek=20=C5=81ukow?= Date: Wed, 13 Oct 2021 19:09:24 +0200 Subject: [PATCH 018/103] Improve API docs in @backstage/catalog-model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jarek Łukow --- .changeset/two-ties-eat.md | 5 ++ packages/catalog-model/api-report.md | 90 +++++++++---------- packages/catalog-model/src/EntityPolicies.ts | 6 +- packages/catalog-model/src/entity/Entity.ts | 13 ++- .../catalog-model/src/entity/constants.ts | 9 +- .../policies/FieldFormatEntityPolicy.ts | 2 + .../policies/SchemaValidEntityPolicy.ts | 2 + packages/catalog-model/src/entity/ref.ts | 33 +++++-- packages/catalog-model/src/entity/util.ts | 4 + .../src/kinds/ApiEntityV1alpha1.ts | 16 +++- .../src/kinds/ComponentEntityV1alpha1.ts | 16 +++- .../src/kinds/DomainEntityV1alpha1.ts | 16 +++- .../src/kinds/GroupEntityV1alpha1.ts | 11 ++- .../src/kinds/LocationEntityV1alpha1.ts | 12 ++- .../src/kinds/ResourceEntityV1alpha1.ts | 16 +++- .../src/kinds/SystemEntityV1alpha1.ts | 16 +++- .../src/kinds/TemplateEntityV1beta2.ts | 12 ++- .../src/kinds/UserEntityV1alpha1.ts | 12 ++- packages/catalog-model/src/kinds/relations.ts | 82 ++++++++++++++--- .../catalog-model/src/location/annotation.ts | 15 +++- .../catalog-model/src/location/helpers.ts | 4 + packages/catalog-model/src/location/types.ts | 21 +++-- .../catalog-model/src/location/validation.ts | 12 ++- packages/catalog-model/src/types.ts | 10 ++- .../entityEnvelopeSchemaValidator.ts | 14 +-- .../validation/entityKindSchemaValidator.ts | 15 ++-- .../src/validation/entitySchemaValidator.ts | 14 +-- .../src/validation/makeValidator.ts | 6 +- .../catalog-model/src/validation/types.ts | 6 +- 29 files changed, 366 insertions(+), 124 deletions(-) create mode 100644 .changeset/two-ties-eat.md diff --git a/.changeset/two-ties-eat.md b/.changeset/two-ties-eat.md new file mode 100644 index 0000000000..69edfb16c8 --- /dev/null +++ b/.changeset/two-ties-eat.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-model': patch +--- + +Improved documentation for exported symbols. diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 1e7eb8fd52..c9429f9bdb 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -9,12 +9,12 @@ import { JsonValue } from '@backstage/config'; import { SerializedError } from '@backstage/errors'; import * as yup from 'yup'; -// @public @deprecated (undocumented) +// @public @deprecated export const analyzeLocationSchema: yup.SchemaOf<{ location: LocationSpec; }>; -// @public (undocumented) +// @public interface ApiEntityV1alpha1 extends Entity { // (undocumented) apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; @@ -32,7 +32,7 @@ interface ApiEntityV1alpha1 extends Entity { export { ApiEntityV1alpha1 as ApiEntity }; export { ApiEntityV1alpha1 }; -// @public (undocumented) +// @public export const apiEntityV1alpha1Validator: KindValidator; // @public @@ -58,7 +58,7 @@ export function compareEntityToRef( context?: EntityRefContext, ): boolean; -// @public (undocumented) +// @public interface ComponentEntityV1alpha1 extends Entity { // (undocumented) apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; @@ -79,7 +79,7 @@ interface ComponentEntityV1alpha1 extends Entity { export { ComponentEntityV1alpha1 as ComponentEntity }; export { ComponentEntityV1alpha1 }; -// @public (undocumented) +// @public export const componentEntityV1alpha1Validator: KindValidator; // @public @@ -89,7 +89,7 @@ export class DefaultNamespaceEntityPolicy implements EntityPolicy { enforce(entity: Entity): Promise; } -// @public (undocumented) +// @public interface DomainEntityV1alpha1 extends Entity { // (undocumented) apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; @@ -103,10 +103,10 @@ interface DomainEntityV1alpha1 extends Entity { export { DomainEntityV1alpha1 as DomainEntity }; export { DomainEntityV1alpha1 }; -// @public (undocumented) +// @public export const domainEntityV1alpha1Validator: KindValidator; -// @public (undocumented) +// @public export const EDIT_URL_ANNOTATION = 'backstage.io/edit-url'; // @public @@ -181,7 +181,7 @@ export type EntityName = { name: string; }; -// @public (undocumented) +// @public export const EntityPolicies: { allOf(policies: EntityPolicy[]): EntityPolicy; oneOf(policies: EntityPolicy[]): EntityPolicy; @@ -250,7 +250,7 @@ export function getEntitySourceLocation(entity: Entity): { target: string; }; -// @public (undocumented) +// @public interface GroupEntityV1alpha1 extends Entity { // (undocumented) apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; @@ -272,10 +272,10 @@ interface GroupEntityV1alpha1 extends Entity { export { GroupEntityV1alpha1 as GroupEntity }; export { GroupEntityV1alpha1 }; -// @public (undocumented) +// @public export const groupEntityV1alpha1Validator: KindValidator; -// @public (undocumented) +// @public export type JSONSchema = JSONSchema7 & { [key in string]?: JsonValue; }; @@ -305,16 +305,16 @@ export class KubernetesValidatorFunctions { static isValidObjectName(value: unknown): boolean; } -// @public (undocumented) +// @public type Location_2 = { id: string; } & LocationSpec; export { Location_2 as Location }; -// @public (undocumented) +// @public export const LOCATION_ANNOTATION = 'backstage.io/managed-by-location'; -// @public (undocumented) +// @public interface LocationEntityV1alpha1 extends Entity { // (undocumented) apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; @@ -330,23 +330,23 @@ interface LocationEntityV1alpha1 extends Entity { export { LocationEntityV1alpha1 as LocationEntity }; export { LocationEntityV1alpha1 }; -// @public (undocumented) +// @public export const locationEntityV1alpha1Validator: KindValidator; -// @public @deprecated (undocumented) +// @public @deprecated export const locationSchema: yup.SchemaOf; -// @public (undocumented) +// @public export type LocationSpec = { type: string; target: string; presence?: 'optional' | 'required'; }; -// @public @deprecated (undocumented) +// @public @deprecated export const locationSpecSchema: yup.SchemaOf; -// @public (undocumented) +// @public export function makeValidator(overrides?: Partial): Validators; // @public @@ -356,7 +356,7 @@ export class NoForeignRootFieldsEntityPolicy implements EntityPolicy { enforce(entity: Entity): Promise; } -// @public (undocumented) +// @public export const ORIGIN_LOCATION_ANNOTATION = 'backstage.io/managed-by-origin-location'; @@ -373,13 +373,9 @@ export function parseEntityRef( defaultKind: string; defaultNamespace: string; }, -): { - kind: string; - namespace: string; - name: string; -}; +): EntityName; -// @public (undocumented) +// @public export function parseEntityRef( ref: EntityRef, context?: { @@ -391,7 +387,7 @@ export function parseEntityRef( name: string; }; -// @public (undocumented) +// @public export function parseEntityRef( ref: EntityRef, context?: { @@ -409,28 +405,28 @@ export function parseLocationReference(ref: string): { target: string; }; -// @public (undocumented) +// @public export const RELATION_API_CONSUMED_BY = 'apiConsumedBy'; -// @public (undocumented) +// @public export const RELATION_API_PROVIDED_BY = 'apiProvidedBy'; -// @public (undocumented) +// @public export const RELATION_CHILD_OF = 'childOf'; // @public export const RELATION_CONSUMES_API = 'consumesApi'; -// @public (undocumented) +// @public export const RELATION_DEPENDENCY_OF = 'dependencyOf'; // @public export const RELATION_DEPENDS_ON = 'dependsOn'; -// @public (undocumented) +// @public export const RELATION_HAS_MEMBER = 'hasMember'; -// @public (undocumented) +// @public export const RELATION_HAS_PART = 'hasPart'; // @public @@ -439,7 +435,7 @@ export const RELATION_MEMBER_OF = 'memberOf'; // @public export const RELATION_OWNED_BY = 'ownedBy'; -// @public (undocumented) +// @public export const RELATION_OWNER_OF = 'ownerOf'; // @public @@ -448,10 +444,10 @@ export const RELATION_PARENT_OF = 'parentOf'; // @public export const RELATION_PART_OF = 'partOf'; -// @public (undocumented) +// @public export const RELATION_PROVIDES_API = 'providesApi'; -// @public (undocumented) +// @public interface ResourceEntityV1alpha1 extends Entity { // (undocumented) apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; @@ -468,7 +464,7 @@ interface ResourceEntityV1alpha1 extends Entity { export { ResourceEntityV1alpha1 as ResourceEntity }; export { ResourceEntityV1alpha1 }; -// @public (undocumented) +// @public export const resourceEntityV1alpha1Validator: KindValidator; // @public @@ -488,7 +484,7 @@ export function serializeEntityRef( }, ): EntityRef; -// @public (undocumented) +// @public export const SOURCE_LOCATION_ANNOTATION = 'backstage.io/source-location'; // @public @@ -508,7 +504,7 @@ export function stringifyLocationReference(ref: { target: string; }): string; -// @public (undocumented) +// @public interface SystemEntityV1alpha1 extends Entity { // (undocumented) apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; @@ -523,10 +519,10 @@ interface SystemEntityV1alpha1 extends Entity { export { SystemEntityV1alpha1 as SystemEntity }; export { SystemEntityV1alpha1 }; -// @public (undocumented) +// @public export const systemEntityV1alpha1Validator: KindValidator; -// @public (undocumented) +// @public export interface TemplateEntityV1beta2 extends Entity { // (undocumented) apiVersion: 'backstage.io/v1beta2'; @@ -550,7 +546,7 @@ export interface TemplateEntityV1beta2 extends Entity { }; } -// @public (undocumented) +// @public export const templateEntityV1beta2Validator: KindValidator; // @alpha @@ -569,7 +565,7 @@ export type UNSTABLE_EntityStatusItem = { // @alpha export type UNSTABLE_EntityStatusLevel = 'info' | 'warning' | 'error'; -// @public (undocumented) +// @public interface UserEntityV1alpha1 extends Entity { // (undocumented) apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; @@ -588,10 +584,10 @@ interface UserEntityV1alpha1 extends Entity { export { UserEntityV1alpha1 as UserEntity }; export { UserEntityV1alpha1 }; -// @public (undocumented) +// @public export const userEntityV1alpha1Validator: KindValidator; -// @public (undocumented) +// @public export type Validators = { isValidApiVersion(value: unknown): boolean; isValidKind(value: unknown): boolean; @@ -609,5 +605,5 @@ export const VIEW_URL_ANNOTATION = 'backstage.io/view-url'; // Warnings were encountered during analysis: // -// src/entity/Entity.d.ts:38:5 - (ae-incompatible-release-tags) The symbol "status" is marked as @public, but its signature references "UNSTABLE_EntityStatus" which is marked as @alpha +// src/entity/Entity.d.ts:41:5 - (ae-incompatible-release-tags) The symbol "status" is marked as @public, but its signature references "UNSTABLE_EntityStatus" which is marked as @alpha ``` diff --git a/packages/catalog-model/src/EntityPolicies.ts b/packages/catalog-model/src/EntityPolicies.ts index 0fff68d569..e5aaeb0c20 100644 --- a/packages/catalog-model/src/EntityPolicies.ts +++ b/packages/catalog-model/src/EntityPolicies.ts @@ -52,7 +52,11 @@ class AnyEntityPolicy implements EntityPolicy { } } -/** @public */ +/** + * Provides helpers for enforcing a set of {@link EntityPolicy} in an `and`/`or` expression. + * + * @public + */ export const EntityPolicies = { allOf(policies: EntityPolicy[]): EntityPolicy { return new AllEntityPolicies(policies); diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts index ca38fef183..7a4729c309 100644 --- a/packages/catalog-model/src/entity/Entity.ts +++ b/packages/catalog-model/src/entity/Entity.ts @@ -21,8 +21,11 @@ import { UNSTABLE_EntityStatus } from './EntityStatus'; /** * The parts of the format that's common to all versions/kinds of entity. * + * @remarks + * + * See also: + * {@link https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/} * @public - * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ */ export type Entity = { /** @@ -63,9 +66,13 @@ export type Entity = { /** * Metadata fields common to all versions/kinds of entity. * + * @remarks + * + * See also: + * {@link https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta} + * {@link https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/} + * * @public - * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ */ export type EntityMeta = JsonObject & { /** diff --git a/packages/catalog-model/src/entity/constants.ts b/packages/catalog-model/src/entity/constants.ts index ff031fc1ae..6daf6b78d9 100644 --- a/packages/catalog-model/src/entity/constants.ts +++ b/packages/catalog-model/src/entity/constants.ts @@ -33,10 +33,15 @@ export const ENTITY_META_GENERATED_FIELDS = [ ] as const; /** - * Annotations for linking to entity from catalog pages. + * Annotation for linking to entity page from catalog pages. * * @public */ export const VIEW_URL_ANNOTATION = 'backstage.io/view-url'; -/** @public */ + +/** + * Annotation for linking to entity edit page from catalog pages. + * + * @public + */ export const EDIT_URL_ANNOTATION = 'backstage.io/edit-url'; diff --git a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts index b99901b89b..51059328a6 100644 --- a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts +++ b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts @@ -27,6 +27,8 @@ import { Entity } from '../Entity'; * Ensures that the format of individual fields of the entity envelope * is valid. * + * @remarks + * * This does not take into account machine generated fields such as uid, etag * and generation. * diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts index 721dc7d2dc..7f33e71f1c 100644 --- a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts @@ -24,6 +24,8 @@ import { EntityPolicy } from './types'; /** * Ensures that the entity spec is valid according to a schema. * + * @remarks + * * This should be the first policy in the list, to ensure that other downstream * policies can work with a structure that is at least valid in therms of the * typescript type. diff --git a/packages/catalog-model/src/entity/ref.ts b/packages/catalog-model/src/entity/ref.ts index 6a38e80dfe..958f750e32 100644 --- a/packages/catalog-model/src/entity/ref.ts +++ b/packages/catalog-model/src/entity/ref.ts @@ -69,6 +69,8 @@ export type EntityRefContext = { * Parses an entity reference, either on string or compound form, and always * returns a complete entity name including kind, namespace and name. * + * @remarks + * * This function automatically assumes the default namespace "default" unless * otherwise specified as part of the options, and will throw an error if no * kind was specified in the input reference and no default kind was given. @@ -100,7 +102,9 @@ export function parseEntityName( * Parses an entity reference, either on string or compound form, and returns * a structure with a name, and optional kind and namespace. * - * The options object can contain default values for the kind and namespace, + * @remarks + * + * The context object can contain default values for the kind and namespace, * that will be used if the input reference did not specify any. * * @public @@ -111,12 +115,12 @@ export function parseEntityName( export function parseEntityRef( ref: EntityRef, context?: { defaultKind: string; defaultNamespace: string }, -): { - kind: string; - namespace: string; - name: string; -}; -/** @public */ +): EntityName; +/** + * parseEntityRef with optional Kind. + * + * @public + */ export function parseEntityRef( ref: EntityRef, context?: { defaultKind: string }, @@ -125,7 +129,11 @@ export function parseEntityRef( namespace?: string; name: string; }; -/** @public */ +/** + * parseEntityRef with optional Namespace. + * + * @public + */ export function parseEntityRef( ref: EntityRef, context?: { defaultNamespace: string }, @@ -134,6 +142,11 @@ export function parseEntityRef( namespace: string; name: string; }; +/** + * parseEntityRef with optional Kind and Namespace. + * + * @public + */ export function parseEntityRef( ref: EntityRef, context: EntityRefContext = {}, @@ -223,6 +236,8 @@ export function serializeEntityRef( * Takes an entity or entity name/reference, and returns the string form of an * entity ref. * + * @remarks + * * This function creates a canonical and unique reference to the entity, converting * all parts of the name to lowercase and inserts the default namespace if needed. * It is typically not the best way to represent the entity reference to the user. @@ -256,6 +271,8 @@ export function stringifyEntityRef( /** * Compares an entity to either a string reference or a compound reference. * + * @remarks + * * The comparison is case insensitive, and all of kind, namespace, and name * must match (after applying the optional context to the ref). * diff --git a/packages/catalog-model/src/entity/util.ts b/packages/catalog-model/src/entity/util.ts index 314f3b7737..ce6415afde 100644 --- a/packages/catalog-model/src/entity/util.ts +++ b/packages/catalog-model/src/entity/util.ts @@ -43,6 +43,8 @@ export function generateEntityEtag(): string { * Checks whether there are any significant changes going from the previous to * the next version of this entity. * + * @remarks + * * Significance, in this case, means that we do not compare generated fields * such as uid, etag and generation. * @@ -98,6 +100,8 @@ export function entityHasChanges(previous: Entity, next: Entity): boolean { * Takes an old revision of an entity and a new desired state, and merges * them into a complete new state. * + * @remarks + * * The previous revision is expected to be a complete model loaded from the * catalog, including the uid, etag and generation fields. * diff --git a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts index 310b6695eb..d733b03c8c 100644 --- a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts @@ -18,7 +18,15 @@ import type { Entity } from '../entity/Entity'; import schema from '../schema/kinds/API.v1alpha1.schema.json'; import { ajvCompiledJsonSchemaValidator } from './util'; -/** @public */ +/** + * Backstage API kind Entity. APIs describe the interfaces for Components to communicate. + * + * @remarks + * + * See {@link https://backstage.io/docs/features/software-catalog/system-model} + * + * @public + */ export interface ApiEntityV1alpha1 extends Entity { apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; kind: 'API'; @@ -31,6 +39,10 @@ export interface ApiEntityV1alpha1 extends Entity { }; } -/** @public */ +/** + * {@link KindValidator} for {@link ApiEntityV1alpha1}. + * + * @public + */ export const apiEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts index 6777778409..f931eab553 100644 --- a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts @@ -18,7 +18,15 @@ import type { Entity } from '../entity/Entity'; import schema from '../schema/kinds/Component.v1alpha1.schema.json'; import { ajvCompiledJsonSchemaValidator } from './util'; -/** @public */ +/** + * Backstage catalog Component kind Entity. Represents a single, individual piece of software. + * + * @remarks + * + * See {@link https://backstage.io/docs/features/software-catalog/system-model} + * + * @public + */ export interface ComponentEntityV1alpha1 extends Entity { apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; kind: 'Component'; @@ -34,6 +42,10 @@ export interface ComponentEntityV1alpha1 extends Entity { }; } -/** @public */ +/** + * {@link KindValidator} for {@link ComponentEntityV1alpha1}. + * + * @public + */ export const componentEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/DomainEntityV1alpha1.ts b/packages/catalog-model/src/kinds/DomainEntityV1alpha1.ts index fe777af24e..e0302fcc52 100644 --- a/packages/catalog-model/src/kinds/DomainEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/DomainEntityV1alpha1.ts @@ -18,7 +18,15 @@ import type { Entity } from '../entity/Entity'; import schema from '../schema/kinds/Domain.v1alpha1.schema.json'; import { ajvCompiledJsonSchemaValidator } from './util'; -/** @public */ +/** + * Backstage Domain kind Entity. Domains group Systems together. + * + * @remarks + * + * See {@link https://backstage.io/docs/features/software-catalog/system-model} + * + * @public + */ export interface DomainEntityV1alpha1 extends Entity { apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; kind: 'Domain'; @@ -27,6 +35,10 @@ export interface DomainEntityV1alpha1 extends Entity { }; } -/** @public */ +/** + * {@link KindValidator} for {@link DomainEntityV1alpha1}. + * + * @public + */ export const domainEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts index f6aab856f0..8d88817dbe 100644 --- a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts @@ -18,7 +18,11 @@ import type { Entity } from '../entity/Entity'; import schema from '../schema/kinds/Group.v1alpha1.schema.json'; import { ajvCompiledJsonSchemaValidator } from './util'; -/** @public */ +/** + * Backstage catalog Group kind Entity. + * + * @public + */ export interface GroupEntityV1alpha1 extends Entity { apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; kind: 'Group'; @@ -35,6 +39,9 @@ export interface GroupEntityV1alpha1 extends Entity { }; } -/** @public */ +/** + * {@link KindValidator} for {@link GroupEntityV1alpha1}. + * @public + */ export const groupEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts index f2ae4bcc91..1872e43526 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts @@ -18,7 +18,11 @@ import type { Entity } from '../entity/Entity'; import schema from '../schema/kinds/Location.v1alpha1.schema.json'; import { ajvCompiledJsonSchemaValidator } from './util'; -/** @public */ +/** + * Backstage catalog Location kind Entity. + * + * @public + */ export interface LocationEntityV1alpha1 extends Entity { apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; kind: 'Location'; @@ -29,6 +33,10 @@ export interface LocationEntityV1alpha1 extends Entity { }; } -/** @public */ +/** + * {@link KindValidator} for {@link LocationEntityV1alpha1}. + * + * @public + */ export const locationEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/ResourceEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ResourceEntityV1alpha1.ts index acc6ab5245..0a360cb60b 100644 --- a/packages/catalog-model/src/kinds/ResourceEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ResourceEntityV1alpha1.ts @@ -18,7 +18,15 @@ import type { Entity } from '../entity/Entity'; import schema from '../schema/kinds/Resource.v1alpha1.schema.json'; import { ajvCompiledJsonSchemaValidator } from './util'; -/** @public */ +/** + * Backstage catalog Resource kind Entity. Represents infrastructure required to operate Components. + * + * @remarks + * + * See {@link https://backstage.io/docs/features/software-catalog/system-model} + * + * @public + */ export interface ResourceEntityV1alpha1 extends Entity { apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; kind: 'Resource'; @@ -30,6 +38,10 @@ export interface ResourceEntityV1alpha1 extends Entity { }; } -/** @public */ +/** + * {@link KindValidator} for {@link ResourceEntityV1alpha1}. + * + * @public + */ export const resourceEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/SystemEntityV1alpha1.ts b/packages/catalog-model/src/kinds/SystemEntityV1alpha1.ts index 5f90351480..7d1ab703a2 100644 --- a/packages/catalog-model/src/kinds/SystemEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/SystemEntityV1alpha1.ts @@ -18,7 +18,15 @@ import type { Entity } from '../entity/Entity'; import schema from '../schema/kinds/System.v1alpha1.schema.json'; import { ajvCompiledJsonSchemaValidator } from './util'; -/** @public */ +/** + * Backstage catalog System kind Entity. Systems group Comopnents, Resources and APIs together. + * + * @remarks + * + * See {@link https://backstage.io/docs/features/software-catalog/system-model} + * + * @public + */ export interface SystemEntityV1alpha1 extends Entity { apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; kind: 'System'; @@ -28,6 +36,10 @@ export interface SystemEntityV1alpha1 extends Entity { }; } -/** @public */ +/** + * {@link KindValidator} for {@link SystemEntityV1alpha1}. + * + * @public + */ export const systemEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts b/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts index e708d4b387..1b1966d05d 100644 --- a/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts +++ b/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts @@ -19,7 +19,11 @@ import type { Entity } from '../entity/Entity'; import schema from '../schema/kinds/Template.v1beta2.schema.json'; import { ajvCompiledJsonSchemaValidator } from './util'; -/** @public */ +/** + * Backstage catalog Template kind Entity. Templates are used by the Scaffolder plugin to create new Components. + * + * @public + */ export interface TemplateEntityV1beta2 extends Entity { apiVersion: 'backstage.io/v1beta2'; kind: 'Template'; @@ -38,6 +42,10 @@ export interface TemplateEntityV1beta2 extends Entity { }; } -/** @public */ +/** + * {@link KindValidator} for {@link TemplateEntityV1beta2}. + * + * @public + */ export const templateEntityV1beta2Validator = ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts b/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts index d719ab35ef..82aaebd9d6 100644 --- a/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts @@ -18,7 +18,11 @@ import type { Entity } from '../entity/Entity'; import schema from '../schema/kinds/User.v1alpha1.schema.json'; import { ajvCompiledJsonSchemaValidator } from './util'; -/** @public */ +/** + * Backstage catalog User kind Entity. + * + * @public + */ export interface UserEntityV1alpha1 extends Entity { apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; kind: 'User'; @@ -32,6 +36,10 @@ export interface UserEntityV1alpha1 extends Entity { }; } -/** @public */ +/** + * {@link KindValidator} for {@link UserEntityV1alpha1}. + * + * @public + */ export const userEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/relations.ts b/packages/catalog-model/src/kinds/relations.ts index 7546d1508b..4d545c9d35 100644 --- a/packages/catalog-model/src/kinds/relations.ts +++ b/packages/catalog-model/src/kinds/relations.ts @@ -24,61 +24,115 @@ Naming rules for relations in priority order: /** * An ownership relation where the owner is usually an organizational - * entity (user or group), and the other entity can be anything. + * entity (user or group), and the other entity can be anything. Reversed + * direction of {@link RELATION_OWNER_OF}. * * @public */ export const RELATION_OWNED_BY = 'ownedBy'; -/** @public */ + +/** + * A relationship from an owner to the owned entity. Reversed direction of + * {@link RELATION_OWNED_BY}. + * + * @public + */ export const RELATION_OWNER_OF = 'ownerOf'; /** - * A relation with an API entity, typically from a component + * A relation with an API entity, typically from a component. Reversed direction of + * {@link RELATION_API_CONSUMED_BY}. * * @public */ export const RELATION_CONSUMES_API = 'consumesApi'; -/** @public */ + +/** + * A relation of an API being consumed, typically by a component. Reversed direction of + * {@link RELATION_CONSUMES_API}. + * + * @public + */ export const RELATION_API_CONSUMED_BY = 'apiConsumedBy'; -/** @public */ + +/** + * A relation from an API provider entity (typically a component) to the API. Reversed direction of + * {@link RELATION_API_PROVIDED_BY}. + * + * @public + */ export const RELATION_PROVIDES_API = 'providesApi'; -/** @public */ + +/** + * A relation from an API to its provider entity (typically a component). Reversed direction of + * {@link RELATION_PROVIDES_API}. + * + * @public + */ export const RELATION_API_PROVIDED_BY = 'apiProvidedBy'; /** - * A relation denoting a dependency on another entity. + * A relation denoting a dependency on another entity. Reversed direction of + * {@link RELATION_DEPENDENCY_OF}. * * @public */ export const RELATION_DEPENDS_ON = 'dependsOn'; -/** @public */ + +/** + * A relation denoting a reverse dependency by another entity. Reversed direction of + * {@link RELATION_DEPENDS_ON}. + * + * @public + */ export const RELATION_DEPENDENCY_OF = 'dependencyOf'; /** * A parent/child relation to build up a tree, used for example to describe - * the organizational structure between groups. + * the organizational structure between groups. Reversed direction of + * {@link RELATION_CHILD_OF}. * * @public */ export const RELATION_PARENT_OF = 'parentOf'; -/** @public */ + +/** + * A relation from a child to a parent entity, used for example to describe + * the organizational structure between groups. Reversed direction of + * {@link RELATION_PARENT_OF}. + * + * @public + */ export const RELATION_CHILD_OF = 'childOf'; /** - * A membership relation, typically for users in a group. + * A membership relation, typically for users in a group. Reversed direction of + * {@link RELATION_HAS_MEMBER}. * * @public */ export const RELATION_MEMBER_OF = 'memberOf'; -/** @public */ + +/** + * A relation from a group to its member, typcally a user in a group. Reversed direction of + * {@link RELATION_MEMBER_OF}. + * + * @public + */ export const RELATION_HAS_MEMBER = 'hasMember'; /** * A part/whole relation, typically for components in a system and systems - * in a domain. + * in a domain. Reversed direction of {@link RELATION_HAS_PART}. * * @public */ export const RELATION_PART_OF = 'partOf'; -/** @public */ + +/** + * A relation from a containing entity to a contained entity. Reversed direction of + * {@link RELATION_PART_OF}. + * + * @public + */ export const RELATION_HAS_PART = 'hasPart'; diff --git a/packages/catalog-model/src/location/annotation.ts b/packages/catalog-model/src/location/annotation.ts index 4bcabdb705..8eb393f555 100644 --- a/packages/catalog-model/src/location/annotation.ts +++ b/packages/catalog-model/src/location/annotation.ts @@ -14,11 +14,20 @@ * limitations under the License. */ -/** @public */ +/** + * Constant storing location annotation. + * + * @public */ export const LOCATION_ANNOTATION = 'backstage.io/managed-by-location'; -/** @public */ +/** + * Constant storing origin location annotation + * + * @public */ export const ORIGIN_LOCATION_ANNOTATION = 'backstage.io/managed-by-origin-location'; -/** @public */ +/** + * Contant storing source location annotation + * + * @public */ export const SOURCE_LOCATION_ANNOTATION = 'backstage.io/source-location'; diff --git a/packages/catalog-model/src/location/helpers.ts b/packages/catalog-model/src/location/helpers.ts index cf12031dc1..1edd4e5ec5 100644 --- a/packages/catalog-model/src/location/helpers.ts +++ b/packages/catalog-model/src/location/helpers.ts @@ -65,6 +65,8 @@ export function parseLocationReference(ref: string): { /** * Turns a location reference into its string form. * + * @remarks + * * Note that the input type is not `LocationSpec`, because we do not want to * conflate the string form with the additional properties of that type. * @@ -90,6 +92,8 @@ export function stringifyLocationReference(ref: { /** * Returns the source code location of the Entity, to the extent that one exists. * + * @remarks + * * If the returned location type is of type 'url', the target should be readable at least * using the UrlReader from `@backstage/backend-common`. If it is not of type 'url', the caller * needs to have explicit handling of each location type or signal that it is not supported. diff --git a/packages/catalog-model/src/location/types.ts b/packages/catalog-model/src/location/types.ts index 1d16a9047d..725f873f2d 100644 --- a/packages/catalog-model/src/location/types.ts +++ b/packages/catalog-model/src/location/types.ts @@ -14,17 +14,28 @@ * limitations under the License. */ -/** @public */ +/** + * Holds the entity location information. + * + * @remarks + * + * `presence` flag: when using repo importer plugin, location is being created before the component yaml file is merged to the main branch. + * This flag is then set to indicate that the file can be not present. + * default value: 'required'. + * + * @public + */ export type LocationSpec = { type: string; target: string; - // When using repo importer plugin, location is being created before the component yaml file is merged to the main branch. - // This flag is then set to indicate that the file can be not present. - // default value: 'required'. presence?: 'optional' | 'required'; }; -/** @public */ +/** + * Entity location for a specific entity. + * + * @public + */ export type Location = { id: string; } & LocationSpec; diff --git a/packages/catalog-model/src/location/validation.ts b/packages/catalog-model/src/location/validation.ts index 452c69af53..2fd692e649 100644 --- a/packages/catalog-model/src/location/validation.ts +++ b/packages/catalog-model/src/location/validation.ts @@ -18,8 +18,10 @@ import * as yup from 'yup'; import { LocationSpec, Location } from './types'; /** + * Deprecated. + * * @public - * @deprecated Use JSONSchema or validators instead. + * @deprecated Use {@link JSONSchema} or validators instead. */ export const locationSpecSchema: yup.SchemaOf = yup .object({ @@ -31,8 +33,10 @@ export const locationSpecSchema: yup.SchemaOf = yup .required(); /** + * Deprecated. + * * @public - * @deprecated Use JSONSchema or validators instead. + * @deprecated Use {@link JSONSchema} or validators instead. */ export const locationSchema: yup.SchemaOf = yup .object({ @@ -45,8 +49,10 @@ export const locationSchema: yup.SchemaOf = yup .required(); /** + * Deprecated. + * * @public - * @deprecated Use JSONSchema or validators instead. + * @deprecated Use {@link JSONSchema} or validators instead. */ export const analyzeLocationSchema: yup.SchemaOf<{ location: LocationSpec }> = yup diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts index 1fbb3be95c..8ec1303a40 100644 --- a/packages/catalog-model/src/types.ts +++ b/packages/catalog-model/src/types.ts @@ -17,7 +17,11 @@ import { JsonValue } from '@backstage/config'; import { JSONSchema7 } from 'json-schema'; -/** @public */ +/** + * JSONSchema extendable by arbitrary JSON attributes + * + * @public + */ export type JSONSchema = JSONSchema7 & { [key in string]?: JsonValue }; /** @@ -35,7 +39,9 @@ export type EntityName = { * A reference by name to an entity, either as a compact string representation, * or as a compound reference structure. * - * The string representation is on the form [:][/]. + * @remarks + * + * The string representation is on the form `[:][/]`. * * Left-out parts of the reference need to be handled by the application, * either by rejecting the reference or by falling back to default values. diff --git a/packages/catalog-model/src/validation/entityEnvelopeSchemaValidator.ts b/packages/catalog-model/src/validation/entityEnvelopeSchemaValidator.ts index daa44ffaf1..a2a8ba64b7 100644 --- a/packages/catalog-model/src/validation/entityEnvelopeSchemaValidator.ts +++ b/packages/catalog-model/src/validation/entityEnvelopeSchemaValidator.ts @@ -25,6 +25,8 @@ import { compileAjvSchema, throwAjvError } from './ajv'; * if it matches that schema, or throws a {@link globals#TypeError} describing the * errors. * + * @remarks + * * Note that this validator is only meant for applying the base schema checks; * it does not take custom policies or additional processor based validation * into account. @@ -33,13 +35,15 @@ import { compileAjvSchema, throwAjvError } from './ajv'; * own, it may contain `$ref` references to the following, which are resolved * automatically for you: * - * - EntityEnvelope - * - Entity - * - EntityMeta - * - common# + * - {@link EntityEnvelope} + * - {@link Entity} + * - {@link EntityMeta} + * - `common#` + * + * See also {@link https://github.com/backstage/backstage/tree/master/packages/catalog-model/src/schema} * * @public - * @see https://github.com/backstage/backstage/tree/master/packages/catalog-model/src/schema + * */ export function entityEnvelopeSchemaValidator< T extends EntityEnvelope = EntityEnvelope, diff --git a/packages/catalog-model/src/validation/entityKindSchemaValidator.ts b/packages/catalog-model/src/validation/entityKindSchemaValidator.ts index 535dba9842..66259cc63d 100644 --- a/packages/catalog-model/src/validation/entityKindSchemaValidator.ts +++ b/packages/catalog-model/src/validation/entityKindSchemaValidator.ts @@ -24,6 +24,8 @@ import { compileAjvSchema, throwAjvError } from './ajv'; * schema apiVersion/kind didn't apply to that data, or throws a * {@link globals#TypeError} describing actual errors. * + * @remarks + * * This validator is highly specialized, in that it has special treatment of * the `kind` and `apiVersion` root keys. This only works if your schema has * their rule set to `"enum"`: @@ -47,13 +49,16 @@ import { compileAjvSchema, throwAjvError } from './ajv'; * The given schema may contain `$ref` references to the following, which are * resolved automatically for you: * - * - EntityEnvelope - * - Entity - * - EntityMeta - * - common# + * - {@link Entity} + * + * - {@link EntityEnvelope} + * + * - {@link EntityMeta} + * + * - `common#` + * @see {@link https://github.com/backstage/backstage/tree/master/packages/catalog-model/src/schema} * * @public - * @see https://github.com/backstage/backstage/tree/master/packages/catalog-model/src/schema */ export function entityKindSchemaValidator( schema: unknown, diff --git a/packages/catalog-model/src/validation/entitySchemaValidator.ts b/packages/catalog-model/src/validation/entitySchemaValidator.ts index dd110efd75..fc1918e566 100644 --- a/packages/catalog-model/src/validation/entitySchemaValidator.ts +++ b/packages/catalog-model/src/validation/entitySchemaValidator.ts @@ -24,21 +24,23 @@ import { compileAjvSchema, throwAjvError } from './ajv'; * returns that data cast to an {@link Entity} (or the given subtype) if it * matches that schema, or throws a {@link globals#TypeError} describing the errors. * + * @remarks + * * Note that this validator is only meant for applying the base schema checks; * it does not take custom policies or additional processor based validation * into account. * - * By default, the plain `Entity` schema is used. If you pass in your own, it + * By default, the plain {@link Entity} schema is used. If you pass in your own, it * may contain `$ref` references to the following, which are resolved * automatically for you: * - * - EntityEnvelope - * - Entity - * - EntityMeta - * - common# + * - {@link Entity} + * - {@link EntityEnvelope} + * - {@link EntityMeta} + * - `common#` * * @public - * @see https://github.com/backstage/backstage/tree/master/packages/catalog-model/src/schema + * @see {@link https://github.com/backstage/backstage/tree/master/packages/catalog-model/src/schema} */ export function entitySchemaValidator( schema?: unknown, diff --git a/packages/catalog-model/src/validation/makeValidator.ts b/packages/catalog-model/src/validation/makeValidator.ts index bd6dccc419..ad3f52e563 100644 --- a/packages/catalog-model/src/validation/makeValidator.ts +++ b/packages/catalog-model/src/validation/makeValidator.ts @@ -30,7 +30,11 @@ const defaultValidators: Validators = { isValidTag: CommonValidatorFunctions.isValidTag, }; -/** @public */ +/** + * Creates a {@link Validators} object from `overrides`, with default values taken from {@link KubernetesValidatorFunctions} + * + * @public + */ export function makeValidator(overrides: Partial = {}): Validators { return { ...defaultValidators, diff --git a/packages/catalog-model/src/validation/types.ts b/packages/catalog-model/src/validation/types.ts index 23d639c166..e3615e6a83 100644 --- a/packages/catalog-model/src/validation/types.ts +++ b/packages/catalog-model/src/validation/types.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -/** @public */ +/** + * Type alias for implementing validators of various entity objects. + * + * @public + */ export type Validators = { isValidApiVersion(value: unknown): boolean; isValidKind(value: unknown): boolean; From 74df0b1e2b0fa7aca9cba075ad1abe7a1ba2e924 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Oct 2021 04:10:55 +0000 Subject: [PATCH 019/103] chore(deps): bump selfsigned from 1.10.8 to 1.10.11 Bumps [selfsigned](https://github.com/jfromaniello/selfsigned) from 1.10.8 to 1.10.11. - [Release notes](https://github.com/jfromaniello/selfsigned/releases) - [Commits](https://github.com/jfromaniello/selfsigned/compare/v1.10.8...v1.10.11) --- updated-dependencies: - dependency-name: selfsigned dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/yarn.lock b/yarn.lock index 20d2a47a1d..c643e836f4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7439,7 +7439,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@>=16.9.0", "@types/react@^16.9": +"@types/react@*", "@types/react@>=16.9.0": version "16.14.15" resolved "https://registry.npmjs.org/@types/react/-/react-16.14.15.tgz#95d8fa3148050e94bcdc5751447921adbe19f9e6" integrity sha512-jOxlBV9RGZhphdeqJTCv35VZOkjY+XIEY2owwSk84BNDdDv2xS6Csj6fhi+B/q30SR9Tz8lDNt/F2Z5RF3TrRg== @@ -7448,6 +7448,15 @@ "@types/scheduler" "*" csstype "^3.0.2" +"@types/react@^16.9": + version "16.14.17" + resolved "https://registry.npmjs.org/@types/react/-/react-16.14.17.tgz#c57fcfb05efa6423f5b65fcd4a75f63f05b162bf" + integrity sha512-pMLc/7+7SEdQa9A+hN9ujI8blkjFqYAZVqh3iNXqdZ0cQ8TIR502HMkNJniaOGv9SAgc47jxVKoiBJ7c0AakvQ== + dependencies: + "@types/prop-types" "*" + "@types/scheduler" "*" + csstype "^3.0.2" + "@types/recharts@^1.8.14", "@types/recharts@^1.8.15": version "1.8.19" resolved "https://registry.npmjs.org/@types/recharts/-/recharts-1.8.19.tgz#047f72cf4c25df545aa1085fe3a085e58a2483c1" @@ -10311,9 +10320,9 @@ canvas@^2.6.1: simple-get "^3.0.3" canvg@^3.0.6: - version "3.0.8" - resolved "https://registry.npmjs.org/canvg/-/canvg-3.0.8.tgz#9125a4b7033d2f237402241b192587385f4589e6" - integrity sha512-9De5heHfVRgCkln3CGEeSJMviN5U2RyxL4uutYoe8HxI60BjH2XnT2ZUHIp+ZaAZNTUd5Asqfut8WEEdANqfAg== + version "3.0.9" + resolved "https://registry.npmjs.org/canvg/-/canvg-3.0.9.tgz#9ba095f158b94b97ca2c9c1c40785b11dc08df6d" + integrity sha512-rDXcnRPuz4QHoCilMeoTxql+fvGqNAxp+qV/KHD8rOiJSAfVjFclbdUNHD2Uqfthr+VMg17bD2bVuk6F07oLGw== dependencies: "@babel/runtime" "^7.12.5" "@types/raf" "^3.4.0" @@ -10322,7 +10331,7 @@ canvg@^3.0.6: regenerator-runtime "^0.13.7" rgbcolor "^1.0.1" stackblur-canvas "^2.0.0" - svg-pathdata "^5.0.5" + svg-pathdata "^6.0.3" capital-case@^1.0.4: version "1.0.4" @@ -11371,9 +11380,9 @@ core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: integrity sha512-GUbtPllXMYRzIgHNZ4dTYTcUemls2cni83Q4Q/TrFONHfhcg9oEGOtaGHfb0cpzec60P96UKPvMkjX1jET8rUw== core-js@^3.6.0, core-js@^3.8.3: - version "3.18.1" - resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.1.tgz#289d4be2ce0085d40fc1244c0b1a54c00454622f" - integrity sha512-vJlUi/7YdlCZeL6fXvWNaLUPh/id12WXj3MbkMw5uOyF0PfWPBNOCNbs53YqgrvtujLNlt9JQpruyIKkUZ+PKA== + version "3.18.3" + resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.3.tgz#86a0bba2d8ec3df860fefcc07a8d119779f01509" + integrity sha512-tReEhtMReZaPFVw7dajMx0vlsz3oOb8ajgPoHVYGxr8ErnZ6PcYEvvmjGmXlfpnxpkYSdOQttjB+MvVbCGfvLw== core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" @@ -24479,20 +24488,13 @@ select-hose@^2.0.0: resolved "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" integrity sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo= -selfsigned@^1.10.11: +selfsigned@^1.10.11, selfsigned@^1.10.7: version "1.10.11" resolved "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.11.tgz#24929cd906fe0f44b6d01fb23999a739537acbe9" integrity sha512-aVmbPOfViZqOZPgRBT0+3u4yZFHpmnIghLMlAcb5/xhp5ZtB/RVnKhz5vl2M32CLXAqR4kha9zfhNg0Lf/sxKA== dependencies: node-forge "^0.10.0" -selfsigned@^1.10.7: - version "1.10.8" - resolved "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.8.tgz#0d17208b7d12c33f8eac85c41835f27fc3d81a30" - integrity sha512-2P4PtieJeEwVgTU9QEcwIRDQ/mXJLX8/+I3ur+Pg16nS8oNbrGxEso9NyYWy8NAmXiNl4dlAp5MwoNeCWzON4w== - dependencies: - node-forge "^0.10.0" - semver-compare@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" @@ -25859,10 +25861,10 @@ svg-parser@^2.0.2: resolved "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz#fdc2e29e13951736140b76cb122c8ee6630eb6b5" integrity sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ== -svg-pathdata@^5.0.5: - version "5.0.5" - resolved "https://registry.npmjs.org/svg-pathdata/-/svg-pathdata-5.0.5.tgz#65e8d765642ba15fe15434444087d082bc526b29" - integrity sha512-TAAvLNSE3fEhyl/Da19JWfMAdhSXTYeviXsLSoDT1UM76ADj5ndwAPX1FKQEgB/gFMPavOy6tOqfalXKUiXrow== +svg-pathdata@^6.0.3: + version "6.0.3" + resolved "https://registry.npmjs.org/svg-pathdata/-/svg-pathdata-6.0.3.tgz#80b0e0283b652ccbafb69ad4f8f73e8d3fbf2cac" + integrity sha512-qsjeeq5YjBZ5eMdFuUa4ZosMLxgr5RZ+F+Y1OrDhuOCEInRMA3x74XdBtggJcj9kOeInz0WE+LgCPDkZFlBYJw== svgo@^1.0.0, svgo@^1.2.2: version "1.3.2" From d8ea1edcdbead3460d1cbfbfc56d067b5b476c87 Mon Sep 17 00:00:00 2001 From: Marcus Eide Date: Tue, 12 Oct 2021 16:11:29 +0200 Subject: [PATCH 020/103] Add support for auth.keyStore in application config Signed-off-by: Marcus Eide --- app-config.yaml | 9 +++++++++ plugins/auth-backend/config.d.ts | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/app-config.yaml b/app-config.yaml index c24ac1701b..60e2a2b9e0 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -292,6 +292,15 @@ scaffolder: visibility: public # or or 'private' auth: + ### Add auth.keyStore.provider to more granularly control how to store JWK data when running + # the auth-backend. + # + # keyStore: + # provider: firestore + # firestore: + # projectId: my-project + # path: my-sessions + environment: development ### Providing an auth.session.secret will enable session support in the auth-backend # session: diff --git a/plugins/auth-backend/config.d.ts b/plugins/auth-backend/config.d.ts index 61bbe15d05..b15b0da628 100644 --- a/plugins/auth-backend/config.d.ts +++ b/plugins/auth-backend/config.d.ts @@ -31,6 +31,24 @@ export interface Config { secret?: string; }; + /** To control how to store JWK data in auth-backend */ + keyStore?: { + provider?: 'postgres' | 'memory' | 'firestore'; + firestore?: { + /** The Google Cloud Project ID */ + projectId?: string; + /** + * Local file containing the Service Account credentials. + * You can omit this value to automatically read from + * GOOGLE_APPLICATION_CREDENTIALS env which is useful for local + * development. + */ + keyFilename?: string; + /** The path to use for the collection. Defaults to 'sessions' */ + path?: string; + }; + }; + /** * The available auth-provider options and attributes */ From 3a62f7619a9a915dec5bfb22ce3164c3a1e947c8 Mon Sep 17 00:00:00 2001 From: Marcus Eide Date: Tue, 12 Oct 2021 16:14:30 +0200 Subject: [PATCH 021/103] Add class to use Firestore as a key-store Signed-off-by: Marcus Eide --- .../src/identity/FirestoreKeyStore.test.ts | 152 ++++++++++++++++++ .../src/identity/FirestoreKeyStore.ts | 90 +++++++++++ yarn.lock | 52 +++++- 3 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts create mode 100644 plugins/auth-backend/src/identity/FirestoreKeyStore.ts diff --git a/plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts b/plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts new file mode 100644 index 0000000000..03fcc15a77 --- /dev/null +++ b/plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts @@ -0,0 +1,152 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Firestore } from '@google-cloud/firestore'; + +import { FirestoreKeyStore } from './FirestoreKeyStore'; +import { AnyJWK } from './types'; + +const data = jest.fn().mockReturnValue('data'); +const toDate = jest.fn().mockReturnValue('date'); + +const firestoreMock = { + collection: jest.fn().mockReturnThis(), + delete: jest.fn(), + doc: jest.fn().mockReturnThis(), + get: jest.fn().mockReturnValue({ + docs: [{ data, createTime: { toDate } }], + }), + set: jest.fn(), +}; + +jest.mock('@google-cloud/firestore', () => ({ + Firestore: jest.fn().mockImplementation(() => firestoreMock), +})); + +describe('FirestoreKeyStore', () => { + const OLD_ENV = process.env; + const key = { + kid: '123', + use: 'sig', + kty: 'plain', + alg: 'Base64', + } as AnyJWK; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterAll(() => { + process.env = OLD_ENV; + }); + + it('can create an instance without settings', () => { + const keyStore = FirestoreKeyStore.create(); + + expect(keyStore).toBeInstanceOf(FirestoreKeyStore); + }); + + it('can set the project id', async () => { + FirestoreKeyStore.create({ projectId: 'my-project' }); + + expect(Firestore).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: 'my-project', + }), + ); + }); + + it('can handle keyfile file', async () => { + FirestoreKeyStore.create({ keyFilename: 'keyFile.json' }); + + expect(Firestore).toHaveBeenCalledWith( + expect.objectContaining({ + keyFilename: 'keyFile.json', + }), + ); + }); + + it('can use default google credentials', () => { + process.env.GOOGLE_APPLICATION_CREDENTIALS = 'cred.json'; + FirestoreKeyStore.create(); + + expect(Firestore).toHaveBeenCalledWith( + expect.objectContaining({ + keyFilename: 'cred.json', + }), + ); + }); + + it('can uses a default path', async () => { + const keyStore = FirestoreKeyStore.create(); + await keyStore.addKey(key); + + expect(firestoreMock.collection).toBeCalledWith('sessions'); + }); + + it('can set the path', async () => { + const keyStore = FirestoreKeyStore.create({ + path: 'my-path', + }); + await keyStore.addKey(key); + + expect(firestoreMock.collection).toBeCalledWith('my-path'); + }); + + it('can add keys', async () => { + const keyStore = FirestoreKeyStore.create(); + await keyStore.addKey(key); + + expect(firestoreMock.collection).toBeCalledWith('sessions'); + expect(firestoreMock.doc).toBeCalledWith(key.kid); + expect(firestoreMock.set).toHaveBeenCalledWith({ + kid: key.kid, + key: JSON.stringify(key), + }); + }); + + it('can delete a single key', async () => { + const keyStore = FirestoreKeyStore.create(); + await keyStore.removeKeys(['123']); + + expect(firestoreMock.collection).toBeCalledWith('sessions'); + expect(firestoreMock.doc).toBeCalledWith('123'); + expect(firestoreMock.delete).toBeCalledTimes(1); + }); + + it('can delete a multiple keys', async () => { + const keyStore = FirestoreKeyStore.create(); + await keyStore.removeKeys(['123', '456']); + + expect(firestoreMock.collection).toBeCalledWith('sessions'); + expect(firestoreMock.doc).toBeCalledWith('123'); + expect(firestoreMock.doc).toBeCalledWith('456'); + expect(firestoreMock.delete).toBeCalledTimes(2); + }); + + it('can list keys', async () => { + const keyStore = FirestoreKeyStore.create(); + const items = await keyStore.listKeys(); + + expect(firestoreMock.collection).toBeCalledWith('sessions'); + expect(firestoreMock.get).toBeCalledTimes(1); + expect(data).toBeCalledTimes(1); + expect(toDate).toBeCalledTimes(1); + expect(items).toMatchObject({ + items: [{ key: 'data', createdAt: 'date' }], + }); + }); +}); diff --git a/plugins/auth-backend/src/identity/FirestoreKeyStore.ts b/plugins/auth-backend/src/identity/FirestoreKeyStore.ts new file mode 100644 index 0000000000..cf8c082540 --- /dev/null +++ b/plugins/auth-backend/src/identity/FirestoreKeyStore.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Firestore, Settings } from '@google-cloud/firestore'; + +import { AnyJWK, KeyStore, StoredKey } from './types'; + +type FirestoreSettings = Settings & { + path?: string; +}; + +export class FirestoreKeyStore implements KeyStore { + static create(settings?: FirestoreSettings): FirestoreKeyStore { + const { projectId, keyFilename, path } = settings ?? {}; + const database = new Firestore({ + projectId, + keyFilename: keyFilename ?? process.env.GOOGLE_APPLICATION_CREDENTIALS, + }); + + return new FirestoreKeyStore(database, path ?? 'sessions'); + } + + private constructor( + private readonly database: Firestore, + private readonly path: string, + ) {} + + async addKey(key: AnyJWK): Promise { + await this.database + .collection(this.path) + .doc(key.kid) + .set({ + kid: key.kid, + key: JSON.stringify(key), + }); + } + + async listKeys(): Promise<{ items: StoredKey[] }> { + const docs = await this.database.collection(this.path).get(); + return { + items: docs.docs.map(doc => ({ + key: doc.data() as AnyJWK, + createdAt: doc.createTime.toDate(), + })), + }; + } + + async removeKeys(kids: string[]): Promise { + // This is probably really slow, but it's done async in the background + for (const kid of kids) { + await this.database.collection(this.path).doc(kid).delete(); + } + + /** + * This could be achieved with batching but there's a couple of limitations with that: + * + * - A batched write can contain a maximum of 500 operations + * https://firebase.google.com/docs/firestore/manage-data/transactions#batched-writes + * + * - The "in" operator can combine a maximum of 10 equality clauses + * https://firebase.google.com/docs/firestore/query-data/queries#in_not-in_and_array-contains-any + * + * Example: + * + * const batch = this.database.batch(); + * const docs = await this.database + * .collection(this.path) + * .where('kid', 'in', kids) + * .get(); + * docs.forEach(doc => { + * batch.delete(doc.ref); + * }); + * await batch.commit(); + * + */ + } +} diff --git a/yarn.lock b/yarn.lock index 9915fb6279..b110393471 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3000,6 +3000,16 @@ dependencies: google-gax "^2.12.0" +"@google-cloud/firestore@^4.15.1": + version "4.15.1" + resolved "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-4.15.1.tgz#ed764fc76823ce120e68fe8c27ef1edd0650cd93" + integrity sha512-2PWsCkEF1W02QbghSeRsNdYKN1qavrHBP3m72gPDMHQSYrGULOaTi7fSJquQmAtc4iPVB2/x6h80rdLHTATQtA== + dependencies: + fast-deep-equal "^3.1.1" + functional-red-black-tree "^1.0.1" + google-gax "^2.24.1" + protobufjs "^6.8.6" + "@google-cloud/paginator@^3.0.0": version "3.0.5" resolved "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-3.0.5.tgz#9d6b96c421a89bd560c1bc2c197c7611ef21db6c" @@ -15053,6 +15063,21 @@ google-auth-library@^7.0.0, google-auth-library@^7.0.2: jws "^4.0.0" lru-cache "^6.0.0" +google-auth-library@^7.6.1: + version "7.10.0" + resolved "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.10.0.tgz#6ab852f8e1abbe425aec695ac6501f80bd5eba56" + integrity sha512-ICsqaU+lxMHVlDUzMrfVIEqnARw2AwBiZ/2KnNM6BcTf9Nott+Af87DTIzmlnW865p3REUP2MVL0xkPC3a61aQ== + dependencies: + arrify "^2.0.0" + base64-js "^1.3.0" + ecdsa-sig-formatter "^1.0.11" + fast-text-encoding "^1.0.0" + gaxios "^4.0.0" + gcp-metadata "^4.2.0" + gtoken "^5.0.4" + jws "^4.0.0" + lru-cache "^6.0.0" + google-gax@^2.12.0: version "2.14.1" resolved "https://registry.npmjs.org/google-gax/-/google-gax-2.14.1.tgz#74885c5d9f01db412917fc49bbf20c4884828d36" @@ -15071,6 +15096,25 @@ google-gax@^2.12.0: protobufjs "^6.10.2" retry-request "^4.0.0" +google-gax@^2.24.1: + version "2.27.0" + resolved "https://registry.npmjs.org/google-gax/-/google-gax-2.27.0.tgz#78ba655ae7707cb92ba37c35932381b5970c1772" + integrity sha512-xcLCeNKCqNm/w0At7/vdZHV/zol/iRS+PSAZTu7i6xNGBra/kWI3cfn4M6ZLQXeUEGbTVLJ4zGm53TVc4lvbDA== + dependencies: + "@grpc/grpc-js" "~1.3.0" + "@grpc/proto-loader" "^0.6.1" + "@types/long" "^4.0.0" + abort-controller "^3.0.0" + duplexify "^4.0.0" + fast-text-encoding "^1.0.3" + google-auth-library "^7.6.1" + is-stream-ended "^0.1.4" + node-fetch "^2.6.1" + object-hash "^2.1.1" + proto3-json-serializer "^0.1.1" + protobufjs "6.11.2" + retry-request "^4.0.0" + google-p12-pem@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.3.tgz#673ac3a75d3903a87f05878f3c75e06fc151669e" @@ -19722,6 +19766,7 @@ minipass-fetch@^1.3.0, minipass-fetch@^1.3.2: resolved "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-1.3.3.tgz#34c7cea038c817a8658461bf35174551dce17a0a" integrity sha512-akCrLDWfbdAWkMLBxJEeWTdNsjML+dt5YgOI4gJ53vuO0vrmYQkUPxa6j6V65s9CcePIr2SSWqjT2EcrNseryQ== dependencies: + encoding "^0.1.12" minipass "^3.1.0" minipass-sized "^1.0.3" minizlib "^2.0.0" @@ -22556,7 +22601,12 @@ proto-list@~1.2.1: resolved "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849" integrity sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk= -protobufjs@^6.10.0, protobufjs@^6.10.2: +proto3-json-serializer@^0.1.1: + version "0.1.4" + resolved "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-0.1.4.tgz#aa2dc4c9c9b7ea05631354b2c2e52c227539a7f0" + integrity sha512-bFzdsKU/zaTobWrRxRniMZIzzcgKYlmBWL1gAcTXZ2M7TQTGPI0JoYYs6bN7tpWj59ZCfwg7Ii/A2e8BbQGYnQ== + +protobufjs@6.11.2, protobufjs@^6.10.0, protobufjs@^6.10.2, protobufjs@^6.8.6: version "6.11.2" resolved "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.2.tgz#de39fabd4ed32beaa08e9bb1e30d08544c1edf8b" integrity sha512-4BQJoPooKJl2G9j3XftkIXjoC9C0Av2NOrWmbLWT1vH32GcSUHjM0Arra6UfTsVyfMAuFzaLucXn1sadxJydAw== From 16288b6580c8701ba0d3bdac26e04f597916a3ae Mon Sep 17 00:00:00 2001 From: Marcus Eide Date: Tue, 12 Oct 2021 16:16:15 +0200 Subject: [PATCH 022/103] Add class that will return a key-store based on application config Signed-off-by: Marcus Eide --- .../src/identity/KeyStores.test.ts | 86 +++++++++++++++++++ .../auth-backend/src/identity/KeyStores.ts | 75 ++++++++++++++++ plugins/auth-backend/src/identity/index.ts | 3 + 3 files changed, 164 insertions(+) create mode 100644 plugins/auth-backend/src/identity/KeyStores.test.ts create mode 100644 plugins/auth-backend/src/identity/KeyStores.ts diff --git a/plugins/auth-backend/src/identity/KeyStores.test.ts b/plugins/auth-backend/src/identity/KeyStores.test.ts new file mode 100644 index 0000000000..957c4c7953 --- /dev/null +++ b/plugins/auth-backend/src/identity/KeyStores.test.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DatabaseManager } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; + +import { MemoryKeyStore } from './MemoryKeyStore'; +import { DatabaseKeyStore } from './DatabaseKeyStore'; +import { FirestoreKeyStore } from './FirestoreKeyStore'; +import { KeyStores } from './KeyStores'; + +describe('KeyStores', () => { + const defaultConfigOptions = { + auth: { + keyStore: { + provider: 'memory', + }, + }, + }; + const defaultConfig = new ConfigReader(defaultConfigOptions); + + it('reads auth section from config', async () => { + const configSpy = jest.spyOn(defaultConfig, 'getOptionalConfig'); + const keyStore = await KeyStores.fromConfig(defaultConfig); + + expect(keyStore).toBeInstanceOf(MemoryKeyStore); + expect(configSpy).toHaveBeenCalledWith('auth.keyStore'); + expect( + defaultConfig + .getOptionalConfig('auth.keyStore') + ?.getOptionalString('provider'), + ).toBe(defaultConfigOptions.auth.keyStore.provider); + }); + + it('can handle without auth config', async () => { + const config = new ConfigReader({ + backend: { + database: { + client: 'sqlite3', + connection: ':memory:', + }, + }, + }); + const database = + DatabaseManager.fromConfig(config).forPlugin('auth-backend'); + const keyStore = await KeyStores.fromConfig(config, { database }); + + expect(keyStore).toBeInstanceOf(DatabaseKeyStore); + }); + + it('can handle additional provider config', async () => { + const configOptions = { + auth: { + keyStore: { + provider: 'firestore', + firestore: { + projectId: 'my-project', + }, + }, + }, + }; + const config = new ConfigReader(configOptions); + const keyStore = await KeyStores.fromConfig(config); + + expect(keyStore).toBeInstanceOf(FirestoreKeyStore); + expect( + config + .getOptionalConfig('auth.keyStore') + ?.getOptionalConfig('firestore') + ?.getOptionalString('projectId'), + ).toBe(configOptions.auth.keyStore.firestore.projectId); + }); +}); diff --git a/plugins/auth-backend/src/identity/KeyStores.ts b/plugins/auth-backend/src/identity/KeyStores.ts new file mode 100644 index 0000000000..0991f7ea7b --- /dev/null +++ b/plugins/auth-backend/src/identity/KeyStores.ts @@ -0,0 +1,75 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Logger } from 'winston'; + +import { PluginDatabaseManager } from '@backstage/backend-common'; +import { Config } from '@backstage/config'; + +import { DatabaseKeyStore } from './DatabaseKeyStore'; +import { MemoryKeyStore } from './MemoryKeyStore'; +import { FirestoreKeyStore } from './FirestoreKeyStore'; +import { KeyStore } from './types'; + +type Options = { + logger?: Logger; + database?: PluginDatabaseManager; +}; + +export class KeyStores { + /** + * Looks at the `auth.keyStore` section in the application configuration + * and returns a KeyStore store. Defaults to `postgres` + * + * @returns a KeyStore store + */ + static async fromConfig( + config: Config, + options?: Options, + ): Promise { + const { logger, database } = options ?? {}; + + const ks = config.getOptionalConfig('auth.keyStore'); + const provider = ks?.getOptionalString('provider') ?? 'postgres'; + const providerConfig = ks?.getOptionalConfig(provider); + + logger?.info(`Configuring "${provider}" as KeyStore provider`); + + if (provider === 'postgres') { + if (!database) { + throw new Error('This KeyStore provider requires a database'); + } + + return await DatabaseKeyStore.create({ + database: await database.getClient(), + }); + } + + if (provider === 'memory') { + return new MemoryKeyStore(); + } + + if (provider === 'firestore') { + return FirestoreKeyStore.create({ + projectId: providerConfig?.getOptionalString('projectId'), + keyFilename: providerConfig?.getOptionalString('keyFilename'), + path: providerConfig?.getOptionalString('path'), + }); + } + + throw new Error(`Unknown KeyStore provider: ${provider}`); + } +} diff --git a/plugins/auth-backend/src/identity/index.ts b/plugins/auth-backend/src/identity/index.ts index d88339e8e4..73858d7e07 100644 --- a/plugins/auth-backend/src/identity/index.ts +++ b/plugins/auth-backend/src/identity/index.ts @@ -18,4 +18,7 @@ export { createOidcRouter } from './router'; export { IdentityClient } from './IdentityClient'; export { TokenFactory } from './TokenFactory'; export { DatabaseKeyStore } from './DatabaseKeyStore'; +export { MemoryKeyStore } from './MemoryKeyStore'; +export { FirestoreKeyStore } from './FirestoreKeyStore'; +export { KeyStores } from './KeyStores'; export type { KeyStore, TokenIssuer, TokenParams } from './types'; From 12d4abe7b25319a085e2cfeab73f52e60004f747 Mon Sep 17 00:00:00 2001 From: Marcus Eide Date: Tue, 12 Oct 2021 16:18:30 +0200 Subject: [PATCH 023/103] Add project dependency to package.json Signed-off-by: Marcus Eide --- plugins/auth-backend/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 342718db55..8f7f623652 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -36,6 +36,7 @@ "@backstage/config": "^0.1.10", "@backstage/errors": "^0.1.1", "@backstage/test-utils": "^0.1.19", + "@google-cloud/firestore": "^4.15.1", "@types/express": "^4.17.6", "@types/passport": "^1.0.3", "compression": "^1.7.4", From 4f4e22d176d12195eb0ee007a0f25423e01d0387 Mon Sep 17 00:00:00 2001 From: Marcus Eide Date: Tue, 12 Oct 2021 16:19:45 +0200 Subject: [PATCH 024/103] Use KeyStores.fromConfig() in router Signed-off-by: Marcus Eide --- plugins/auth-backend/src/service/router.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index 423975aa75..8cce643987 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -29,7 +29,7 @@ import { import { NotFoundError } from '@backstage/errors'; import { CatalogClient } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; -import { createOidcRouter, DatabaseKeyStore, TokenFactory } from '../identity'; +import { createOidcRouter, TokenFactory, KeyStores } from '../identity'; import session from 'express-session'; import passport from 'passport'; import { Minimatch } from 'minimatch'; @@ -38,7 +38,7 @@ type ProviderFactories = { [s: string]: AuthProviderFactory }; export interface RouterOptions { logger: Logger; - database: PluginDatabaseManager; + database?: PluginDatabaseManager; config: Config; discovery: PluginEndpointDiscovery; providerFactories?: ProviderFactories; @@ -56,11 +56,9 @@ export async function createRouter({ const appUrl = config.getString('app.baseUrl'); const authUrl = await discovery.getExternalBaseUrl('auth'); + const keyStore = await KeyStores.fromConfig(config, { logger, database }); const keyDurationSeconds = 3600; - const keyStore = await DatabaseKeyStore.create({ - database: await database.getClient(), - }); const tokenIssuer = new TokenFactory({ issuer: authUrl, keyStore, From d2f755fa732b6d777f51a35630cb1110b5b9ba54 Mon Sep 17 00:00:00 2001 From: Marcus Eide Date: Wed, 13 Oct 2021 09:21:30 +0200 Subject: [PATCH 025/103] Update api-reports Signed-off-by: Marcus Eide --- plugins/auth-backend/api-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/auth-backend/api-report.md b/plugins/auth-backend/api-report.md index 0bdd73f3e8..a20538e223 100644 --- a/plugins/auth-backend/api-report.md +++ b/plugins/auth-backend/api-report.md @@ -501,7 +501,7 @@ export interface RouterOptions { // (undocumented) config: Config; // (undocumented) - database: PluginDatabaseManager; + database?: PluginDatabaseManager; // (undocumented) discovery: PluginEndpointDiscovery; // (undocumented) From bf259c989020210b4879aa8bec6e6024d3292814 Mon Sep 17 00:00:00 2001 From: Marcus Eide Date: Wed, 13 Oct 2021 09:58:47 +0200 Subject: [PATCH 026/103] Add Firestore to vale vocab Signed-off-by: Marcus Eide --- .github/styles/vocab.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index a38baab3dc..a93f75192b 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -310,3 +310,4 @@ Zalando Zhou zoomable zsh +Firestore From ab9b4a6ea6a441e96e429def3bf43548ac089a98 Mon Sep 17 00:00:00 2001 From: Marcus Eide Date: Wed, 13 Oct 2021 09:59:00 +0200 Subject: [PATCH 027/103] Add changeset Signed-off-by: Marcus Eide --- .changeset/pretty-ways-search.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/pretty-ways-search.md diff --git a/.changeset/pretty-ways-search.md b/.changeset/pretty-ways-search.md new file mode 100644 index 0000000000..212c23b09a --- /dev/null +++ b/.changeset/pretty-ways-search.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Add Firestore as key-store provider. +Add `auth.keyStore` section to application config. From 5ad1f523d569e020548fe17713165775d53995b8 Mon Sep 17 00:00:00 2001 From: Marcus Eide Date: Wed, 13 Oct 2021 14:25:06 +0200 Subject: [PATCH 028/103] Make the create function async Signed-off-by: Marcus Eide --- .../src/identity/FirestoreKeyStore.test.ts | 24 +++++++++---------- .../src/identity/FirestoreKeyStore.ts | 4 +++- .../auth-backend/src/identity/KeyStores.ts | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts b/plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts index 03fcc15a77..a12e2ed40e 100644 --- a/plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts +++ b/plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts @@ -53,14 +53,14 @@ describe('FirestoreKeyStore', () => { process.env = OLD_ENV; }); - it('can create an instance without settings', () => { - const keyStore = FirestoreKeyStore.create(); + it('can create an instance without settings', async () => { + const keyStore = await FirestoreKeyStore.create(); expect(keyStore).toBeInstanceOf(FirestoreKeyStore); }); it('can set the project id', async () => { - FirestoreKeyStore.create({ projectId: 'my-project' }); + await FirestoreKeyStore.create({ projectId: 'my-project' }); expect(Firestore).toHaveBeenCalledWith( expect.objectContaining({ @@ -70,7 +70,7 @@ describe('FirestoreKeyStore', () => { }); it('can handle keyfile file', async () => { - FirestoreKeyStore.create({ keyFilename: 'keyFile.json' }); + await FirestoreKeyStore.create({ keyFilename: 'keyFile.json' }); expect(Firestore).toHaveBeenCalledWith( expect.objectContaining({ @@ -79,9 +79,9 @@ describe('FirestoreKeyStore', () => { ); }); - it('can use default google credentials', () => { + it('can use default google credentials', async () => { process.env.GOOGLE_APPLICATION_CREDENTIALS = 'cred.json'; - FirestoreKeyStore.create(); + await FirestoreKeyStore.create(); expect(Firestore).toHaveBeenCalledWith( expect.objectContaining({ @@ -91,14 +91,14 @@ describe('FirestoreKeyStore', () => { }); it('can uses a default path', async () => { - const keyStore = FirestoreKeyStore.create(); + const keyStore = await FirestoreKeyStore.create(); await keyStore.addKey(key); expect(firestoreMock.collection).toBeCalledWith('sessions'); }); it('can set the path', async () => { - const keyStore = FirestoreKeyStore.create({ + const keyStore = await FirestoreKeyStore.create({ path: 'my-path', }); await keyStore.addKey(key); @@ -107,7 +107,7 @@ describe('FirestoreKeyStore', () => { }); it('can add keys', async () => { - const keyStore = FirestoreKeyStore.create(); + const keyStore = await FirestoreKeyStore.create(); await keyStore.addKey(key); expect(firestoreMock.collection).toBeCalledWith('sessions'); @@ -119,7 +119,7 @@ describe('FirestoreKeyStore', () => { }); it('can delete a single key', async () => { - const keyStore = FirestoreKeyStore.create(); + const keyStore = await FirestoreKeyStore.create(); await keyStore.removeKeys(['123']); expect(firestoreMock.collection).toBeCalledWith('sessions'); @@ -128,7 +128,7 @@ describe('FirestoreKeyStore', () => { }); it('can delete a multiple keys', async () => { - const keyStore = FirestoreKeyStore.create(); + const keyStore = await FirestoreKeyStore.create(); await keyStore.removeKeys(['123', '456']); expect(firestoreMock.collection).toBeCalledWith('sessions'); @@ -138,7 +138,7 @@ describe('FirestoreKeyStore', () => { }); it('can list keys', async () => { - const keyStore = FirestoreKeyStore.create(); + const keyStore = await FirestoreKeyStore.create(); const items = await keyStore.listKeys(); expect(firestoreMock.collection).toBeCalledWith('sessions'); diff --git a/plugins/auth-backend/src/identity/FirestoreKeyStore.ts b/plugins/auth-backend/src/identity/FirestoreKeyStore.ts index cf8c082540..74122f2849 100644 --- a/plugins/auth-backend/src/identity/FirestoreKeyStore.ts +++ b/plugins/auth-backend/src/identity/FirestoreKeyStore.ts @@ -23,7 +23,9 @@ type FirestoreSettings = Settings & { }; export class FirestoreKeyStore implements KeyStore { - static create(settings?: FirestoreSettings): FirestoreKeyStore { + static async create( + settings?: FirestoreSettings, + ): Promise { const { projectId, keyFilename, path } = settings ?? {}; const database = new Firestore({ projectId, diff --git a/plugins/auth-backend/src/identity/KeyStores.ts b/plugins/auth-backend/src/identity/KeyStores.ts index 0991f7ea7b..ec236e2074 100644 --- a/plugins/auth-backend/src/identity/KeyStores.ts +++ b/plugins/auth-backend/src/identity/KeyStores.ts @@ -63,7 +63,7 @@ export class KeyStores { } if (provider === 'firestore') { - return FirestoreKeyStore.create({ + return await FirestoreKeyStore.create({ projectId: providerConfig?.getOptionalString('projectId'), keyFilename: providerConfig?.getOptionalString('keyFilename'), path: providerConfig?.getOptionalString('path'), From 33b9694f5ce8c920daae2d9370bfb29d085ad0fc Mon Sep 17 00:00:00 2001 From: Marcus Eide Date: Thu, 14 Oct 2021 14:43:56 +0200 Subject: [PATCH 029/103] Add support for more settings Signed-off-by: Marcus Eide --- plugins/auth-backend/config.d.ts | 6 ++ .../src/identity/FirestoreKeyStore.test.ts | 79 +++++++------------ .../src/identity/FirestoreKeyStore.ts | 11 +-- .../auth-backend/src/identity/KeyStores.ts | 14 ++-- 4 files changed, 44 insertions(+), 66 deletions(-) diff --git a/plugins/auth-backend/config.d.ts b/plugins/auth-backend/config.d.ts index b15b0da628..280d92a261 100644 --- a/plugins/auth-backend/config.d.ts +++ b/plugins/auth-backend/config.d.ts @@ -35,6 +35,12 @@ export interface Config { keyStore?: { provider?: 'postgres' | 'memory' | 'firestore'; firestore?: { + /** The host to connect to */ + host?: string; + /** The port to connect to */ + port?: number; + /** Whether to use SSL when connecting. */ + ssl?: boolean; /** The Google Cloud Project ID */ projectId?: string; /** diff --git a/plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts b/plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts index a12e2ed40e..398e3c2b34 100644 --- a/plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts +++ b/plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts @@ -16,7 +16,10 @@ import { Firestore } from '@google-cloud/firestore'; -import { FirestoreKeyStore } from './FirestoreKeyStore'; +import { + FirestoreKeyStore, + FirestoreKeyStoreSettings, +} from './FirestoreKeyStore'; import { AnyJWK } from './types'; const data = jest.fn().mockReturnValue('data'); @@ -37,7 +40,6 @@ jest.mock('@google-cloud/firestore', () => ({ })); describe('FirestoreKeyStore', () => { - const OLD_ENV = process.env; const key = { kid: '123', use: 'sig', @@ -45,72 +47,45 @@ describe('FirestoreKeyStore', () => { alg: 'Base64', } as AnyJWK; + const settings = { + projectId: 'my-project', + host: 'my-host', + port: 8080, + ssl: false, + keyFilename: 'cred.json', + }; + const path = 'my-path'; + const firestoreSettings = { ...settings, path } as FirestoreKeyStoreSettings; + beforeEach(() => { jest.clearAllMocks(); }); - afterAll(() => { - process.env = OLD_ENV; - }); - it('can create an instance without settings', async () => { const keyStore = await FirestoreKeyStore.create(); expect(keyStore).toBeInstanceOf(FirestoreKeyStore); + expect(Firestore).toHaveBeenCalledWith({}); }); - it('can set the project id', async () => { - await FirestoreKeyStore.create({ projectId: 'my-project' }); + it('can create an instance with settings', async () => { + await FirestoreKeyStore.create(firestoreSettings); - expect(Firestore).toHaveBeenCalledWith( - expect.objectContaining({ - projectId: 'my-project', - }), - ); + expect(Firestore).toHaveBeenCalledWith(settings); }); - it('can handle keyfile file', async () => { - await FirestoreKeyStore.create({ keyFilename: 'keyFile.json' }); - - expect(Firestore).toHaveBeenCalledWith( - expect.objectContaining({ - keyFilename: 'keyFile.json', - }), - ); - }); - - it('can use default google credentials', async () => { - process.env.GOOGLE_APPLICATION_CREDENTIALS = 'cred.json'; - await FirestoreKeyStore.create(); - - expect(Firestore).toHaveBeenCalledWith( - expect.objectContaining({ - keyFilename: 'cred.json', - }), - ); - }); - - it('can uses a default path', async () => { + it('can use a default path', async () => { const keyStore = await FirestoreKeyStore.create(); await keyStore.addKey(key); expect(firestoreMock.collection).toBeCalledWith('sessions'); }); - it('can set the path', async () => { - const keyStore = await FirestoreKeyStore.create({ - path: 'my-path', - }); - await keyStore.addKey(key); - - expect(firestoreMock.collection).toBeCalledWith('my-path'); - }); - it('can add keys', async () => { - const keyStore = await FirestoreKeyStore.create(); + const keyStore = await FirestoreKeyStore.create(firestoreSettings); await keyStore.addKey(key); - expect(firestoreMock.collection).toBeCalledWith('sessions'); + expect(firestoreMock.collection).toBeCalledWith(path); expect(firestoreMock.doc).toBeCalledWith(key.kid); expect(firestoreMock.set).toHaveBeenCalledWith({ kid: key.kid, @@ -119,29 +94,29 @@ describe('FirestoreKeyStore', () => { }); it('can delete a single key', async () => { - const keyStore = await FirestoreKeyStore.create(); + const keyStore = await FirestoreKeyStore.create(firestoreSettings); await keyStore.removeKeys(['123']); - expect(firestoreMock.collection).toBeCalledWith('sessions'); + expect(firestoreMock.collection).toBeCalledWith(path); expect(firestoreMock.doc).toBeCalledWith('123'); expect(firestoreMock.delete).toBeCalledTimes(1); }); it('can delete a multiple keys', async () => { - const keyStore = await FirestoreKeyStore.create(); + const keyStore = await FirestoreKeyStore.create(firestoreSettings); await keyStore.removeKeys(['123', '456']); - expect(firestoreMock.collection).toBeCalledWith('sessions'); + expect(firestoreMock.collection).toBeCalledWith(path); expect(firestoreMock.doc).toBeCalledWith('123'); expect(firestoreMock.doc).toBeCalledWith('456'); expect(firestoreMock.delete).toBeCalledTimes(2); }); it('can list keys', async () => { - const keyStore = await FirestoreKeyStore.create(); + const keyStore = await FirestoreKeyStore.create(firestoreSettings); const items = await keyStore.listKeys(); - expect(firestoreMock.collection).toBeCalledWith('sessions'); + expect(firestoreMock.collection).toBeCalledWith(path); expect(firestoreMock.get).toBeCalledTimes(1); expect(data).toBeCalledTimes(1); expect(toDate).toBeCalledTimes(1); diff --git a/plugins/auth-backend/src/identity/FirestoreKeyStore.ts b/plugins/auth-backend/src/identity/FirestoreKeyStore.ts index 74122f2849..f606bacd38 100644 --- a/plugins/auth-backend/src/identity/FirestoreKeyStore.ts +++ b/plugins/auth-backend/src/identity/FirestoreKeyStore.ts @@ -18,19 +18,16 @@ import { Firestore, Settings } from '@google-cloud/firestore'; import { AnyJWK, KeyStore, StoredKey } from './types'; -type FirestoreSettings = Settings & { +export type FirestoreKeyStoreSettings = Settings & { path?: string; }; export class FirestoreKeyStore implements KeyStore { static async create( - settings?: FirestoreSettings, + settings?: FirestoreKeyStoreSettings, ): Promise { - const { projectId, keyFilename, path } = settings ?? {}; - const database = new Firestore({ - projectId, - keyFilename: keyFilename ?? process.env.GOOGLE_APPLICATION_CREDENTIALS, - }); + const { path, ...firestoreSettings } = settings ?? {}; + const database = new Firestore(firestoreSettings); return new FirestoreKeyStore(database, path ?? 'sessions'); } diff --git a/plugins/auth-backend/src/identity/KeyStores.ts b/plugins/auth-backend/src/identity/KeyStores.ts index ec236e2074..1b19bde911 100644 --- a/plugins/auth-backend/src/identity/KeyStores.ts +++ b/plugins/auth-backend/src/identity/KeyStores.ts @@ -21,7 +21,10 @@ import { Config } from '@backstage/config'; import { DatabaseKeyStore } from './DatabaseKeyStore'; import { MemoryKeyStore } from './MemoryKeyStore'; -import { FirestoreKeyStore } from './FirestoreKeyStore'; +import { + FirestoreKeyStore, + FirestoreKeyStoreSettings, +} from './FirestoreKeyStore'; import { KeyStore } from './types'; type Options = { @@ -44,7 +47,6 @@ export class KeyStores { const ks = config.getOptionalConfig('auth.keyStore'); const provider = ks?.getOptionalString('provider') ?? 'postgres'; - const providerConfig = ks?.getOptionalConfig(provider); logger?.info(`Configuring "${provider}" as KeyStore provider`); @@ -63,11 +65,9 @@ export class KeyStores { } if (provider === 'firestore') { - return await FirestoreKeyStore.create({ - projectId: providerConfig?.getOptionalString('projectId'), - keyFilename: providerConfig?.getOptionalString('keyFilename'), - path: providerConfig?.getOptionalString('path'), - }); + const settings = ks?.getOptional(provider) as FirestoreKeyStoreSettings; + + return await FirestoreKeyStore.create(settings); } throw new Error(`Unknown KeyStore provider: ${provider}`); From fc5150de4152cd0157cd6f99885c5748b7cb7a3a Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 14 Oct 2021 14:46:17 +0200 Subject: [PATCH 030/103] Fix typo Signed-off-by: Oliver Sand --- docs/integrations/azure/org.md | 2 +- .../src/processors/MicrosoftGraphOrgEntityProvider.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/integrations/azure/org.md b/docs/integrations/azure/org.md index c359960f4d..8e17bb2cc1 100644 --- a/docs/integrations/azure/org.md +++ b/docs/integrations/azure/org.md @@ -7,7 +7,7 @@ description: Importing users and groups from a Microsoft Azure Active Directory --- The Backstage catalog can be set up to ingest organizational data - users and -teams - directly from an tenant in Microsoft Azure Active Directory via the +teams - directly from a tenant in Microsoft Azure Active Directory via the Microsoft Graph API. More details on this are available in the diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts index 47f9167fbc..55b0933b99 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -167,12 +167,12 @@ function trackProgress(logger: Logger) { // Makes sure that emitted entities have a proper location based on their uuid function withLocations(providerId: string, entity: Entity): Entity { - const dn = + const uuid = entity.metadata.annotations?.[MICROSOFT_GRAPH_USER_ID_ANNOTATION] || entity.metadata.annotations?.[MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] || entity.metadata.annotations?.[MICROSOFT_GRAPH_TENANT_ID_ANNOTATION] || entity.metadata.name; - const location = `msgraph:${providerId}/${encodeURIComponent(dn)}`; + const location = `msgraph:${providerId}/${encodeURIComponent(uuid)}`; return merge( { metadata: { From 3ba87f514e224393c94cd71f9d8ed674fc3b206f Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Fri, 15 Oct 2021 11:16:21 +0200 Subject: [PATCH 031/103] Add `GitHubOrgEntityProvider` Signed-off-by: Oliver Sand --- .changeset/sharp-crabs-stare.md | 5 + plugins/catalog-backend/api-report.md | 28 +++ .../catalog-backend/src/ingestion/index.ts | 11 +- .../GithubOrgReaderProcessor.test.ts | 21 +- .../processors/GithubOrgReaderProcessor.ts | 23 +-- .../src/ingestion/processors/github/index.ts | 3 +- .../ingestion/processors/github/util.test.ts | 29 +++ .../src/ingestion/processors/github/util.ts | 25 +++ .../providers/GitHubOrgEntityProvider.test.ts | 83 ++++++++ .../providers/GitHubOrgEntityProvider.ts | 187 ++++++++++++++++++ .../src/ingestion/providers/index.ts | 17 ++ 11 files changed, 394 insertions(+), 38 deletions(-) create mode 100644 .changeset/sharp-crabs-stare.md create mode 100644 plugins/catalog-backend/src/ingestion/processors/github/util.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/github/util.ts create mode 100644 plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts create mode 100644 plugins/catalog-backend/src/ingestion/providers/index.ts diff --git a/.changeset/sharp-crabs-stare.md b/.changeset/sharp-crabs-stare.md new file mode 100644 index 0000000000..a81e83686f --- /dev/null +++ b/.changeset/sharp-crabs-stare.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Add a `GitHubOrgEntityProvider` that can be used instead of the `GithubOrgReaderProcessor`. diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 6b7d966b28..fadc2993ff 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -16,6 +16,7 @@ import { EntityName } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; import { EntityRelationSpec } from '@backstage/catalog-model'; import express from 'express'; +import { GitHubIntegrationConfig } from '@backstage/integration'; import { IndexableDocument } from '@backstage/search-common'; import { JsonObject } from '@backstage/config'; import { JsonValue } from '@backstage/config'; @@ -1019,6 +1020,33 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { ): Promise; } +// Warning: (ae-missing-release-tag) "GitHubOrgEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class GitHubOrgEntityProvider implements EntityProvider { + constructor(options: { + id: string; + orgUrl: string; + gitHubConfig: GitHubIntegrationConfig; + logger: Logger_2; + }); + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + config: Config, + options: { + id: string; + orgUrl: string; + logger: Logger_2; + }, + ): GitHubOrgEntityProvider; + // (undocumented) + getProviderName(): string; + // (undocumented) + read(): Promise; +} + // Warning: (ae-missing-release-tag) "GithubOrgReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index 33c81c56cc..fcb634914b 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -14,14 +14,15 @@ * limitations under the License. */ -export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules'; export { DefaultCatalogRulesEnforcer } from './CatalogRules'; +export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules'; +export * from './processors'; +export * from './providers'; export type { - AnalyzeLocationRequest, - AnalyzeLocationResponse, - LocationAnalyzer, AnalyzeLocationEntityField, AnalyzeLocationExistingEntity, AnalyzeLocationGenerateEntity, + AnalyzeLocationRequest, + AnalyzeLocationResponse, + LocationAnalyzer, } from './types'; -export * from './processors'; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts index 8894482484..c05d3c03ab 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts @@ -16,24 +16,15 @@ jest.mock('@octokit/graphql'); import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; -import { - ScmIntegrations, - GithubCredentialsProvider, -} from '@backstage/integration'; -import { GithubOrgReaderProcessor, parseUrl } from './GithubOrgReaderProcessor'; -import { graphql } from '@octokit/graphql'; import { ConfigReader } from '@backstage/config'; +import { + GithubCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; +import { graphql } from '@octokit/graphql'; +import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; describe('GithubOrgReaderProcessor', () => { - describe('parseUrl', () => { - it('only supports clean org urls, and decodes them', () => { - expect(() => parseUrl('https://github.com')).toThrow(); - expect(() => parseUrl('https://github.com/org/foo')).toThrow(); - expect(() => parseUrl('https://github.com/org/foo/teams')).toThrow(); - expect(parseUrl('https://github.com/foo%32')).toEqual({ org: 'foo2' }); - }); - }); - describe('implementation', () => { const logger = getVoidLogger(); const integrations = ScmIntegrations.fromConfig( diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 3a82ff6516..9b102601ca 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -23,7 +23,11 @@ import { } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; -import { getOrganizationTeams, getOrganizationUsers } from './github'; +import { + getOrganizationTeams, + getOrganizationUsers, + parseGitHubOrgUrl, +} from './github'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; import { buildOrgHierarchy } from './util/org'; @@ -61,7 +65,7 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { } const { client, tokenType } = await this.createClient(location.target); - const { org } = parseUrl(location.target); + const { org } = parseGitHubOrgUrl(location.target); // Read out all of the raw data const startTimestamp = Date.now(); @@ -126,18 +130,3 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { return { client, tokenType }; } } - -/* - * Helpers - */ - -export function parseUrl(urlString: string): { org: string } { - const path = new URL(urlString).pathname.substr(1).split('/'); - - // /backstage - if (path.length === 1 && path[0].length) { - return { org: decodeURIComponent(path[0]) }; - } - - throw new Error(`Expected a URL pointing to /`); -} diff --git a/plugins/catalog-backend/src/ingestion/processors/github/index.ts b/plugins/catalog-backend/src/ingestion/processors/github/index.ts index 7c14e21108..3d9e881ddd 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/index.ts @@ -17,7 +17,8 @@ export { readGithubConfig, readGithubMultiOrgConfig } from './config'; export type { GithubMultiOrgConfig, ProviderConfig } from './config'; export { + getOrganizationRepositories, getOrganizationTeams, getOrganizationUsers, - getOrganizationRepositories, } from './github'; +export { parseGitHubOrgUrl } from './util'; diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts new file mode 100644 index 0000000000..02f54e0675 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { parseGitHubOrgUrl } from './util'; + +describe('parseGitHubOrgUrl', () => { + it('only supports clean org urls, and decodes them', () => { + expect(() => parseGitHubOrgUrl('https://github.com')).toThrow(); + expect(() => parseGitHubOrgUrl('https://github.com/org/foo')).toThrow(); + expect(() => + parseGitHubOrgUrl('https://github.com/org/foo/teams'), + ).toThrow(); + expect(parseGitHubOrgUrl('https://github.com/foo%32')).toEqual({ + org: 'foo2', + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.ts b/plugins/catalog-backend/src/ingestion/processors/github/util.ts new file mode 100644 index 0000000000..d8df376038 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/github/util.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export function parseGitHubOrgUrl(urlString: string): { org: string } { + const path = new URL(urlString).pathname.substr(1).split('/'); + + // /backstage + if (path.length === 1 && path[0].length) { + return { org: decodeURIComponent(path[0]) }; + } + + throw new Error(`Expected a URL pointing to /`); +} diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts new file mode 100644 index 0000000000..955116f457 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts @@ -0,0 +1,83 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { withLocations } from './GitHubOrgEntityProvider'; + +describe('GitHubOrgEntityProvider', () => { + describe('withLocations', () => { + it('should set location for user', () => { + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + }, + spec: { + memberOf: [], + }, + }; + + expect(withLocations('https://github.com', 'backstage', entity)).toEqual({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + }, + }, + spec: { + memberOf: [], + }, + }); + }); + + it('should set location for group', () => { + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'mygroup', + }, + spec: { + type: 'team', + children: [], + }, + }; + + expect(withLocations('https://github.com', 'backstage', entity)).toEqual({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'mygroup', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/mygroup', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/mygroup', + }, + }, + spec: { + type: 'team', + children: [], + }, + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts new file mode 100644 index 0000000000..c2b38142ea --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts @@ -0,0 +1,187 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Entity, + LOCATION_ANNOTATION, + ORIGIN_LOCATION_ANNOTATION, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + GithubCredentialsProvider, + GitHubIntegrationConfig, + ScmIntegrations, +} from '@backstage/integration'; +import { graphql } from '@octokit/graphql'; +import { merge } from 'lodash'; +import { Logger } from 'winston'; +import { + getOrganizationTeams, + getOrganizationUsers, + parseGitHubOrgUrl, +} from '../processors/github'; +import { buildOrgHierarchy } from '../processors/util/org'; +import { + EntityProvider, + EntityProviderConnection, +} from '../../providers/types'; + +// TODO: Consider supporting an (optional) webhook that reacts on org changes + +export class GitHubOrgEntityProvider implements EntityProvider { + private connection?: EntityProviderConnection; + + static fromConfig( + config: Config, + options: { id: string; orgUrl: string; logger: Logger }, + ) { + const integrations = ScmIntegrations.fromConfig(config); + const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config; + + if (!gitHubConfig) { + throw new Error( + `There is no GitHub Org provider that matches ${options.orgUrl}. Please add a configuration for an integration.`, + ); + } + + const logger = options.logger.child({ + target: options.orgUrl, + }); + + return new GitHubOrgEntityProvider({ + id: options.id, + orgUrl: options.orgUrl, + logger, + gitHubConfig, + }); + } + + constructor( + private options: { + id: string; + orgUrl: string; + gitHubConfig: GitHubIntegrationConfig; + logger: Logger; + }, + ) {} + + getProviderName() { + return `GitHubOrgEntityProvider:${this.options.id}`; + } + + async connect(connection: EntityProviderConnection) { + this.connection = connection; + } + + async read() { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const { markReadComplete } = trackProgress(this.options.logger); + + const credentialsProvider = GithubCredentialsProvider.create( + this.options.gitHubConfig, + ); + const { headers, type: tokenType } = + await credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { org } = parseGitHubOrgUrl(this.options.orgUrl); + const { users } = await getOrganizationUsers(client, org, tokenType); + const { groups, groupMemberUsers } = await getOrganizationTeams( + client, + org, + ); + // Fill out the hierarchy + const usersByName = new Map(users.map(u => [u.metadata.name, u])); + for (const [groupName, userNames] of groupMemberUsers.entries()) { + for (const userName of userNames) { + const user = usersByName.get(userName); + if (user && !user.spec.memberOf.includes(groupName)) { + user.spec.memberOf.push(groupName); + } + } + } + buildOrgHierarchy(groups); + + const { markCommitComplete } = markReadComplete({ users, groups }); + + await this.connection.applyMutation({ + type: 'full', + entities: [...users, ...groups].map(entity => ({ + locationKey: `github-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + org, + entity, + ), + })), + }); + + markCommitComplete(); + } +} + +// Helps wrap the timing and logging behaviors +function trackProgress(logger: Logger) { + let timestamp = Date.now(); + let summary: string; + + logger.info('Reading GitHub users and groups'); + + function markReadComplete(read: { users: unknown[]; groups: unknown[] }) { + summary = `${read.users.length} GitHub users and ${read.groups.length} GitHub groups`; + const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + timestamp = Date.now(); + logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`); + return { markCommitComplete }; + } + + function markCommitComplete() { + const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + logger.info(`Committed ${summary} in ${commitDuration} seconds.`); + } + + return { markReadComplete }; +} + +// Makes sure that emitted entities have a proper location +export function withLocations( + baseUrl: string, + org: string, + entity: Entity, +): Entity { + const location = + entity.kind === 'Group' + ? `url:${baseUrl}/orgs/${org}/teams/${entity.metadata.name}` + : `url:${baseUrl}/${entity.metadata.name}`; + return merge( + { + metadata: { + annotations: { + [LOCATION_ANNOTATION]: location, + [ORIGIN_LOCATION_ANNOTATION]: location, + }, + }, + }, + entity, + ) as Entity; +} diff --git a/plugins/catalog-backend/src/ingestion/providers/index.ts b/plugins/catalog-backend/src/ingestion/providers/index.ts new file mode 100644 index 0000000000..cd1bc5cb36 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/providers/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; From 177401b571501c576fa5ae9f8c1380821fb524eb Mon Sep 17 00:00:00 2001 From: Phil Kuang Date: Fri, 15 Oct 2021 14:44:50 -0400 Subject: [PATCH 032/103] refactor(search): display entity title for result items if defined Signed-off-by: Phil Kuang --- .changeset/four-days-sneeze.md | 6 +++++ .changeset/twelve-candles-mix.md | 5 ++++ .../src/search/DefaultCatalogCollator.test.ts | 23 +++++++++++++++++++ .../src/search/DefaultCatalogCollator.ts | 2 +- .../search/DefaultTechDocsCollator.test.ts | 3 +++ .../src/search/DefaultTechDocsCollator.ts | 2 ++ .../DocsResultListItem.test.tsx | 21 +++++++++++++++++ .../DocsResultListItem/DocsResultListItem.tsx | 6 ++++- 8 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 .changeset/four-days-sneeze.md create mode 100644 .changeset/twelve-candles-mix.md diff --git a/.changeset/four-days-sneeze.md b/.changeset/four-days-sneeze.md new file mode 100644 index 0000000000..93517d3a73 --- /dev/null +++ b/.changeset/four-days-sneeze.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs-backend': patch +--- + +Display entity title (if defined) in titles of TechDocs search results diff --git a/.changeset/twelve-candles-mix.md b/.changeset/twelve-candles-mix.md new file mode 100644 index 0000000000..3218b2997c --- /dev/null +++ b/.changeset/twelve-candles-mix.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Use entity title (if defined) as title of documents indexed by `DefaultCatalogCollator` diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.test.ts index bb556fc49f..20e141b921 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollator.test.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.test.ts @@ -37,6 +37,20 @@ const expectedEntities: Entity[] = [ owner: 'someone', }, }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + title: 'Test Entity', + name: 'test-entity-2', + description: 'The expected description 2', + }, + spec: { + type: 'some-type', + lifecycle: 'experimental', + owner: 'someone', + }, + }, ]; describe('DefaultCatalogCollator', () => { @@ -89,6 +103,15 @@ describe('DefaultCatalogCollator', () => { lifecycle: expectedEntities[0]!.spec!.lifecycle, owner: expectedEntities[0]!.spec!.owner, }); + expect(documents[1]).toMatchObject({ + title: expectedEntities[1].metadata.title, + location: '/catalog/default/component/test-entity-2', + text: expectedEntities[1].metadata.description, + namespace: 'default', + componentType: expectedEntities[1]!.spec!.type, + lifecycle: expectedEntities[1]!.spec!.lifecycle, + owner: expectedEntities[1]!.spec!.owner, + }); }); it('maps a returned entity with a custom locationTemplate', async () => { diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts index 6292b2a833..a28a7645c4 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts @@ -87,7 +87,7 @@ export class DefaultCatalogCollator implements DocumentCollator { }); return response.items.map((entity: Entity): CatalogEntityDocument => { return { - title: entity.metadata.name, + title: entity.metadata.title ?? entity.metadata.name, location: this.applyArgsToFormat(this.locationTemplate, { namespace: entity.metadata.namespace || 'default', kind: entity.kind, diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts index 0df22bee53..006cfdd073 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts @@ -58,6 +58,7 @@ const expectedEntities: Entity[] = [ apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { + title: 'Test Entity with Docs!', name: 'test-entity-with-docs', description: 'Documented description', annotations: { @@ -133,6 +134,7 @@ describe('DefaultTechDocsCollator with legacyPathCasing configuration', () => { location: `/docs/default/Component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`, text: mockSearchDocIndex.docs[idx].text, namespace: 'default', + entityTitle: entity!.metadata.title, componentType: entity!.spec!.type, lifecycle: entity!.spec!.lifecycle, owner: '', @@ -177,6 +179,7 @@ describe('DefaultTechDocsCollator', () => { location: `/docs/default/component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`, text: mockSearchDocIndex.docs[idx].text, namespace: 'default', + entityTitle: entity!.metadata.title, componentType: entity!.spec!.type, lifecycle: entity!.spec!.lifecycle, owner: '', diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts index cb7d40206b..c0a96b7862 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts @@ -93,6 +93,7 @@ export class DefaultTechDocsCollator implements DocumentCollator { 'namespace', 'metadata.annotations', 'metadata.name', + 'metadata.title', 'metadata.namespace', 'spec.type', 'spec.lifecycle', @@ -130,6 +131,7 @@ export class DefaultTechDocsCollator implements DocumentCollator { }), path: doc.location, ...entityInfo, + entityTitle: entity.metadata.title, componentType: entity.spec?.type?.toString() || 'other', lifecycle: (entity.spec?.lifecycle as string) || '', owner: diff --git a/plugins/techdocs/src/components/DocsResultListItem/DocsResultListItem.test.tsx b/plugins/techdocs/src/components/DocsResultListItem/DocsResultListItem.test.tsx index 22dc86f440..807914c5ad 100644 --- a/plugins/techdocs/src/components/DocsResultListItem/DocsResultListItem.test.tsx +++ b/plugins/techdocs/src/components/DocsResultListItem/DocsResultListItem.test.tsx @@ -33,6 +33,17 @@ const validResult = { lifecycle: 'production', }; +const validResultWithTitle = { + location: 'https://backstage.io/docs', + title: 'Documentation', + text: 'Backstage is an open-source developer portal that puts the developer experience first.', + kind: 'library', + namespace: '', + name: 'Backstage', + entityTitle: 'Backstage App', + lifecycle: 'production', +}; + describe('DocsResultListItem test', () => { it('should render search doc passed in', async () => { const { findByText } = render(); @@ -59,4 +70,14 @@ describe('DocsResultListItem test', () => { ), ).toBeInTheDocument(); }); + + it('should use entity title if defined', async () => { + const { findByText } = render( + , + ); + + expect( + await findByText('Documentation | Backstage App docs'), + ).toBeInTheDocument(); + }); }); diff --git a/plugins/techdocs/src/components/DocsResultListItem/DocsResultListItem.tsx b/plugins/techdocs/src/components/DocsResultListItem/DocsResultListItem.tsx index 54e6110b88..03cf011c32 100644 --- a/plugins/techdocs/src/components/DocsResultListItem/DocsResultListItem.tsx +++ b/plugins/techdocs/src/components/DocsResultListItem/DocsResultListItem.tsx @@ -47,7 +47,11 @@ export const DocsResultListItem = ({ Date: Fri, 15 Oct 2021 16:08:06 -0400 Subject: [PATCH 033/103] fix(catalog:register): support optional locations Signed-off-by: Phil Kuang --- .changeset/fair-jeans-heal.md | 5 ++ .../actions/builtin/catalog/register.test.ts | 38 ++++++++++++++ .../actions/builtin/catalog/register.ts | 52 +++++++++++++------ 3 files changed, 79 insertions(+), 16 deletions(-) create mode 100644 .changeset/fair-jeans-heal.md diff --git a/.changeset/fair-jeans-heal.md b/.changeset/fair-jeans-heal.md new file mode 100644 index 0000000000..48c0adbfc2 --- /dev/null +++ b/.changeset/fair-jeans-heal.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Allow `catalog:register` action to register optional locations diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts index eb704c729e..e1f8823487 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts @@ -174,4 +174,42 @@ describe('catalog:register', () => { 'http://foo/var', ); }); + + it('should ignore failures when dry running the location in the catalog if `optional` is set', async () => { + addLocation + .mockResolvedValueOnce({ + entities: [], + }) + .mockRejectedValueOnce(new Error('Not found')); + await action.handler({ + ...mockContext, + input: { + catalogInfoUrl: 'http://foo/var', + optional: true, + }, + }); + + expect(addLocation).toHaveBeenNthCalledWith( + 1, + { + type: 'url', + target: 'http://foo/var', + }, + {}, + ); + expect(addLocation).toHaveBeenNthCalledWith( + 2, + { + dryRun: true, + type: 'url', + target: 'http://foo/var', + }, + {}, + ); + + expect(mockContext.output).toBeCalledWith( + 'catalogInfoUrl', + 'http://foo/var', + ); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts index 9d4961bfd9..0c08131925 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts @@ -27,8 +27,8 @@ export function createCatalogRegisterAction(options: { const { catalogClient, integrations } = options; return createTemplateAction< - | { catalogInfoUrl: string } - | { repoContentsUrl: string; catalogInfoPath?: string } + | { catalogInfoUrl: string; optional?: boolean } + | { repoContentsUrl: string; catalogInfoPath?: string; optional?: boolean } >({ id: 'catalog:register', description: @@ -46,6 +46,12 @@ export function createCatalogRegisterAction(options: { 'An absolute URL pointing to the catalog info file location', type: 'string', }, + optional: { + title: 'Optional', + description: + 'Permit the registered location to optionally exist. Default: false', + type: 'boolean', + }, }, }, { @@ -64,6 +70,12 @@ export function createCatalogRegisterAction(options: { 'A relative path from the repo root pointing to the catalog info file, defaults to /catalog-info.yaml', type: 'string', }, + optional: { + title: 'Optional', + description: + 'Permit the registered location to optionally exist. Default: false', + type: 'boolean', + }, }, }, ], @@ -100,22 +112,30 @@ export function createCatalogRegisterAction(options: { }, ctx.token ? { token: ctx.token } : {}, ); - const result = await catalogClient.addLocation( - { - dryRun: true, - type: 'url', - target: catalogInfoUrl, - }, - ctx.token ? { token: ctx.token } : {}, - ); - if (result.entities.length > 0) { - const { entities } = result; - const entity = - entities.find(e => !e.metadata.name.startsWith('generated-')) ?? - entities[0]; - ctx.output('entityRef', stringifyEntityRef(entity)); + try { + const result = await catalogClient.addLocation( + { + dryRun: true, + type: 'url', + target: catalogInfoUrl, + }, + ctx.token ? { token: ctx.token } : {}, + ); + + if (result.entities.length > 0) { + const { entities } = result; + const entity = + entities.find(e => !e.metadata.name.startsWith('generated-')) ?? + entities[0]; + ctx.output('entityRef', stringifyEntityRef(entity)); + } + } catch (e) { + if (!input.optional) { + throw e; + } } + ctx.output('catalogInfoUrl', catalogInfoUrl); }, }); From 8a92dc3155001bec1ba9d2f8358e0069aab46841 Mon Sep 17 00:00:00 2001 From: Marley Powell Date: Fri, 15 Oct 2021 21:38:20 +0100 Subject: [PATCH 034/103] refactor(`@backstage/plugin-azure-devops-backend`): re-exported types from `azure-devops-node-api` rexported types from `azure-devops-node-api` and consume those re-exported in `@backstage/plugin-azure-devops-frontend`. Signed-off-by: Marley Powell --- .../src/api/AzureDevOpsApi.ts | 73 ++++++++++--------- plugins/azure-devops-backend/src/api/index.ts | 1 + plugins/azure-devops-backend/src/api/types.ts | 17 ++++- plugins/azure-devops-backend/src/index.ts | 2 +- .../src/service/router.ts | 14 ++-- plugins/azure-devops/package.json | 2 +- plugins/azure-devops/src/api/types.ts | 2 +- .../src/components/BuildTable/BuildTable.tsx | 43 +++++------ .../azure-devops/src/hooks/useRepoBuilds.ts | 21 +++--- 9 files changed, 100 insertions(+), 75 deletions(-) diff --git a/plugins/azure-devops-backend/src/api/AzureDevOpsApi.ts b/plugins/azure-devops-backend/src/api/AzureDevOpsApi.ts index 89e32aa2f3..1a80a3d42b 100644 --- a/plugins/azure-devops-backend/src/api/AzureDevOpsApi.ts +++ b/plugins/azure-devops-backend/src/api/AzureDevOpsApi.ts @@ -14,42 +14,47 @@ * limitations under the License. */ -import { Logger } from 'winston'; -import { WebApi } from 'azure-devops-node-api'; import { Build, BuildResult, BuildStatus, -} from 'azure-devops-node-api/interfaces/BuildInterfaces'; -import { GitPullRequest, GitPullRequestSearchCriteria, -} from 'azure-devops-node-api/interfaces/GitInterfaces'; -import { PullRequest, PullRequestOptions, RepoBuild } from './types'; + GitRepository, + PullRequest, + PullRequestOptions, + RepoBuild, +} from './types'; + +import { Logger } from 'winston'; +import { WebApi } from 'azure-devops-node-api'; export class AzureDevOpsApi { - constructor( + public constructor( private readonly logger: Logger, private readonly webApi: WebApi, ) {} - async getGitRepository(projectName: string, repoName: string) { - if (this.logger) { - this.logger.debug( - `Calling Azure DevOps REST API, getting Repository ${repoName} for Project ${projectName}`, - ); - } + public async getGitRepository( + projectName: string, + repoName: string, + ): Promise { + this.logger?.debug( + `Calling Azure DevOps REST API, getting Repository ${repoName} for Project ${projectName}`, + ); const client = await this.webApi.getGitApi(); return client.getRepository(repoName, projectName); } - async getBuildList(projectName: string, repoId: string, top: number) { - if (this.logger) { - this.logger.debug( - `Calling Azure DevOps REST API, getting up to ${top} Builds for Repository Id ${repoId} for Project ${projectName}`, - ); - } + public async getBuildList( + projectName: string, + repoId: string, + top: number, + ): Promise { + this.logger?.debug( + `Calling Azure DevOps REST API, getting up to ${top} Builds for Repository Id ${repoId} for Project ${projectName}`, + ); const client = await this.webApi.getBuildApi(); return client.getBuilds( @@ -77,12 +82,14 @@ export class AzureDevOpsApi { ); } - async getRepoBuilds(projectName: string, repoName: string, top: number) { - if (this.logger) { - this.logger.debug( - `Calling Azure DevOps REST API, getting up to ${top} Builds for Repository ${repoName} for Project ${projectName}`, - ); - } + public async getRepoBuilds( + projectName: string, + repoName: string, + top: number, + ) { + this.logger?.debug( + `Calling Azure DevOps REST API, getting up to ${top} Builds for Repository ${repoName} for Project ${projectName}`, + ); const gitRepository = await this.getGitRepository(projectName, repoName); const buildList = await this.getBuildList( @@ -98,16 +105,14 @@ export class AzureDevOpsApi { return repoBuilds; } - async getPullRequests( + public async getPullRequests( projectName: string, repoName: string, options: PullRequestOptions, - ) { - if (this.logger) { - this.logger.debug( - `Calling Azure DevOps REST API, getting up to ${top} Pull Requests for Repository ${repoName} for Project ${projectName}`, - ); - } + ): Promise { + this.logger?.debug( + `Calling Azure DevOps REST API, getting up to ${top} Pull Requests for Repository ${repoName} for Project ${projectName}`, + ); const gitRepository = await this.getGitRepository(projectName, repoName); const client = await this.webApi.getGitApi(); @@ -133,7 +138,7 @@ export class AzureDevOpsApi { } } -export function mappedRepoBuild(build: Build) { +export function mappedRepoBuild(build: Build): RepoBuild { return { id: build.id, title: [build.definition?.name, build.buildNumber] @@ -150,7 +155,7 @@ export function mappedRepoBuild(build: Build) { export function mappedPullRequest( pullRequest: GitPullRequest, linkBaseUrl: string, -) { +): PullRequest { return { pullRequestId: pullRequest.pullRequestId, repoName: pullRequest.repository?.name, diff --git a/plugins/azure-devops-backend/src/api/index.ts b/plugins/azure-devops-backend/src/api/index.ts index 893f0f4f2c..97c48cb0bf 100644 --- a/plugins/azure-devops-backend/src/api/index.ts +++ b/plugins/azure-devops-backend/src/api/index.ts @@ -15,4 +15,5 @@ */ export { AzureDevOpsApi } from './AzureDevOpsApi'; +export { BuildResult, BuildStatus } from './types'; export type { RepoBuild, PullRequest } from './types'; diff --git a/plugins/azure-devops-backend/src/api/types.ts b/plugins/azure-devops-backend/src/api/types.ts index a53f501f26..724a65b986 100644 --- a/plugins/azure-devops-backend/src/api/types.ts +++ b/plugins/azure-devops-backend/src/api/types.ts @@ -15,10 +15,25 @@ */ import { + Build, BuildResult, BuildStatus, } from 'azure-devops-node-api/interfaces/BuildInterfaces'; -import { PullRequestStatus } from 'azure-devops-node-api/interfaces/GitInterfaces'; +import { + GitPullRequest, + GitPullRequestSearchCriteria, + GitRepository, + PullRequestStatus, +} from 'azure-devops-node-api/interfaces/GitInterfaces'; + +export { BuildResult, BuildStatus }; +export type { + Build, + GitPullRequest, + GitPullRequestSearchCriteria, + GitRepository, + PullRequestStatus, +}; export type RepoBuild = { id?: number; diff --git a/plugins/azure-devops-backend/src/index.ts b/plugins/azure-devops-backend/src/index.ts index ff4d3973bf..7a2a347b65 100644 --- a/plugins/azure-devops-backend/src/index.ts +++ b/plugins/azure-devops-backend/src/index.ts @@ -13,6 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { AzureDevOpsApi } from './api'; +export { AzureDevOpsApi, BuildResult, BuildStatus } from './api'; export type { RepoBuild, PullRequest } from './api'; export * from './service/router'; diff --git a/plugins/azure-devops-backend/src/service/router.ts b/plugins/azure-devops-backend/src/service/router.ts index 0929d5cde9..b3505eae1c 100644 --- a/plugins/azure-devops-backend/src/service/router.ts +++ b/plugins/azure-devops-backend/src/service/router.ts @@ -14,15 +14,15 @@ * limitations under the License. */ +import { PullRequestOptions, PullRequestStatus } from '../api/types'; +import { WebApi, getPersonalAccessTokenHandler } from 'azure-devops-node-api'; + +import { AzureDevOpsApi } from '../api'; +import { Config } from '@backstage/config'; +import { Logger } from 'winston'; +import Router from 'express-promise-router'; import { errorHandler } from '@backstage/backend-common'; import express from 'express'; -import Router from 'express-promise-router'; -import { Logger } from 'winston'; -import { Config } from '@backstage/config'; -import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api'; -import { AzureDevOpsApi } from '../api'; -import { PullRequestStatus } from 'azure-devops-node-api/interfaces/GitInterfaces'; -import { PullRequestOptions } from '../api/types'; const DEFAULT_TOP: number = 10; diff --git a/plugins/azure-devops/package.json b/plugins/azure-devops/package.json index c4aea032d2..f9dff7229c 100644 --- a/plugins/azure-devops/package.json +++ b/plugins/azure-devops/package.json @@ -31,12 +31,12 @@ "@backstage/core-components": "^0.7.0", "@backstage/core-plugin-api": "^0.1.10", "@backstage/errors": "^0.1.2", + "@backstage/plugin-azure-devops-backend": "^0.1.2", "@backstage/plugin-catalog-react": "^0.6.0", "@backstage/theme": "^0.2.11", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.57", - "azure-devops-node-api": "^11.0.1", "luxon": "^2.0.2", "react": "^16.13.1", "react-dom": "^16.13.1", diff --git a/plugins/azure-devops/src/api/types.ts b/plugins/azure-devops/src/api/types.ts index 1fbc34231e..46a9212aca 100644 --- a/plugins/azure-devops/src/api/types.ts +++ b/plugins/azure-devops/src/api/types.ts @@ -17,7 +17,7 @@ import { BuildResult, BuildStatus, -} from 'azure-devops-node-api/interfaces/BuildInterfaces'; +} from '@backstage/plugin-azure-devops-backend'; export type RepoBuild = { id?: number; diff --git a/plugins/azure-devops/src/components/BuildTable/BuildTable.tsx b/plugins/azure-devops/src/components/BuildTable/BuildTable.tsx index bead424ffb..983f4959b4 100644 --- a/plugins/azure-devops/src/components/BuildTable/BuildTable.tsx +++ b/plugins/azure-devops/src/components/BuildTable/BuildTable.tsx @@ -14,26 +14,27 @@ * limitations under the License. */ -import React from 'react'; -import { DateTime } from 'luxon'; -import { - Link, - Table, - TableColumn, - StatusError, - StatusOK, - StatusWarning, - StatusAborted, - StatusRunning, - StatusPending, - ResponseErrorPanel, -} from '@backstage/core-components'; import { Box, Typography } from '@material-ui/core'; -import { RepoBuild } from '../../api/types'; import { BuildResult, BuildStatus, -} from 'azure-devops-node-api/interfaces/BuildInterfaces'; +} from '@backstage/plugin-azure-devops-backend'; +import { + Link, + ResponseErrorPanel, + StatusAborted, + StatusError, + StatusOK, + StatusPending, + StatusRunning, + StatusWarning, + Table, + TableColumn, +} from '@backstage/core-components'; + +import { DateTime } from 'luxon'; +import React from 'react'; +import { RepoBuild } from '../../api/types'; const getBuildResultComponent = (result: number | undefined) => { switch (result) { @@ -154,13 +155,13 @@ const columns: TableColumn[] = [ }, ]; -type Props = { - items?: RepoBuild[]; +type BuildTableProps = { + items: RepoBuild[] | null; loading: boolean; - error?: any; + error: Error | null; }; -export const BuildTable = ({ items, loading, error }: Props) => { +export const BuildTable = ({ items, loading, error }: BuildTableProps) => { if (error) { return (
@@ -180,7 +181,7 @@ export const BuildTable = ({ items, loading, error }: Props) => { showEmptyDataSourceMessage: !loading, }} title={`Builds (${items ? items.length : 0})`} - data={items || []} + data={items ?? []} /> ); }; diff --git a/plugins/azure-devops/src/hooks/useRepoBuilds.ts b/plugins/azure-devops/src/hooks/useRepoBuilds.ts index 14a8a79428..56528f84a0 100644 --- a/plugins/azure-devops/src/hooks/useRepoBuilds.ts +++ b/plugins/azure-devops/src/hooks/useRepoBuilds.ts @@ -14,35 +14,38 @@ * limitations under the License. */ -import { useAsync } from 'react-use'; -import { Entity } from '@backstage/catalog-model'; -import { useApi } from '@backstage/core-plugin-api'; -import { azureDevOpsApiRef } from '../api'; import { RepoBuild, RepoBuildOptions } from '../api/types'; -import { useProjectRepoFromEntity } from './useProjectRepoFromEntity'; + import { AZURE_DEVOPS_DEFAULT_TOP } from '../constants'; +import { Entity } from '@backstage/catalog-model'; +import { azureDevOpsApiRef } from '../api'; +import { useApi } from '@backstage/core-plugin-api'; +import { useAsync } from 'react-use'; +import { useProjectRepoFromEntity } from './useProjectRepoFromEntity'; export function useRepoBuilds( entity: Entity, defaultLimit?: number, ): { - items: RepoBuild[] | undefined; + items: RepoBuild[] | null; loading: boolean; - error: any; + error: Error | null; } { const top = defaultLimit ?? AZURE_DEVOPS_DEFAULT_TOP; const options: RepoBuildOptions = { top: top, }; + const api = useApi(azureDevOpsApiRef); const { project, repo } = useProjectRepoFromEntity(entity); + const { value, loading, error } = useAsync(() => { return api.getRepoBuilds(project, repo, options); }, [api, project, repo, entity]); return { - items: value?.items, + items: value?.items ?? null, loading, - error, + error: error ?? null, }; } From f67dff0d2095acc9e799fb24600de6a79a9272eb Mon Sep 17 00:00:00 2001 From: Marley Powell Date: Fri, 15 Oct 2021 21:43:53 +0100 Subject: [PATCH 035/103] chore: updated changeset Signed-off-by: Marley Powell --- .changeset/nice-pillows-move.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/nice-pillows-move.md diff --git a/.changeset/nice-pillows-move.md b/.changeset/nice-pillows-move.md new file mode 100644 index 0000000000..ca534406b9 --- /dev/null +++ b/.changeset/nice-pillows-move.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-azure-devops': minor +'@backstage/plugin-azure-devops-backend': minor +--- + +Re-exported types from azure-devops-node-api in @backstage/plugin-azure-devops-backend and consume those re-exported types in @backstage/plugin-azure-devops-frontend From afb67603b390368dc8a8db4e24f45aa663f2ed9e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 5 Oct 2021 18:58:23 +0200 Subject: [PATCH 036/103] errors: added new assertion APIs Signed-off-by: Patrik Oldsberg --- packages/errors/api-report.md | 19 +++++ packages/errors/src/errors/assertion.test.ts | 79 ++++++++++++++++++++ packages/errors/src/errors/assertion.ts | 48 ++++++++++++ packages/errors/src/errors/index.ts | 2 + 4 files changed, 148 insertions(+) create mode 100644 packages/errors/src/errors/assertion.test.ts create mode 100644 packages/errors/src/errors/assertion.ts diff --git a/packages/errors/api-report.md b/packages/errors/api-report.md index 429807d9db..4290ea0db1 100644 --- a/packages/errors/api-report.md +++ b/packages/errors/api-report.md @@ -5,6 +5,11 @@ ```ts import { JsonObject } from '@backstage/config'; +// Warning: (ae-missing-release-tag) "assertError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export function assertError(val: unknown): asserts val is ErrorLike; + // @public export class AuthenticationError extends CustomErrorBase {} @@ -23,6 +28,15 @@ export function deserializeError( data: SerializedError, ): T; +// Warning: (ae-missing-release-tag) "ErrorLike" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type ErrorLike = { + name: string; + message: string; + stack?: string; +}; + // @public export type ErrorResponse = { error: SerializedError; @@ -38,6 +52,11 @@ export type ErrorResponse = { // @public export class InputError extends CustomErrorBase {} +// Warning: (ae-missing-release-tag) "isError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export function isError(val: unknown): val is ErrorLike; + // @public export class NotAllowedError extends CustomErrorBase {} diff --git a/packages/errors/src/errors/assertion.test.ts b/packages/errors/src/errors/assertion.test.ts new file mode 100644 index 0000000000..28367c2a68 --- /dev/null +++ b/packages/errors/src/errors/assertion.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { assertError, isError } from './assertion'; +import { NotFoundError } from './common'; +import { CustomErrorBase } from './CustomErrorBase'; + +const areErrors = [ + { name: 'e', message: '' }, + new Error(), + new NotFoundError(), + Object.create({ name: 'e', message: '' }), + Object.assign(Object.create({ name: 'e' }), { + get message() { + return ''; + }, + }), + new (class extends class { + message = ''; + } { + name = 'e'; + })(), + new (class SubclassError extends CustomErrorBase {})(), + new (class SubclassError extends NotFoundError {})(), +]; + +const notErrors = [ + null, + 0, + 'loller', + Symbol(), + [], + BigInt(0), + false, + true, + { name: 'e' }, + { message: '' }, + { name: '', message: 'oh no' }, + new (class {})(), +]; + +describe('assertError', () => { + it.each(areErrors)('should assert that things are errors %#', error => { + expect(assertError(error)).toBeUndefined(); + }); + + it.each(notErrors)( + 'should assert that things are not errors %#', + notError => { + expect(() => assertError(notError)).toThrow(); + }, + ); +}); + +describe('isError', () => { + it.each(areErrors)('should assert that things are errors %#', error => { + expect(isError(error)).toBe(true); + }); + + it.each(notErrors)( + 'should assert that things are not errors %#', + notError => { + expect(isError(notError)).toBe(false); + }, + ); +}); diff --git a/packages/errors/src/errors/assertion.ts b/packages/errors/src/errors/assertion.ts new file mode 100644 index 0000000000..a0b2d59c11 --- /dev/null +++ b/packages/errors/src/errors/assertion.ts @@ -0,0 +1,48 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type ErrorLike = { + name: string; + message: string; + stack?: string; +}; + +export function isError(val: unknown): val is ErrorLike { + if (typeof val !== 'object' || val === null || Array.isArray(val)) { + return false; + } + const maybe = val as Partial; + if (typeof maybe.name !== 'string' || maybe.name === '') { + return false; + } + if (typeof maybe.message !== 'string') { + return false; + } + return true; +} + +export function assertError(val: unknown): asserts val is ErrorLike { + if (typeof val !== 'object' || val === null || Array.isArray(val)) { + throw new Error(`Encountered invalid error, not an object, got '${val}'`); + } + const maybe = val as Partial; + if (typeof maybe.name !== 'string' || maybe.name === '') { + throw new Error(`Encountered error object without a name, got '${val}'`); + } + if (typeof maybe.message !== 'string') { + throw new Error(`Encountered error object without a message, got '${val}'`); + } +} diff --git a/packages/errors/src/errors/index.ts b/packages/errors/src/errors/index.ts index cc9d15b1ff..02670f4de9 100644 --- a/packages/errors/src/errors/index.ts +++ b/packages/errors/src/errors/index.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +export { assertError, isError } from './assertion'; +export type { ErrorLike } from './assertion'; export { AuthenticationError, ConflictError, From 93c537f40a6b238ef8776310ac4d8688989131e6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 5 Oct 2021 22:39:16 +0200 Subject: [PATCH 037/103] catalog-backend: use assertError Signed-off-by: Patrik Oldsberg --- .../src/ingestion/processors/UrlReaderProcessor.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index cf889eb577..6a85cc07c7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -16,6 +16,7 @@ import { UrlReader } from '@backstage/backend-common'; import { Entity, LocationSpec } from '@backstage/catalog-model'; +import { assertError } from '@backstage/errors'; import parseGitUrl from 'git-url-parse'; import limiterFactory from 'p-limit'; import { Logger } from 'winston'; @@ -91,6 +92,7 @@ export class UrlReaderProcessor implements CatalogProcessor { }); } } catch (error) { + assertError(error); const message = `Unable to read ${location.type}, ${error}`; if (error.name === 'NotModifiedError' && cacheItem) { for (const parseResult of cacheItem.value) { From 6077d61e735e7dbfda2919be090fb30e28034c76 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 7 Oct 2021 12:17:33 +0200 Subject: [PATCH 038/103] errors: added changeset for new assertion helpers Signed-off-by: Patrik Oldsberg --- .changeset/real-mice-beg.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/real-mice-beg.md diff --git a/.changeset/real-mice-beg.md b/.changeset/real-mice-beg.md new file mode 100644 index 0000000000..bcb4f0ca9a --- /dev/null +++ b/.changeset/real-mice-beg.md @@ -0,0 +1,5 @@ +--- +'@backstage/errors': patch +--- + +Two new helpers have been added that make it easier to migrate to considering thrown errors to be of the type `unknown` in TypeScript. The helpers are `assertError` and `isError`, and can be called to make sure that an unknown value conforms to the shape of an `ErrorLike` object. The `assertError` function is a type-guard that throws in the case of a mismatch, while `isError` returns false. From 9d18bc8ba46c7ed5583544f8d0ca587d37c79eb9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 7 Oct 2021 23:54:24 +0200 Subject: [PATCH 039/103] errors: added ForwardedError, unknown cause, and unknown fields in ErrorLike Signed-off-by: Patrik Oldsberg --- .changeset/real-mice-beg.md | 2 ++ packages/errors/src/errors/CustomErrorBase.ts | 23 ++++++++++++++----- packages/errors/src/errors/assertion.ts | 1 + packages/errors/src/errors/common.test.ts | 3 ++- packages/errors/src/errors/common.ts | 17 ++++++++++++++ packages/errors/src/errors/index.ts | 1 + 6 files changed, 40 insertions(+), 7 deletions(-) diff --git a/.changeset/real-mice-beg.md b/.changeset/real-mice-beg.md index bcb4f0ca9a..7dde97f6c2 100644 --- a/.changeset/real-mice-beg.md +++ b/.changeset/real-mice-beg.md @@ -3,3 +3,5 @@ --- Two new helpers have been added that make it easier to migrate to considering thrown errors to be of the type `unknown` in TypeScript. The helpers are `assertError` and `isError`, and can be called to make sure that an unknown value conforms to the shape of an `ErrorLike` object. The `assertError` function is a type-guard that throws in the case of a mismatch, while `isError` returns false. + +A new error constructor has also been added, `ForwardedError`, which can be used to add context to a forwarded error. It requires both a message and a cause, and inherits the `name` property from the `cause`. diff --git a/packages/errors/src/errors/CustomErrorBase.ts b/packages/errors/src/errors/CustomErrorBase.ts index a392135c13..4e8674b4dc 100644 --- a/packages/errors/src/errors/CustomErrorBase.ts +++ b/packages/errors/src/errors/CustomErrorBase.ts @@ -14,17 +14,28 @@ * limitations under the License. */ +import { isError } from './assertion'; + /** @public */ export class CustomErrorBase extends Error { readonly cause?: Error; - constructor(message?: string, cause?: Error) { + constructor(message?: string, cause?: Error | unknown) { + let assignedCause: Error | undefined = undefined; + let fullMessage = message; - if (cause) { - if (fullMessage) { - fullMessage += `; caused by ${cause}`; + if (cause !== undefined) { + let causeStr; + if (isError(cause)) { + assignedCause = cause; + causeStr = String(cause); } else { - fullMessage = `caused by ${cause}`; + causeStr = `unknown error '${cause}'`; + } + if (fullMessage) { + fullMessage += `; caused by ${causeStr}`; + } else { + fullMessage = `caused by ${causeStr}`; } } @@ -33,6 +44,6 @@ export class CustomErrorBase extends Error { Error.captureStackTrace?.(this, this.constructor); this.name = this.constructor.name; - this.cause = cause; + this.cause = assignedCause; } } diff --git a/packages/errors/src/errors/assertion.ts b/packages/errors/src/errors/assertion.ts index a0b2d59c11..4e8d939461 100644 --- a/packages/errors/src/errors/assertion.ts +++ b/packages/errors/src/errors/assertion.ts @@ -18,6 +18,7 @@ export type ErrorLike = { name: string; message: string; stack?: string; + [unknownKeys: string]: unknown; }; export function isError(val: unknown): val is ErrorLike { diff --git a/packages/errors/src/errors/common.test.ts b/packages/errors/src/errors/common.test.ts index 232c3e78ef..1b57be2540 100644 --- a/packages/errors/src/errors/common.test.ts +++ b/packages/errors/src/errors/common.test.ts @@ -18,7 +18,8 @@ import * as errors from './common'; describe('common', () => { it('extends Error properly', () => { - for (const [name, E] of Object.entries(errors)) { + const { ForwardedError: _, ...optionalCauseErrors } = { ...errors }; + for (const [name, E] of Object.entries(optionalCauseErrors)) { const error = new E('abcdef'); expect(error.name).toBe(name); expect(error.message).toBe('abcdef'); diff --git a/packages/errors/src/errors/common.ts b/packages/errors/src/errors/common.ts index dad236d4b5..f9300b9371 100644 --- a/packages/errors/src/errors/common.ts +++ b/packages/errors/src/errors/common.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { isError } from './assertion'; import { CustomErrorBase } from './CustomErrorBase'; /* @@ -72,3 +73,19 @@ export class ConflictError extends CustomErrorBase {} * @public */ export class NotModifiedError extends CustomErrorBase {} + +/** + * An error that forwards an underlying cause with additional context in the message. + * + * The `name` property of the error will be inherited from the `cause` if + * possible, and will otherwise be set to `'Error'`. + * + * @public + */ +export class ForwardedError extends CustomErrorBase { + constructor(message: string, cause: Error | unknown) { + super(message, cause); + + this.name = isError(cause) ? this.constructor.name : 'Error'; + } +} diff --git a/packages/errors/src/errors/index.ts b/packages/errors/src/errors/index.ts index 02670f4de9..bc2fb2aa4e 100644 --- a/packages/errors/src/errors/index.ts +++ b/packages/errors/src/errors/index.ts @@ -19,6 +19,7 @@ export type { ErrorLike } from './assertion'; export { AuthenticationError, ConflictError, + ForwardedError, InputError, NotAllowedError, NotFoundError, From 36e67d2f24027f795f277a72e710870e7308360e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 8 Oct 2021 17:41:29 +0200 Subject: [PATCH 040/103] more strict error type checking in most packages and backend plugins Signed-off-by: Patrik Oldsberg --- .changeset/tiny-panthers-jump.md | 22 ++++++++++++++ .../src/database/connectors/postgres.ts | 4 +-- .../src/reading/AwsS3UrlReader.test.ts | 4 +-- .../src/reading/AwsS3UrlReader.ts | 5 ++-- .../src/reading/integration.test.ts | 6 +++- .../src/util/DockerContainerRunner.ts | 6 ++-- packages/cli/package.json | 1 + .../commands/create-plugin/createPlugin.ts | 3 ++ packages/cli/src/commands/index.ts | 2 ++ .../commands/remove-plugin/removePlugin.ts | 8 +++++ packages/cli/src/commands/versions/bump.ts | 5 ++-- packages/cli/src/lib/run.ts | 11 +++++-- packages/config-loader/package.json | 1 + packages/config-loader/src/lib/env.ts | 2 ++ .../config-loader/src/lib/schema/collect.ts | 2 ++ .../config-loader/src/lib/transform/apply.ts | 2 ++ packages/config-loader/src/loader.ts | 5 ++-- .../DiscoveryApi/UrlPatternDiscovery.test.ts | 10 +++---- .../DiscoveryApi/UrlPatternDiscovery.ts | 29 ++++++++++--------- .../src/routing/RoutingProvider.test.tsx | 6 ++-- .../LoginRequestListItem.tsx | 13 +++------ .../src/layout/SignInPage/auth0Provider.tsx | 3 +- .../src/layout/SignInPage/commonProvider.tsx | 3 +- .../src/extensions/extensions.tsx | 18 ++++++++---- packages/create-app/src/createApp.ts | 6 ++-- packages/e2e-test/package.json | 1 + packages/e2e-test/src/lib/helpers.ts | 7 +++-- .../src/stages/generate/helpers.ts | 10 +++++-- .../src/stages/generate/techdocs.ts | 6 ++-- .../techdocs-common/src/stages/prepare/url.ts | 5 ++-- .../src/stages/publish/awsS3.test.ts | 2 +- .../src/stages/publish/awsS3.ts | 10 +++++-- .../stages/publish/azureBlobStorage.test.ts | 4 +-- .../src/stages/publish/azureBlobStorage.ts | 7 ++++- .../src/stages/publish/googleStorage.ts | 3 ++ .../src/stages/publish/local.ts | 2 ++ .../publish/migrations/GoogleMigration.ts | 2 ++ .../src/stages/publish/openStackSwift.ts | 7 ++++- .../src/lib/oauth/OAuthAdapter.ts | 12 ++++---- .../src/providers/saml/provider.ts | 9 +++--- plugins/auth-backend/src/service/router.ts | 3 +- .../catalog-backend-module-ldap/package.json | 1 + .../src/ldap/client.ts | 5 ++-- .../src/database/DefaultProcessingDatabase.ts | 7 +++-- .../processors/AwsS3DiscoveryProcessor.ts | 3 +- .../src/legacy/ingestion/LocationReaders.ts | 2 +- .../DefaultCatalogProcessingEngine.ts | 3 +- .../DefaultCatalogProcessingOrchestrator.ts | 8 ++++- .../processing/ProcessorOutputCollector.ts | 2 ++ plugins/catalog-import/package.json | 1 + .../StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx | 4 +-- .../StepPrepareCreatePullRequest.tsx | 2 ++ .../StepReviewLocation/StepReviewLocation.tsx | 2 ++ plugins/catalog-react/package.json | 1 + .../UnregisterEntityDialog.tsx | 3 ++ .../DeleteEntityDialog.tsx | 2 ++ .../src/service/CoverageUtils.test.ts | 4 +-- plugins/kubernetes-backend/package.json | 1 + .../cluster-locator/GkeClusterLocator.test.ts | 2 +- .../src/cluster-locator/GkeClusterLocator.ts | 6 ++-- .../actions/builtin/github/githubWebhook.ts | 2 ++ .../src/scaffolder/actions/builtin/helpers.ts | 2 ++ .../actions/builtin/publish/github.ts | 4 +++ .../src/scaffolder/tasks/StorageTaskBroker.ts | 2 ++ .../src/scaffolder/tasks/TaskWorker.ts | 3 ++ .../scaffolder-backend/src/service/helpers.ts | 2 ++ .../src/DocsBuilder/builder.ts | 8 +++-- .../src/service/DocsSynchronizer.ts | 3 +- 68 files changed, 248 insertions(+), 104 deletions(-) create mode 100644 .changeset/tiny-panthers-jump.md diff --git a/.changeset/tiny-panthers-jump.md b/.changeset/tiny-panthers-jump.md new file mode 100644 index 0000000000..d059b826da --- /dev/null +++ b/.changeset/tiny-panthers-jump.md @@ -0,0 +1,22 @@ +--- +'@backstage/backend-common': patch +'@backstage/cli': patch +'@backstage/config-loader': patch +'@backstage/core-app-api': patch +'@backstage/core-components': patch +'@backstage/core-plugin-api': patch +'@backstage/create-app': patch +'@backstage/techdocs-common': patch +'@backstage/plugin-auth-backend': patch +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-catalog-backend-module-ldap': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-code-coverage-backend': patch +'@backstage/plugin-kubernetes-backend': patch +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-techdocs-backend': patch +--- + +Internal updates to apply more strict checks to throw errors. diff --git a/packages/backend-common/src/database/connectors/postgres.ts b/packages/backend-common/src/database/connectors/postgres.ts index f6e42d1945..62de2e001e 100644 --- a/packages/backend-common/src/database/connectors/postgres.ts +++ b/packages/backend-common/src/database/connectors/postgres.ts @@ -17,6 +17,7 @@ import knexFactory, { Knex } from 'knex'; import { Config } from '@backstage/config'; +import { ForwardedError } from '@backstage/errors'; import { mergeDatabaseConfig } from '../config'; import { DatabaseConnector } from '../types'; import defaultNameOverride from './defaultNameOverride'; @@ -94,8 +95,7 @@ function requirePgConnectionString() { try { return require('pg-connection-string').parse; } catch (e) { - const message = `Postgres: Install 'pg-connection-string'`; - throw new Error(`${message}\n${e.message}`); + throw new ForwardedError("Postgres: Install 'pg-connection-string'", e); } } diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 0bc0e4f97f..6959edcc8b 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -167,7 +167,7 @@ describe('AwsS3UrlReader', () => { ), ).rejects.toThrow( Error( - `Could not retrieve file from S3: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + `Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, ), ); }); @@ -216,7 +216,7 @@ describe('AwsS3UrlReader', () => { ), ).rejects.toThrow( Error( - `Could not retrieve file from S3: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + `Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, ), ); }); diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index cb03ec4f71..7fd0f6ba19 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -27,6 +27,7 @@ import { } from './types'; import getRawBody from 'raw-body'; import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; +import { ForwardedError } from '@backstage/errors'; import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3'; const parseURL = ( @@ -162,7 +163,7 @@ export class AwsS3UrlReader implements UrlReader { etag: etag, }; } catch (e) { - throw new Error(`Could not retrieve file from S3: ${e.message}`); + throw new ForwardedError('Could not retrieve file from S3', e); } } @@ -203,7 +204,7 @@ export class AwsS3UrlReader implements UrlReader { return await this.deps.treeResponseFactory.fromReadableArray(responses); } catch (e) { - throw new Error(`Could not retrieve file tree from S3: ${e.message}`); + throw new ForwardedError('Could not retrieve file tree from S3', e); } } diff --git a/packages/backend-common/src/reading/integration.test.ts b/packages/backend-common/src/reading/integration.test.ts index ae557be583..951df29605 100644 --- a/packages/backend-common/src/reading/integration.test.ts +++ b/packages/backend-common/src/reading/integration.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { isError } from '@backstage/errors'; import { getVoidLogger } from '../logging'; import { UrlReaders } from './UrlReaders'; @@ -71,7 +72,10 @@ function withRetries(count: number, fn: () => Promise) { error = err; } } - if (!error.message.match(/rate limit|Too Many Requests/)) { + if ( + isError(error) && + !error.message.match(/rate limit|Too Many Requests/) + ) { throw error; } else { console.warn('Request was rate limited', error); diff --git a/packages/backend-common/src/util/DockerContainerRunner.ts b/packages/backend-common/src/util/DockerContainerRunner.ts index da3ce66ec2..b81bd995ca 100644 --- a/packages/backend-common/src/util/DockerContainerRunner.ts +++ b/packages/backend-common/src/util/DockerContainerRunner.ts @@ -16,6 +16,7 @@ import Docker from 'dockerode'; import fs from 'fs-extra'; +import { ForwardedError } from '@backstage/errors'; import { PassThrough } from 'stream'; import { ContainerRunner, RunContainerOptions } from './ContainerRunner'; @@ -45,8 +46,9 @@ export class DockerContainerRunner implements ContainerRunner { try { await this.dockerClient.ping(); } catch (e) { - throw new Error( - `This operation requires Docker. Docker does not appear to be available. Docker.ping() failed with: ${e.message}`, + throw new ForwardedError( + 'This operation requires Docker. Docker does not appear to be available. Docker.ping() failed with', + e, ); } diff --git a/packages/cli/package.json b/packages/cli/package.json index ecc3d3b4e1..ebd8f0169c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -31,6 +31,7 @@ "@backstage/cli-common": "^0.1.4", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.6.10", + "@backstage/errors": "^0.1.2", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", diff --git a/packages/cli/src/commands/create-plugin/createPlugin.ts b/packages/cli/src/commands/create-plugin/createPlugin.ts index 6ddd1f7160..a5ea565cae 100644 --- a/packages/cli/src/commands/create-plugin/createPlugin.ts +++ b/packages/cli/src/commands/create-plugin/createPlugin.ts @@ -24,6 +24,7 @@ import camelCase from 'lodash/camelCase'; import upperFirst from 'lodash/upperFirst'; import os from 'os'; import { Command } from 'commander'; +import { assertError } from '@backstage/errors'; import { parseOwnerIds, addCodeownersEntry, @@ -173,6 +174,7 @@ async function buildPlugin(pluginFolder: string) { ); }); } catch (error) { + assertError(error); Task.error(error.message); break; } @@ -329,6 +331,7 @@ export default async (cmd: Command) => { Task.log(); Task.exit(); } catch (error) { + assertError(error); Task.error(error.message); Task.log('It seems that something went wrong when creating the plugin 🤔'); diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index d6686f2655..302f76cc5d 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { assertError } from '@backstage/errors'; import { CommanderStatic } from 'commander'; import { exitWithError } from '../lib/errors'; @@ -252,6 +253,7 @@ function lazy( process.exit(0); } catch (error) { + assertError(error); exitWithError(error); } }; diff --git a/packages/cli/src/commands/remove-plugin/removePlugin.ts b/packages/cli/src/commands/remove-plugin/removePlugin.ts index 9e1e18bb7b..11fdc8bb11 100644 --- a/packages/cli/src/commands/remove-plugin/removePlugin.ts +++ b/packages/cli/src/commands/remove-plugin/removePlugin.ts @@ -20,6 +20,7 @@ import inquirer, { Answers, Question } from 'inquirer'; import { getCodeownersFilePath } from '../../lib/codeowners'; import { paths } from '../../lib/paths'; import { Task } from '../../lib/tasks'; +import { assertError } from '@backstage/errors'; const BACKSTAGE = '@backstage'; @@ -35,6 +36,7 @@ export const checkExists = async (rootDir: string, pluginName: string) => { ); } } catch (e) { + assertError(e); throw new Error( chalk.red( ` There was an error removing plugin ${chalk.cyan(pluginName)}: ${ @@ -51,6 +53,7 @@ export const removePluginDirectory = async (destination: string) => { try { await fse.remove(destination); } catch (e) { + assertError(e); throw Error( chalk.red( ` There was a problem removing the plugin directory: ${e.message}`, @@ -67,6 +70,7 @@ export const removeSymLink = async (destination: string) => { try { await fse.remove(destination); } catch (e) { + assertError(e); throw Error( chalk.red( ` Could not remove symbolic link\t${chalk.cyan(destination)}: ${ @@ -106,6 +110,7 @@ export const removeReferencesFromPluginsFile = async ( try { await removeAllStatementsContainingID(pluginsFile, pluginNameCapitalized); } catch (e) { + assertError(e); throw new Error( chalk.red( ` There was an error removing export statement for plugin ${chalk.cyan( @@ -125,6 +130,7 @@ export const removePluginFromCodeOwners = async ( try { await removeAllStatementsContainingID(codeOwnersFile, pluginName); } catch (e) { + assertError(e); throw new Error( chalk.red( ` There was an error removing code owners statement for plugin ${chalk.cyan( @@ -165,6 +171,7 @@ export const removeReferencesFromAppPackage = async ( 'utf-8', ); } catch (e) { + assertError(e); throw new Error( chalk.red( ` Failed to remove plugin as dependency in app: ${chalk.cyan( @@ -245,6 +252,7 @@ export default async () => { ); Task.log(); } catch (error) { + assertError(error); Task.error(error.message); Task.log('It seems that something went wrong when removing the plugin 🤔'); } diff --git a/packages/cli/src/commands/versions/bump.ts b/packages/cli/src/commands/versions/bump.ts index 257a83e2be..7b967839b2 100644 --- a/packages/cli/src/commands/versions/bump.ts +++ b/packages/cli/src/commands/versions/bump.ts @@ -17,6 +17,7 @@ import fs from 'fs-extra'; import chalk from 'chalk'; import semver from 'semver'; +import { isError } from '@backstage/errors'; import { resolve as resolvePath } from 'path'; import { run } from '../../lib/run'; import { paths } from '../../lib/paths'; @@ -59,7 +60,7 @@ export default async () => { try { target = await findTargetVersion(name); } catch (error) { - if (error.name === 'NotFoundError') { + if (isError(error) && error.name === 'NotFoundError') { console.log(`Package info not found, ignoring package ${name}`); return; } @@ -97,7 +98,7 @@ export default async () => { try { target = await findTargetVersion(name); } catch (error) { - if (error.name === 'NotFoundError') { + if (isError(error) && error.name === 'NotFoundError') { console.log(`Package info not found, ignoring package ${name}`); return; } diff --git a/packages/cli/src/lib/run.ts b/packages/cli/src/lib/run.ts index 7cf22a6df9..4efb791c30 100644 --- a/packages/cli/src/lib/run.ts +++ b/packages/cli/src/lib/run.ts @@ -23,6 +23,7 @@ import { import { ExitCodeError } from './errors'; import { promisify } from 'util'; import { LogFunc } from './logging'; +import { assertError, ForwardedError } from '@backstage/errors'; const execFile = promisify(execFileCb); @@ -75,10 +76,14 @@ export async function runPlain(cmd: string, ...args: string[]) { const { stdout } = await execFile(cmd, args, { shell: true }); return stdout.trim(); } catch (error) { - if (error.stderr) { - process.stderr.write(error.stderr); + assertError(error); + if ('stderr' in error) { + process.stderr.write(error.stderr as Buffer); } - throw new ExitCodeError(error.code, [cmd, ...args].join(' ')); + if (typeof error.code === 'number') { + throw new ExitCodeError(error.code, [cmd, ...args].join(' ')); + } + throw new ForwardedError('Unknown execution error', error); } } diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index a719c6389b..ebca3ffdf3 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -32,6 +32,7 @@ "dependencies": { "@backstage/cli-common": "^0.1.4", "@backstage/config": "^0.1.9", + "@backstage/errors": "^0.1.2", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", "chokidar": "^3.5.2", diff --git a/packages/config-loader/src/lib/env.ts b/packages/config-loader/src/lib/env.ts index b244c06c64..57cfc077a7 100644 --- a/packages/config-loader/src/lib/env.ts +++ b/packages/config-loader/src/lib/env.ts @@ -15,6 +15,7 @@ */ import { AppConfig, JsonObject } from '@backstage/config'; +import { assertError } from '@backstage/errors'; const ENV_PREFIX = 'APP_CONFIG_'; @@ -96,6 +97,7 @@ function safeJsonParse(str: string): [Error | null, any] { try { return [null, JSON.parse(str)]; } catch (err) { + assertError(err); return [err, str]; } } diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index f8b134119b..de6aff376f 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -24,6 +24,7 @@ import { import { ConfigSchemaPackageEntry } from './types'; import { getProgramFromFiles, generateSchema } from 'typescript-json-schema'; import { JsonObject } from '@backstage/config'; +import { assertError } from '@backstage/errors'; type Item = { name?: string; @@ -189,6 +190,7 @@ function compileTsSchemas(paths: string[]) { [path.split(sep).join('/')], // Unix paths are expected for all OSes here ) as JsonObject | null; } catch (error) { + assertError(error); if (error.message !== 'type Config not found') { throw error; } diff --git a/packages/config-loader/src/lib/transform/apply.ts b/packages/config-loader/src/lib/transform/apply.ts index 690d280266..091f9c23c5 100644 --- a/packages/config-loader/src/lib/transform/apply.ts +++ b/packages/config-loader/src/lib/transform/apply.ts @@ -15,6 +15,7 @@ */ import { JsonObject, JsonValue } from '@backstage/config'; +import { assertError } from '@backstage/errors'; import { TransformFunc } from './types'; import { isObject } from './utils'; @@ -46,6 +47,7 @@ export async function applyConfigTransforms( break; } } catch (error) { + assertError(error); throw new Error(`error at ${path}, ${error.message}`); } } diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 315b44c4ee..4abcc2e314 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -19,6 +19,7 @@ import yaml from 'yaml'; import chokidar from 'chokidar'; import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path'; import { AppConfig } from '@backstage/config'; +import { ForwardedError } from '@backstage/errors'; import { applyConfigTransforms, readEnvConfig, @@ -114,9 +115,7 @@ export async function loadConfig( try { fileConfigs = await loadConfigFiles(); } catch (error) { - throw new Error( - `Failed to read static configuration file, ${error.message}`, - ); + throw new ForwardedError('Failed to read static configuration file', error); } const envConfigs = await readEnvConfig(process.env); diff --git a/packages/core-app-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.test.ts b/packages/core-app-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.test.ts index 9cd666e8a3..7fcbde97df 100644 --- a/packages/core-app-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.test.ts +++ b/packages/core-app-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.test.ts @@ -45,25 +45,25 @@ describe('UrlPatternDiscovery', () => { it('should validate that the pattern is a valid URL', () => { expect(() => { UrlPatternDiscovery.compile('example.com'); - }).toThrow('Invalid discovery URL pattern, Invalid URL: example.com'); + }).toThrow("Invalid discovery URL pattern, URL 'example.com' is invalid"); expect(() => { UrlPatternDiscovery.compile('http://'); - }).toThrow('Invalid discovery URL pattern, Invalid URL: http://'); + }).toThrow("Invalid discovery URL pattern, URL 'http://' is invalid"); expect(() => { UrlPatternDiscovery.compile('abc123'); - }).toThrow('Invalid discovery URL pattern, Invalid URL: abc123'); + }).toThrow("Invalid discovery URL pattern, URL 'abc123' is invalid"); expect(() => { UrlPatternDiscovery.compile('http://example.com:{{pluginId}}'); }).toThrow( - 'Invalid discovery URL pattern, Invalid URL: http://example.com:pluginId', + "Invalid discovery URL pattern, URL 'http://example.com:pluginId' is invalid", ); expect(() => { UrlPatternDiscovery.compile('/{{pluginId}}'); - }).toThrow('Invalid discovery URL pattern, Invalid URL: /pluginId'); + }).toThrow("Invalid discovery URL pattern, URL '/pluginId' is invalid"); expect(() => { UrlPatternDiscovery.compile('http://localhost/{{pluginId}}?forbidden'); diff --git a/packages/core-app-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.ts b/packages/core-app-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.ts index 81cc5a26b1..58e14c9f04 100644 --- a/packages/core-app-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.ts +++ b/packages/core-app-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.ts @@ -16,6 +16,8 @@ import { DiscoveryApi } from '@backstage/core-plugin-api'; +const ERROR_PREFIX = 'Invalid discovery URL pattern,'; + /** * UrlPatternDiscovery is a lightweight DiscoveryApi implementation. * It uses a single template string to construct URLs for each plugin. @@ -30,21 +32,22 @@ export class UrlPatternDiscovery implements DiscoveryApi { */ static compile(pattern: string): UrlPatternDiscovery { const parts = pattern.split(/\{\{\s*pluginId\s*\}\}/); + const urlStr = parts.join('pluginId'); + let url; try { - const urlStr = parts.join('pluginId'); - const url = new URL(urlStr); - if (url.hash) { - throw new Error('URL must not have a hash'); - } - if (url.search) { - throw new Error('URL must not have a query'); - } - if (urlStr.endsWith('/')) { - throw new Error('URL must not end with a slash'); - } - } catch (error) { - throw new Error(`Invalid discovery URL pattern, ${error.message}`); + url = new URL(urlStr); + } catch { + throw new Error(`${ERROR_PREFIX} URL '${urlStr}' is invalid`); + } + if (url.hash) { + throw new Error(`${ERROR_PREFIX} URL must not have a hash`); + } + if (url.search) { + throw new Error(`${ERROR_PREFIX} URL must not have a query`); + } + if (urlStr.endsWith('/')) { + throw new Error(`${ERROR_PREFIX} URL must not end with a slash`); } return new UrlPatternDiscovery(parts); diff --git a/packages/core-app-api/src/routing/RoutingProvider.test.tsx b/packages/core-app-api/src/routing/RoutingProvider.test.tsx index 12d9e9ee33..fbdd34ae68 100644 --- a/packages/core-app-api/src/routing/RoutingProvider.test.tsx +++ b/packages/core-app-api/src/routing/RoutingProvider.test.tsx @@ -83,7 +83,7 @@ const MockRouteSource = (props: { } catch (ex) { return (
- Error at {props.name}: {ex.message} + Error at {props.name}, {String(ex)}
); } @@ -293,12 +293,12 @@ describe('discovery', () => { expect( rendered.getByText( - `Error at outsideWithParams: Cannot route to ${ref3} with parent ${ref5} as it has parameters`, + `Error at outsideWithParams, Error: Cannot route to ${ref3} with parent ${ref5} as it has parameters`, ), ).toBeInTheDocument(); expect( rendered.getByText( - `Error at outsideNoParams: Cannot route to ${ref3} with parent ${ref5} as it has parameters`, + `Error at outsideNoParams, Error: Cannot route to ${ref3} with parent ${ref5} as it has parameters`, ), ).toBeInTheDocument(); }); diff --git a/packages/core-components/src/components/OAuthRequestDialog/LoginRequestListItem.tsx b/packages/core-components/src/components/OAuthRequestDialog/LoginRequestListItem.tsx index b911a9a1d9..ef0d2373e4 100644 --- a/packages/core-components/src/components/OAuthRequestDialog/LoginRequestListItem.tsx +++ b/packages/core-components/src/components/OAuthRequestDialog/LoginRequestListItem.tsx @@ -21,6 +21,7 @@ import ListItemText from '@material-ui/core/ListItemText'; import Typography from '@material-ui/core/Typography'; import Button from '@material-ui/core/Button'; import React, { useState } from 'react'; +import { isError } from '@backstage/errors'; import { PendingAuthRequest } from '@backstage/core-plugin-api'; export type LoginRequestListItemClassKey = 'root'; @@ -42,14 +43,14 @@ type RowProps = { const LoginRequestListItem = ({ request, busy, setBusy }: RowProps) => { const classes = useItemStyles(); - const [error, setError] = useState(); + const [error, setError] = useState(); const handleContinue = async () => { setBusy(true); try { await request.trigger(); } catch (e) { - setError(e); + setError(isError(e) ? e.message : 'An unspecified error occurred'); } finally { setBusy(false); } @@ -64,13 +65,7 @@ const LoginRequestListItem = ({ request, busy, setBusy }: RowProps) => { - {error.message || 'An unspecified error occurred'} - - ) - } + secondary={error && {error}} />