From 96f1f522b9a47166bad78ea4b940ecd808eee3b5 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 20 Aug 2020 14:38:33 +0200 Subject: [PATCH 001/103] core-api: add DiscoveryApi + UrlPatternDiscovery implementation --- .../src/apis/definitions/DiscoveryApi.ts | 47 +++++++++++ .../core-api/src/apis/definitions/index.ts | 1 + .../DiscoveryApi/UrlPatternDiscovery.test.ts | 84 +++++++++++++++++++ .../DiscoveryApi/UrlPatternDiscovery.ts | 58 +++++++++++++ .../implementations/DiscoveryApi/index.ts | 21 +++++ .../src/apis/implementations/index.ts | 1 + 6 files changed, 212 insertions(+) create mode 100644 packages/core-api/src/apis/definitions/DiscoveryApi.ts create mode 100644 packages/core-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.test.ts create mode 100644 packages/core-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.ts create mode 100644 packages/core-api/src/apis/implementations/DiscoveryApi/index.ts diff --git a/packages/core-api/src/apis/definitions/DiscoveryApi.ts b/packages/core-api/src/apis/definitions/DiscoveryApi.ts new file mode 100644 index 0000000000..b0773086c7 --- /dev/null +++ b/packages/core-api/src/apis/definitions/DiscoveryApi.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createApiRef } from '../ApiRef'; + +/** + * The discovery API is used to provide a mechanism for plugins to + * discover the endpoint to use to talk to their backend counterpart. + * + * The purpose of the discovery API is to allow for many different deployment + * setups and routing methods through a central configuration, instead + * of letting each individual plugin manage that configuration. + * + * Implementations of the discovery API can be a simple as a URL pattern + * using the pluginId, but could also have overrides for individual plugins, + * or query a separate discovery service. + */ +export type DiscoveryApi = { + /** + * Returns the HTTP base backend URL for a given plugin, without a trailing slash. + * + * This method must always be called just before making a request. as opposed to + * fetching the URL when constructing an API client. That is to ensure that more + * flexible routing patterns can be supported. + * + * For example, asking for the URL for `auth` may return something + * like `https://backstage.example.com/api/auth` + */ + getBaseUrl(pluginId: string): Promise; +}; + +export const discoveryApiRef = createApiRef({ + id: 'core.discovery', + description: 'Provides service discovery of backend plugins', +}); diff --git a/packages/core-api/src/apis/definitions/index.ts b/packages/core-api/src/apis/definitions/index.ts index c5d4a15117..678dce9e32 100644 --- a/packages/core-api/src/apis/definitions/index.ts +++ b/packages/core-api/src/apis/definitions/index.ts @@ -27,6 +27,7 @@ export * from './AppThemeApi'; export * from './ConfigApi'; export * from './ErrorApi'; export * from './FeatureFlagsApi'; +export * from './DiscoveryApi'; export * from './IdentityApi'; export * from './OAuthRequestApi'; export * from './StorageApi'; diff --git a/packages/core-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.test.ts b/packages/core-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.test.ts new file mode 100644 index 0000000000..9597443b98 --- /dev/null +++ b/packages/core-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlPatternDiscovery } from './UrlPatternDiscovery'; + +describe('UrlPatternDiscovery', () => { + it('should not require interpolation', async () => { + const discoveryApi = UrlPatternDiscovery.compile('http://example.com'); + await expect(discoveryApi.getBaseUrl('my-plugin')).resolves.toBe( + 'http://example.com', + ); + }); + + it('should use a plain pattern', async () => { + const discoveryApi = UrlPatternDiscovery.compile( + 'http://localhost:7000/{{ pluginId }}', + ); + await expect(discoveryApi.getBaseUrl('my-plugin')).resolves.toBe( + 'http://localhost:7000/my-plugin', + ); + }); + + it('should allow for multiple interpolation points', async () => { + const discoveryApi = UrlPatternDiscovery.compile( + 'https://{{pluginId }}.example.com/api/{{ pluginId}}', + ); + await expect(discoveryApi.getBaseUrl('my-plugin')).resolves.toBe( + 'https://my-plugin.example.com/api/my-plugin', + ); + }); + + it('should validate that the pattern is a valid URL', () => { + expect(() => { + UrlPatternDiscovery.compile('example.com'); + }).toThrow('Invalid discovery URL pattern, Invalid URL: example.com'); + + expect(() => { + UrlPatternDiscovery.compile('http://'); + }).toThrow('Invalid discovery URL pattern, Invalid URL: http://'); + + expect(() => { + UrlPatternDiscovery.compile('abc123'); + }).toThrow('Invalid discovery URL pattern, Invalid URL: abc123'); + + expect(() => { + UrlPatternDiscovery.compile('http://example.com:{{pluginId}}'); + }).toThrow( + 'Invalid discovery URL pattern, Invalid URL: http://example.com:pluginId', + ); + + expect(() => { + UrlPatternDiscovery.compile('/{{pluginId}}'); + }).toThrow('Invalid discovery URL pattern, Invalid URL: /pluginId'); + + expect(() => { + UrlPatternDiscovery.compile('http://localhost/{{pluginId}}?forbidden'); + }).toThrow('Invalid discovery URL pattern, URL must not have a query'); + + expect(() => { + UrlPatternDiscovery.compile('http://localhost/{{pluginId}}#forbidden'); + }).toThrow('Invalid discovery URL pattern, URL must not have a hash'); + + expect(() => { + UrlPatternDiscovery.compile('http://localhost/{{pluginId}}/'); + }).toThrow('Invalid discovery URL pattern, URL must not end with a slash'); + + expect(() => { + UrlPatternDiscovery.compile('http://localhost/'); + }).toThrow('Invalid discovery URL pattern, URL must not end with a slash'); + }); +}); diff --git a/packages/core-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.ts b/packages/core-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.ts new file mode 100644 index 0000000000..ca48784584 --- /dev/null +++ b/packages/core-api/src/apis/implementations/DiscoveryApi/UrlPatternDiscovery.ts @@ -0,0 +1,58 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DiscoveryApi } from '../../definitions/DiscoveryApi'; + +/** + * UrlPatternDiscovery is a lightweight DiscoveryApi implementation. + * It uses a single template string to construct URLs for each plugin. + */ +export class UrlPatternDiscovery implements DiscoveryApi { + /** + * Creates a new UrlPatternDiscovery given a template. The the only + * interpolation done for the template is to replace instances of `{{pluginId}}` + * with the ID of the plugin being requested. + * + * Example pattern: `http://localhost:7000/api/{{ pluginId }}` + */ + static compile(pattern: string): UrlPatternDiscovery { + const parts = pattern.split(/\{\{\s*pluginId\s*\}\}/); + + try { + const urlStr = parts.join('pluginId'); + const url = new URL(urlStr); + if (url.hash) { + throw new Error('URL must not have a hash'); + } + if (url.search) { + throw new Error('URL must not have a query'); + } + if (urlStr.endsWith('/')) { + throw new Error('URL must not end with a slash'); + } + } catch (error) { + throw new Error(`Invalid discovery URL pattern, ${error.message}`); + } + + return new UrlPatternDiscovery(parts); + } + + private constructor(private readonly parts: string[]) {} + + async getBaseUrl(pluginId: string): Promise { + return this.parts.join(pluginId); + } +} diff --git a/packages/core-api/src/apis/implementations/DiscoveryApi/index.ts b/packages/core-api/src/apis/implementations/DiscoveryApi/index.ts new file mode 100644 index 0000000000..60a5b815e7 --- /dev/null +++ b/packages/core-api/src/apis/implementations/DiscoveryApi/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This folder contains implementations for all core APIs. +// +// Plugins should rely on these APIs for functionality as much as possible. + +export { UrlPatternDiscovery } from './UrlPatternDiscovery'; diff --git a/packages/core-api/src/apis/implementations/index.ts b/packages/core-api/src/apis/implementations/index.ts index e6d23fee21..30aeb81d44 100644 --- a/packages/core-api/src/apis/implementations/index.ts +++ b/packages/core-api/src/apis/implementations/index.ts @@ -24,5 +24,6 @@ export * from './AlertApi'; export * from './AppThemeApi'; export * from './ConfigApi'; export * from './ErrorApi'; +export * from './DiscoveryApi'; export * from './OAuthRequestApi'; export * from './StorageApi'; From ea1a36433d8b2b63e5a63e912f82f07d1dc336bf Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 20 Aug 2020 17:56:33 +0200 Subject: [PATCH 002/103] app,core-api: install DiscoveryApi in default app and use it for all auth providers --- packages/app/src/apis.ts | 29 ++++++----- .../implementations/auth/auth0/Auth0Auth.ts | 17 +++--- .../implementations/auth/github/GithubAuth.ts | 17 +++--- .../implementations/auth/gitlab/GitlabAuth.ts | 16 +++--- .../implementations/auth/google/GoogleAuth.ts | 17 +++--- .../implementations/auth/oauth2/OAuth2.ts | 16 +++--- .../implementations/auth/okta/OktaAuth.ts | 16 +++--- .../DefaultAuthConnector.test.ts | 11 ++-- .../lib/AuthConnector/DefaultAuthConnector.ts | 52 +++++++++---------- packages/dev-utils/src/devApp/apiFactories.ts | 37 +++++++------ .../ProfileCatalog/ProfileCatalog.test.tsx | 6 ++- 11 files changed, 120 insertions(+), 114 deletions(-) diff --git a/packages/app/src/apis.ts b/packages/app/src/apis.ts index df023b80cb..eb8cfb92af 100644 --- a/packages/app/src/apis.ts +++ b/packages/app/src/apis.ts @@ -24,6 +24,8 @@ import { ErrorAlerter, featureFlagsApiRef, FeatureFlags, + discoveryApiRef, + UrlPatternDiscovery, GoogleAuth, GithubAuth, OAuth2, @@ -74,7 +76,10 @@ import { TravisCIApi, travisCIApiRef, } from '@roadiehq/backstage-plugin-travis-ci'; -import { GithubPullRequestsClient, githubPullRequestsApiRef } from '@roadiehq/backstage-plugin-github-pull-requests'; +import { + GithubPullRequestsClient, + githubPullRequestsApiRef, +} from '@roadiehq/backstage-plugin-github-pull-requests'; export const apis = (config: ConfigApi) => { // eslint-disable-next-line no-console @@ -85,6 +90,10 @@ export const apis = (config: ConfigApi) => { const builder = ApiRegistry.builder(); + const discoveryApi = builder.add( + discoveryApiRef, + UrlPatternDiscovery.compile(`${backendUrl}/{{ pluginId }}`), + ); const alertApi = builder.add(alertApiRef, new AlertApiForwarder()); const errorApi = builder.add( errorApiRef, @@ -116,8 +125,7 @@ export const apis = (config: ConfigApi) => { builder.add( googleAuthApiRef, GoogleAuth.create({ - backendUrl, - basePath: '/auth/', + discoveryApi, oauthRequestApi, }), ); @@ -125,8 +133,7 @@ export const apis = (config: ConfigApi) => { const githubAuthApi = builder.add( githubAuthApiRef, GithubAuth.create({ - backendUrl, - basePath: '/auth/', + discoveryApi, oauthRequestApi, }), ); @@ -134,8 +141,7 @@ export const apis = (config: ConfigApi) => { builder.add( oktaAuthApiRef, OktaAuth.create({ - backendUrl, - basePath: '/auth/', + discoveryApi, oauthRequestApi, }), ); @@ -143,8 +149,7 @@ export const apis = (config: ConfigApi) => { builder.add( gitlabAuthApiRef, GitlabAuth.create({ - backendUrl, - basePath: '/auth/', + discoveryApi, oauthRequestApi, }), ); @@ -152,8 +157,7 @@ export const apis = (config: ConfigApi) => { builder.add( auth0AuthApiRef, Auth0Auth.create({ - backendUrl, - basePath: '/auth/', + discoveryApi, oauthRequestApi, }), ); @@ -161,8 +165,7 @@ export const apis = (config: ConfigApi) => { builder.add( oauth2ApiRef, OAuth2.create({ - backendUrl, - basePath: '/auth/', + discoveryApi, oauthRequestApi, }), ); diff --git a/packages/core-api/src/apis/implementations/auth/auth0/Auth0Auth.ts b/packages/core-api/src/apis/implementations/auth/auth0/Auth0Auth.ts index e69733741f..505c283b71 100644 --- a/packages/core-api/src/apis/implementations/auth/auth0/Auth0Auth.ts +++ b/packages/core-api/src/apis/implementations/auth/auth0/Auth0Auth.ts @@ -27,16 +27,17 @@ import { AuthRequestOptions, BackstageIdentity, } from '../../../definitions/auth'; -import { OAuthRequestApi, AuthProvider } from '../../../definitions'; +import { + OAuthRequestApi, + AuthProvider, + DiscoveryApi, +} from '../../../definitions'; import { SessionManager } from '../../../../lib/AuthSessionManager/types'; import { RefreshingAuthSessionManager } from '../../../../lib/AuthSessionManager'; import { Observable } from '../../../../types'; type CreateOptions = { - // TODO(Following the words of Rugvip): These two should be grabbed from global config when available, they're not unique to Auth0Auth - backendUrl: string; - basePath: string; - + discoveryApi: DiscoveryApi; oauthRequestApi: OAuthRequestApi; environment?: string; @@ -67,15 +68,13 @@ class Auth0Auth BackstageIdentityApi, SessionStateApi { static create({ - backendUrl, - basePath, + discoveryApi, environment = 'development', provider = DEFAULT_PROVIDER, oauthRequestApi, }: CreateOptions) { const connector = new DefaultAuthConnector({ - backendUrl, - basePath, + discoveryApi, environment, provider, oauthRequestApi: oauthRequestApi, diff --git a/packages/core-api/src/apis/implementations/auth/github/GithubAuth.ts b/packages/core-api/src/apis/implementations/auth/github/GithubAuth.ts index 1ad3e08699..8b9f807cd8 100644 --- a/packages/core-api/src/apis/implementations/auth/github/GithubAuth.ts +++ b/packages/core-api/src/apis/implementations/auth/github/GithubAuth.ts @@ -25,7 +25,11 @@ import { BackstageIdentity, AuthRequestOptions, } from '../../../definitions/auth'; -import { OAuthRequestApi, AuthProvider } from '../../../definitions'; +import { + OAuthRequestApi, + AuthProvider, + DiscoveryApi, +} from '../../../definitions'; import { SessionManager } from '../../../../lib/AuthSessionManager/types'; import { AuthSessionStore, @@ -34,10 +38,7 @@ import { import { Observable } from '../../../../types'; type CreateOptions = { - // TODO(Rugvip): These two should be grabbed from global config when available, they're not unique to GithubAuth - backendUrl: string; - basePath: string; - + discoveryApi: DiscoveryApi; oauthRequestApi: OAuthRequestApi; environment?: string; @@ -62,15 +63,13 @@ const DEFAULT_PROVIDER = { class GithubAuth implements OAuthApi, SessionStateApi { static create({ - backendUrl, - basePath, + discoveryApi, environment = 'development', provider = DEFAULT_PROVIDER, oauthRequestApi, }: CreateOptions) { const connector = new DefaultAuthConnector({ - backendUrl, - basePath, + discoveryApi, environment, provider, oauthRequestApi: oauthRequestApi, diff --git a/packages/core-api/src/apis/implementations/auth/gitlab/GitlabAuth.ts b/packages/core-api/src/apis/implementations/auth/gitlab/GitlabAuth.ts index e612c4ca61..1734e930fa 100644 --- a/packages/core-api/src/apis/implementations/auth/gitlab/GitlabAuth.ts +++ b/packages/core-api/src/apis/implementations/auth/gitlab/GitlabAuth.ts @@ -25,15 +25,17 @@ import { BackstageIdentity, AuthRequestOptions, } from '../../../definitions/auth'; -import { OAuthRequestApi, AuthProvider } from '../../../definitions'; +import { + OAuthRequestApi, + AuthProvider, + DiscoveryApi, +} from '../../../definitions'; import { SessionManager } from '../../../../lib/AuthSessionManager/types'; import { StaticAuthSessionManager } from '../../../../lib/AuthSessionManager'; import { Observable } from '../../../../types'; type CreateOptions = { - backendUrl: string; - basePath: string; - + discoveryApi: DiscoveryApi; oauthRequestApi: OAuthRequestApi; environment?: string; @@ -58,15 +60,13 @@ const DEFAULT_PROVIDER = { class GitlabAuth implements OAuthApi, SessionStateApi { static create({ - backendUrl, - basePath, + discoveryApi, environment = 'development', provider = DEFAULT_PROVIDER, oauthRequestApi, }: CreateOptions) { const connector = new DefaultAuthConnector({ - backendUrl, - basePath, + discoveryApi, environment, provider, oauthRequestApi, diff --git a/packages/core-api/src/apis/implementations/auth/google/GoogleAuth.ts b/packages/core-api/src/apis/implementations/auth/google/GoogleAuth.ts index c6a21dbb20..fdf9d46ba8 100644 --- a/packages/core-api/src/apis/implementations/auth/google/GoogleAuth.ts +++ b/packages/core-api/src/apis/implementations/auth/google/GoogleAuth.ts @@ -28,16 +28,17 @@ import { AuthRequestOptions, BackstageIdentity, } from '../../../definitions/auth'; -import { OAuthRequestApi, AuthProvider } from '../../../definitions'; +import { + OAuthRequestApi, + AuthProvider, + DiscoveryApi, +} from '../../../definitions'; import { SessionManager } from '../../../../lib/AuthSessionManager/types'; import { RefreshingAuthSessionManager } from '../../../../lib/AuthSessionManager'; import { Observable } from '../../../../types'; type CreateOptions = { - // TODO(Rugvip): These two should be grabbed from global config when available, they're not unique to GoogleAuth - backendUrl: string; - basePath: string; - + discoveryApi: DiscoveryApi; oauthRequestApi: OAuthRequestApi; environment?: string; @@ -71,15 +72,13 @@ class GoogleAuth BackstageIdentityApi, SessionStateApi { static create({ - backendUrl, - basePath, + discoveryApi, environment = 'development', provider = DEFAULT_PROVIDER, oauthRequestApi, }: CreateOptions) { const connector = new DefaultAuthConnector({ - backendUrl, - basePath, + discoveryApi, environment, provider, oauthRequestApi: oauthRequestApi, diff --git a/packages/core-api/src/apis/implementations/auth/oauth2/OAuth2.ts b/packages/core-api/src/apis/implementations/auth/oauth2/OAuth2.ts index 4b6177bed9..cf582b3a13 100644 --- a/packages/core-api/src/apis/implementations/auth/oauth2/OAuth2.ts +++ b/packages/core-api/src/apis/implementations/auth/oauth2/OAuth2.ts @@ -19,7 +19,11 @@ import { DefaultAuthConnector } from '../../../../lib/AuthConnector'; import { RefreshingAuthSessionManager } from '../../../../lib/AuthSessionManager'; import { SessionManager } from '../../../../lib/AuthSessionManager/types'; import { Observable } from '../../../../types'; -import { AuthProvider, OAuthRequestApi } from '../../../definitions'; +import { + AuthProvider, + OAuthRequestApi, + DiscoveryApi, +} from '../../../definitions'; import { AuthRequestOptions, BackstageIdentity, @@ -33,9 +37,7 @@ import { import { OAuth2Session } from './types'; type CreateOptions = { - backendUrl: string; - basePath: string; - + discoveryApi: DiscoveryApi; oauthRequestApi: OAuthRequestApi; environment?: string; @@ -64,15 +66,13 @@ const SCOPE_PREFIX = ''; class OAuth2 implements OAuthApi, OpenIdConnectApi, ProfileInfoApi, SessionStateApi { static create({ - backendUrl, - basePath, + discoveryApi, environment = 'development', provider = DEFAULT_PROVIDER, oauthRequestApi, }: CreateOptions) { const connector = new DefaultAuthConnector({ - backendUrl, - basePath, + discoveryApi, environment, provider, oauthRequestApi: oauthRequestApi, diff --git a/packages/core-api/src/apis/implementations/auth/okta/OktaAuth.ts b/packages/core-api/src/apis/implementations/auth/okta/OktaAuth.ts index f60b182118..1f804748f0 100644 --- a/packages/core-api/src/apis/implementations/auth/okta/OktaAuth.ts +++ b/packages/core-api/src/apis/implementations/auth/okta/OktaAuth.ts @@ -28,15 +28,17 @@ import { AuthRequestOptions, BackstageIdentity, } from '../../../definitions/auth'; -import { OAuthRequestApi, AuthProvider } from '../../../definitions'; +import { + OAuthRequestApi, + AuthProvider, + DiscoveryApi, +} from '../../../definitions'; import { SessionManager } from '../../../../lib/AuthSessionManager/types'; import { RefreshingAuthSessionManager } from '../../../../lib/AuthSessionManager'; import { Observable } from '../../../../types'; type CreateOptions = { - backendUrl: string; - basePath: string; - + discoveryApi: DiscoveryApi; oauthRequestApi: OAuthRequestApi; environment?: string; @@ -80,15 +82,13 @@ class OktaAuth BackstageIdentityApi, SessionStateApi { static create({ - backendUrl, - basePath, + discoveryApi, environment = 'development', provider = DEFAULT_PROVIDER, oauthRequestApi, }: CreateOptions) { const connector = new DefaultAuthConnector({ - backendUrl, - basePath, + discoveryApi, environment, provider, oauthRequestApi: oauthRequestApi, diff --git a/packages/core-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts b/packages/core-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts index b6c31405ae..5781130799 100644 --- a/packages/core-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts +++ b/packages/core-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts @@ -18,11 +18,12 @@ import ProviderIcon from '@material-ui/icons/AcUnit'; import { DefaultAuthConnector } from './DefaultAuthConnector'; import MockOAuthApi from '../../apis/implementations/OAuthRequestApi/MockOAuthApi'; import * as loginPopup from '../loginPopup'; +import { UrlPatternDiscovery } from '../../apis'; const anyFetch = fetch as any; const defaultOptions = { - backendUrl: 'http://my-origin', + discoveryApi: UrlPatternDiscovery.compile('http://my-host/api/{{pluginId}}'), environment: 'production', provider: { id: 'my-provider', @@ -115,7 +116,7 @@ describe('DefaultAuthConnector', () => { expect(popupSpy).toBeCalledTimes(1); expect(popupSpy.mock.calls[0][0]).toMatchObject({ url: - 'http://my-origin/api/auth/my-provider/start?scope=a%20b&env=production', + 'http://my-host/api/auth/my-provider/start?scope=a%20b&env=production', }); await expect(sessionPromise).resolves.toEqual({ @@ -141,9 +142,9 @@ describe('DefaultAuthConnector', () => { instantPopup: true, }); - expect(popupSpy).toBeCalledTimes(1); - await expect(sessionPromise).resolves.toBe('my-session'); + + expect(popupSpy).toBeCalledTimes(1); }); it('should use join func to join scopes', async () => { @@ -164,7 +165,7 @@ describe('DefaultAuthConnector', () => { expect(popupSpy).toBeCalledTimes(1); expect(popupSpy.mock.calls[0][0]).toMatchObject({ url: - 'http://my-origin/api/auth/my-provider/start?scope=-ab-&env=production', + 'http://my-host/api/auth/my-provider/start?scope=-ab-&env=production', }); }); }); diff --git a/packages/core-api/src/lib/AuthConnector/DefaultAuthConnector.ts b/packages/core-api/src/lib/AuthConnector/DefaultAuthConnector.ts index 41af7f4670..1c7e92daa8 100644 --- a/packages/core-api/src/lib/AuthConnector/DefaultAuthConnector.ts +++ b/packages/core-api/src/lib/AuthConnector/DefaultAuthConnector.ts @@ -15,21 +15,19 @@ */ import { AuthRequester } from '../../apis'; -import { OAuthRequestApi, AuthProvider } from '../../apis/definitions'; +import { + OAuthRequestApi, + AuthProvider, + DiscoveryApi, +} from '../../apis/definitions'; import { showLoginPopup } from '../loginPopup'; import { AuthConnector, CreateSessionOptions } from './types'; -const DEFAULT_BASE_PATH = '/api/auth/'; - type Options = { /** - * The base URL of the auth backend. + * DiscoveryApi instance used to locate the auth backend endpoint. */ - backendUrl?: string; - /** - * Base path of the auth requests, defaults to /api/auth/ - */ - basePath?: string; + discoveryApi: DiscoveryApi; /** * Environment hint passed on to auth backend, for example 'production' or 'development' */ @@ -64,8 +62,7 @@ function defaultJoinScopes(scopes: Set) { */ export class DefaultAuthConnector implements AuthConnector { - private readonly backendUrl: string; - private readonly basePath: string; + private readonly discoveryApi: DiscoveryApi; private readonly environment: string; private readonly provider: AuthProvider & { id: string }; private readonly joinScopesFunc: (scopes: Set) => string; @@ -74,8 +71,7 @@ export class DefaultAuthConnector constructor(options: Options) { const { - backendUrl = window.location.origin, - basePath = DEFAULT_BASE_PATH, + discoveryApi, environment, provider, joinScopes = defaultJoinScopes, @@ -88,8 +84,7 @@ export class DefaultAuthConnector onAuthRequest: scopes => this.showPopup(scopes), }); - this.backendUrl = backendUrl; - this.basePath = basePath; + this.discoveryApi = discoveryApi; this.environment = environment; this.provider = provider; this.joinScopesFunc = joinScopes; @@ -104,12 +99,15 @@ export class DefaultAuthConnector } async refreshSession(): Promise { - const res = await fetch(this.buildUrl('/refresh', { optional: true }), { - headers: { - 'x-requested-with': 'XMLHttpRequest', + const res = await fetch( + await this.buildUrl('/refresh', { optional: true }), + { + headers: { + 'x-requested-with': 'XMLHttpRequest', + }, + credentials: 'include', }, - credentials: 'include', - }).catch(error => { + ).catch(error => { throw new Error(`Auth refresh request failed, ${error}`); }); @@ -134,7 +132,7 @@ export class DefaultAuthConnector } async removeSession(): Promise { - const res = await fetch(this.buildUrl('/logout'), { + const res = await fetch(await this.buildUrl('/logout'), { method: 'POST', headers: { 'x-requested-with': 'XMLHttpRequest', @@ -153,13 +151,12 @@ export class DefaultAuthConnector private async showPopup(scopes: Set): Promise { const scope = this.joinScopesFunc(scopes); - const popupUrl = this.buildUrl('/start', { scope }); - const { origin } = new URL(this.backendUrl); + const popupUrl = await this.buildUrl('/start', { scope }); const payload = await showLoginPopup({ url: popupUrl, name: `${this.provider.title} Login`, - origin, + origin: new URL(popupUrl).origin, width: 450, height: 730, }); @@ -167,16 +164,17 @@ export class DefaultAuthConnector return await this.sessionTransform(payload); } - private buildUrl( + private async buildUrl( path: string, query?: { [key: string]: string | boolean | undefined }, - ): string { + ): Promise { + const baseUrl = await this.discoveryApi.getBaseUrl('auth'); const queryString = this.buildQueryString({ ...query, env: this.environment, }); - return `${this.backendUrl}${this.basePath}${this.provider.id}${path}${queryString}`; + return `${baseUrl}/${this.provider.id}${path}${queryString}`; } private buildQueryString(query?: { diff --git a/packages/dev-utils/src/devApp/apiFactories.ts b/packages/dev-utils/src/devApp/apiFactories.ts index 5acf64e0d0..4cf208f5d2 100644 --- a/packages/dev-utils/src/devApp/apiFactories.ts +++ b/packages/dev-utils/src/devApp/apiFactories.ts @@ -24,6 +24,8 @@ import { AlertApiForwarder, oauthRequestApiRef, OAuthRequestManager, + UrlPatternDiscovery, + discoveryApiRef, GoogleAuth, googleAuthApiRef, GithubAuth, @@ -58,46 +60,49 @@ export const oauthRequestApiFactory = createApiFactory({ factory: () => new OAuthRequestManager(), }); +export const discoveryApiFactory = createApiFactory({ + implements: discoveryApiRef, + deps: {}, + factory: () => + UrlPatternDiscovery.compile(`http://localhost:7000/{{ pluginId }}`), +}); + export const googleAuthApiFactory = createApiFactory({ implements: googleAuthApiRef, - deps: { oauthRequestApi: oauthRequestApiRef }, - factory: ({ oauthRequestApi }) => + deps: { discoveryApi: discoveryApiRef, oauthRequestApi: oauthRequestApiRef }, + factory: ({ discoveryApi, oauthRequestApi }) => GoogleAuth.create({ - backendUrl: 'http://localhost:7000', - basePath: '/auth/', + discoveryApi, oauthRequestApi, }), }); export const githubAuthApiFactory = createApiFactory({ implements: githubAuthApiRef, - deps: { oauthRequestApi: oauthRequestApiRef }, - factory: ({ oauthRequestApi }) => + deps: { discoveryApi: discoveryApiRef, oauthRequestApi: oauthRequestApiRef }, + factory: ({ discoveryApi, oauthRequestApi }) => GithubAuth.create({ - backendUrl: 'http://localhost:7000', - basePath: '/auth/', + discoveryApi, oauthRequestApi, }), }); export const gitlabAuthApiFactory = createApiFactory({ implements: gitlabAuthApiRef, - deps: { oauthRequestApi: oauthRequestApiRef }, - factory: ({ oauthRequestApi }) => + deps: { discoveryApi: discoveryApiRef, oauthRequestApi: oauthRequestApiRef }, + factory: ({ discoveryApi, oauthRequestApi }) => GitlabAuth.create({ - backendUrl: 'http://localhost:7000', - basePath: '/auth/', + discoveryApi, oauthRequestApi, }), }); export const auth0AuthApiFactory = createApiFactory({ implements: auth0AuthApiRef, - deps: { oauthRequestApi: oauthRequestApiRef }, - factory: ({ oauthRequestApi }) => + deps: { discoveryApi: discoveryApiRef, oauthRequestApi: oauthRequestApiRef }, + factory: ({ discoveryApi, oauthRequestApi }) => Auth0Auth.create({ - backendUrl: 'http://localhost:7000', - basePath: '/auth/', + discoveryApi, oauthRequestApi, }), }); diff --git a/plugins/gitops-profiles/src/components/ProfileCatalog/ProfileCatalog.test.tsx b/plugins/gitops-profiles/src/components/ProfileCatalog/ProfileCatalog.test.tsx index a502f6fb4e..06074ff84d 100644 --- a/plugins/gitops-profiles/src/components/ProfileCatalog/ProfileCatalog.test.tsx +++ b/plugins/gitops-profiles/src/components/ProfileCatalog/ProfileCatalog.test.tsx @@ -26,6 +26,7 @@ import { githubAuthApiRef, GithubAuth, OAuthRequestManager, + UrlPatternDiscovery, } from '@backstage/core'; import { gitOpsApiRef, GitOpsRestApi } from '../../api'; @@ -37,8 +38,9 @@ describe('ProfileCatalog', () => { [ githubAuthApiRef, GithubAuth.create({ - backendUrl: 'http://localhost:7000', - basePath: '/auth/', + discoveryApi: UrlPatternDiscovery.compile( + 'http://example.com/{{pluginId}}', + ), oauthRequestApi, }), ], From 382491dc5af1ca2567a93a1be4b218b4b08a93e6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 21 Aug 2020 14:25:09 +0200 Subject: [PATCH 003/103] plugins/catalog: switch CatalogClient to use DiscoveryApi --- packages/app/src/apis.ts | 8 +---- plugins/catalog/src/api/CatalogClient.test.ts | 33 ++++++++----------- plugins/catalog/src/api/CatalogClient.ts | 23 +++++-------- 3 files changed, 22 insertions(+), 42 deletions(-) diff --git a/packages/app/src/apis.ts b/packages/app/src/apis.ts index eb8cfb92af..c787ffc589 100644 --- a/packages/app/src/apis.ts +++ b/packages/app/src/apis.ts @@ -178,13 +178,7 @@ export const apis = (config: ConfigApi) => { }), ); - builder.add( - catalogApiRef, - new CatalogClient({ - apiOrigin: backendUrl, - basePath: '/catalog', - }), - ); + builder.add(catalogApiRef, new CatalogClient({ discoveryApi })); builder.add( scaffolderApiRef, diff --git a/plugins/catalog/src/api/CatalogClient.test.ts b/plugins/catalog/src/api/CatalogClient.test.ts index 7803f2e173..18f062db77 100644 --- a/plugins/catalog/src/api/CatalogClient.test.ts +++ b/plugins/catalog/src/api/CatalogClient.test.ts @@ -18,25 +18,21 @@ import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { CatalogClient } from './CatalogClient'; import { Entity } from '@backstage/catalog-model'; +import { UrlPatternDiscovery } from '@backstage/core'; const server = setupServer(); +const mockBaseUrl = 'http://backstage:9191/i-am-a-mock-base'; +const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl); describe('CatalogClient', () => { beforeAll(() => server.listen()); afterEach(() => server.resetHandlers()); afterAll(() => server.close()); - const mockApiOrigin = 'http://backstage:9191'; - const mockBasePath = '/i-am-a-mock-base'; - let client = new CatalogClient({ - apiOrigin: mockApiOrigin, - basePath: mockBasePath, - }); + + let client = new CatalogClient({ discoveryApi }); beforeEach(() => { - client = new CatalogClient({ - apiOrigin: mockApiOrigin, - basePath: mockBasePath, - }); + client = new CatalogClient({ discoveryApi }); }); describe('getEntiies', () => { @@ -61,7 +57,7 @@ describe('CatalogClient', () => { beforeEach(() => { server.use( - rest.get(`${mockApiOrigin}${mockBasePath}/entities`, (_, res, ctx) => { + rest.get(`${mockBaseUrl}/entities`, (_, res, ctx) => { return res(ctx.json(defaultResponse)); }), ); @@ -75,15 +71,12 @@ describe('CatalogClient', () => { it('builds entity search filters properly', async () => { expect.assertions(2); server.use( - rest.get( - `${mockApiOrigin}${mockBasePath}/entities`, - (req, res, ctx) => { - expect(req.url.searchParams.toString()).toBe( - 'a=1&b=2&b=3&%C3%B6=%3D', - ); - return res(ctx.json([])); - }, - ), + rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => { + expect(req.url.searchParams.toString()).toBe( + 'a=1&b=2&b=3&%C3%B6=%3D', + ); + return res(ctx.json([])); + }), ); const entities = await client.getEntities({ diff --git a/plugins/catalog/src/api/CatalogClient.ts b/plugins/catalog/src/api/CatalogClient.ts index 3804315ace..d5ff033caa 100644 --- a/plugins/catalog/src/api/CatalogClient.ts +++ b/plugins/catalog/src/api/CatalogClient.ts @@ -20,24 +20,17 @@ import { LOCATION_ANNOTATION, } from '@backstage/catalog-model'; import { CatalogApi, EntityCompoundName } from './types'; +import { DiscoveryApi } from '@backstage/core'; export class CatalogClient implements CatalogApi { - private apiOrigin: string; - private basePath: string; + private readonly discoveryApi: DiscoveryApi; - constructor({ - apiOrigin, - basePath, - }: { - apiOrigin: string; - basePath: string; - }) { - this.apiOrigin = apiOrigin; - this.basePath = basePath; + constructor(options: { discoveryApi: DiscoveryApi }) { + this.discoveryApi = options.discoveryApi; } private async getRequired(path: string): Promise { - const url = `${this.apiOrigin}${this.basePath}${path}`; + const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; const response = await fetch(url); if (!response.ok) { @@ -50,7 +43,7 @@ export class CatalogClient implements CatalogApi { } private async getOptional(path: string): Promise { - const url = `${this.apiOrigin}${this.basePath}${path}`; + const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; const response = await fetch(url); if (!response.ok) { @@ -100,7 +93,7 @@ export class CatalogClient implements CatalogApi { async addLocation(type: string, target: string) { const response = await fetch( - `${this.apiOrigin}${this.basePath}/locations`, + `${await this.discoveryApi.getBaseUrl('catalog')}/locations`, { headers: { 'Content-Type': 'application/json', @@ -135,7 +128,7 @@ export class CatalogClient implements CatalogApi { async removeEntityByUid(uid: string): Promise { const response = await fetch( - `${this.apiOrigin}${this.basePath}/entities/by-uid/${uid}`, + `${await this.discoveryApi.getBaseUrl('catalog')}/entities/by-uid/${uid}`, { method: 'DELETE', }, From be837b7b361965aee91becb5e56ed2bda5127291 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 21 Aug 2020 14:28:23 +0200 Subject: [PATCH 004/103] plugins/scaffolder: switch ScaffolderApi to use DiscoveryApi --- packages/app/src/apis.ts | 8 +------- plugins/scaffolder/src/api.ts | 24 ++++++++---------------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/packages/app/src/apis.ts b/packages/app/src/apis.ts index c787ffc589..72dad858c7 100644 --- a/packages/app/src/apis.ts +++ b/packages/app/src/apis.ts @@ -180,13 +180,7 @@ export const apis = (config: ConfigApi) => { builder.add(catalogApiRef, new CatalogClient({ discoveryApi })); - builder.add( - scaffolderApiRef, - new ScaffolderApi({ - apiOrigin: backendUrl, - basePath: '/scaffolder/v1', - }), - ); + builder.add(scaffolderApiRef, new ScaffolderApi({ discoveryApi })); builder.add(gitOpsApiRef, new GitOpsRestApi('http://localhost:3008')); diff --git a/plugins/scaffolder/src/api.ts b/plugins/scaffolder/src/api.ts index 4a0d508a2b..3c42ed2ca4 100644 --- a/plugins/scaffolder/src/api.ts +++ b/plugins/scaffolder/src/api.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { createApiRef } from '@backstage/core'; +import { createApiRef, DiscoveryApi } from '@backstage/core'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; export const scaffolderApiRef = createApiRef({ @@ -23,18 +23,10 @@ export const scaffolderApiRef = createApiRef({ }); export class ScaffolderApi { - private apiOrigin: string; - private basePath: string; + private readonly discoveryApi: DiscoveryApi; - constructor({ - apiOrigin, - basePath, - }: { - apiOrigin: string; - basePath: string; - }) { - this.apiOrigin = apiOrigin; - this.basePath = basePath; + constructor(options: { discoveryApi: DiscoveryApi }) { + this.discoveryApi = options.discoveryApi; } /** @@ -46,7 +38,7 @@ export class ScaffolderApi { template: TemplateEntityV1alpha1, values: Record, ) { - const url = `${this.apiOrigin}${this.basePath}/jobs`; + const url = `${await this.discoveryApi.getBaseUrl('scaffolder')}/v1/jobs`; const response = await fetch(url, { method: 'POST', headers: { @@ -65,9 +57,9 @@ export class ScaffolderApi { } async getJob(jobId: string) { - const url = `${this.apiOrigin}${this.basePath}/job/${encodeURIComponent( - jobId, - )}`; + const url = `${await this.discoveryApi.getBaseUrl( + 'scaffolder', + )}/v1/job/${encodeURIComponent(jobId)}`; return fetch(url).then(x => x.json()); } } From 6f0c438519b7b3deae60440090d163a317d90740 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 21 Aug 2020 15:13:52 +0200 Subject: [PATCH 005/103] plugins/rollbar: switch RollbarClient to use DiscoveryApi --- packages/app/src/apis.ts | 8 +------- plugins/rollbar/src/api/RollbarClient.ts | 17 +++++------------ 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/packages/app/src/apis.ts b/packages/app/src/apis.ts index 72dad858c7..0c409994cb 100644 --- a/packages/app/src/apis.ts +++ b/packages/app/src/apis.ts @@ -201,13 +201,7 @@ export const apis = (config: ConfigApi) => { ]), ); - builder.add( - rollbarApiRef, - new RollbarClient({ - apiOrigin: backendUrl, - basePath: '/rollbar', - }), - ); + builder.add(rollbarApiRef, new RollbarClient({ discoveryApi })); builder.add( techdocsStorageApiRef, diff --git a/plugins/rollbar/src/api/RollbarClient.ts b/plugins/rollbar/src/api/RollbarClient.ts index 5cabbcbc24..1862ad8270 100644 --- a/plugins/rollbar/src/api/RollbarClient.ts +++ b/plugins/rollbar/src/api/RollbarClient.ts @@ -20,20 +20,13 @@ import { RollbarProject, RollbarTopActiveItem, } from './types'; +import { DiscoveryApi } from '@backstage/core'; export class RollbarClient implements RollbarApi { - private apiOrigin: string; - private basePath: string; + private readonly discoveryApi: DiscoveryApi; - constructor({ - apiOrigin, - basePath, - }: { - apiOrigin: string; - basePath: string; - }) { - this.apiOrigin = apiOrigin; - this.basePath = basePath; + constructor(options: { discoveryApi: DiscoveryApi }) { + this.discoveryApi = options.discoveryApi; } async getAllProjects(): Promise { @@ -59,7 +52,7 @@ export class RollbarClient implements RollbarApi { } private async get(path: string): Promise { - const url = `${this.apiOrigin}${this.basePath}${path}`; + const url = `${await this.discoveryApi.getBaseUrl('rollbar')}${path}`; const response = await fetch(url); if (!response.ok) { From a21cfcd062c2f2944197d4a36fbefb90b5971c22 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 22 Aug 2020 11:11:10 +0200 Subject: [PATCH 006/103] create-app: update template to use DiscoveryApi --- .../default-app/packages/app/src/apis.ts | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/create-app/templates/default-app/packages/app/src/apis.ts b/packages/create-app/templates/default-app/packages/app/src/apis.ts index 35f3e076d6..4cc2ebe03a 100644 --- a/packages/create-app/templates/default-app/packages/app/src/apis.ts +++ b/packages/create-app/templates/default-app/packages/app/src/apis.ts @@ -6,6 +6,8 @@ import { ConfigApi, ErrorApiForwarder, ErrorAlerter, + discoveryApiRef, + UrlPatternDiscovery, oauthRequestApiRef, OAuthRequestManager, storageApiRef, @@ -24,6 +26,10 @@ export const apis = (config: ConfigApi) => { const builder = ApiRegistry.builder(); + const discoveryApi = builder.add( + discoveryApiRef, + UrlPatternDiscovery.compile(`${backendUrl}/{{ pluginId }}`), + ); const alertApi = builder.add(alertApiRef, new AlertApiForwarder()); const errorApi = builder.add( errorApiRef, @@ -33,21 +39,9 @@ export const apis = (config: ConfigApi) => { builder.add(storageApiRef, WebStorage.create({ errorApi })); builder.add(oauthRequestApiRef, new OAuthRequestManager()); - builder.add( - catalogApiRef, - new CatalogClient({ - apiOrigin: backendUrl, - basePath: '/catalog', - }), - ); + builder.add(catalogApiRef, new CatalogClient({ discoveryApi })); - builder.add( - scaffolderApiRef, - new ScaffolderApi({ - apiOrigin: backendUrl, - basePath: '/scaffolder/v1', - }), - ); + builder.add(scaffolderApiRef, new ScaffolderApi({ discoveryApi })); return builder.build(); }; From 257a3b52ed64d82230b270a2c04a9543fc1f42ec Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 00:17:57 +1200 Subject: [PATCH 007/103] Add Azure ingestion processor --- .../src/ingestion/LocationReaders.ts | 2 + .../AzureApiReaderProcessor.test.ts | 84 +++++++++++ .../processors/AzureApiReaderProcessor.ts | 134 ++++++++++++++++++ .../RegisterComponentForm.tsx | 2 +- .../RegisterComponentPage.tsx | 13 +- 5 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 05f93815b8..29892bb2dc 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -31,6 +31,7 @@ import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor' import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; +import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor'; import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import { StaticLocationProcessor } from './processors/StaticLocationProcessor'; @@ -79,6 +80,7 @@ export class LocationReaders implements LocationReader { new GitlabApiReaderProcessor(), new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(), + new AzureApiReaderProcessor(), new UrlReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts new file mode 100644 index 0000000000..35101b2607 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; + +describe('BitbucketApiReaderProcessor', () => { + it('should build raw api', () => { + const processor = new AzureApiReaderProcessor(); + const tests = [ + { + target: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + url: new URL( + 'https://dev.azure.com/org-name/project-name/_apis/sourceProviders/TfsGit/filecontents?repository=repo-name&commitOrBranch=master&path=my-template.yaml&api-version=6.0-preview.1', + ), + err: undefined, + }, + { + target: 'https://api.com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', + }, + { + target: 'com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + ]; + + for (const test of tests) { + if (test.err) { + expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); + } else if (test.url) { + expect(processor.buildRawUrl(test.target).toString()).toEqual( + test.url.toString(), + ); + } else { + throw new Error( + 'This should not have happened. Either err or url should have matched.', + ); + } + } + }); + + it('should return request options', () => { + const tests = [ + { + token: '0123456789', + expect: { + headers: { + Authorization: 'Basic OjAxMjM0NTY3ODk=', + }, + }, + }, + { + token: '', + expect: { + headers: {}, + }, + }, + ]; + + for (const test of tests) { + process.env.AZURE_PRIVATE_TOKEN = test.token; + const processor = new AzureApiReaderProcessor(); + expect(processor.getRequestOptions()).toEqual(test.expect); + } + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts new file mode 100644 index 0000000000..41b4336b4f --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class AzureApiReaderProcessor implements LocationProcessor { + private privateToken: string = process.env.AZURE_PRIVATE_TOKEN || ''; + + getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.privateToken !== '') { + headers.Authorization = `Basic ${Buffer.from( + `:${this.privateToken}`, + 'utf8', + ).toString('base64')}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'azure/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // Converts + // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents + // to: https://dev.azure.com/{organization}/{project}/_apis/sourceProviders/{providerName}/filecontents?repository={repository}&commitOrBranch={commitOrBranch}&path={path}&api-version=6.0-preview.1 + + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + project, + srcKeyword, + repoName, + ] = url.pathname.split('/'); + + const path = url.searchParams.get('path') || ''; + const ref = url.searchParams.get('version')?.substr(2); + + if ( + url.hostname !== 'dev.azure.com' || + empty !== '' || + userOrOrg === '' || + project === '' || + srcKeyword !== '_git' || + repoName === '' || + path === '' || + ref === '' || + !path.match(/\.yaml$/) + ) { + throw new Error('Wrong Azure Devops URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + userOrOrg, + project, + '_apis', + 'sourceProviders', + 'TfsGit', + 'filecontents', + ].join('/'); + + url.search = [ + `repository=${repoName}`, + `commitOrBranch=${ref}`, + `path=${path}`, + 'api-version=6.0-preview.1', + ].join('&'); + + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx index c1ea9c454d..a61469e969 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx @@ -71,7 +71,7 @@ const RegisterComponentForm: FC = ({ onSubmit, submitting }) => { name="componentLocation" required margin="normal" - helperText="Enter the full path to the component.yaml file in GitHub to start tracking your component. It must be in a public repo." + helperText="Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config." inputRef={register({ required: true, validate: ComponentIdValidators, diff --git a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx index 85d63fe2cb..be497e9aa7 100644 --- a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx +++ b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx @@ -79,7 +79,18 @@ const RegisterComponentPage: FC<{}> = () => { setFormState(FormStates.Submitting); const { componentLocation: target } = formData; try { - const data = await catalogApi.addLocation('github', target); + var typeMapping = [ + { url: /https:\/\/gitlab\.com\/.*/, type: 'gitlab' }, + { url: /https:\/\/bitbucket\.org\/.*/, type: 'bitbucket/api' }, + { url: /https:\/\/dev\.azure\.com\/.*/, type: 'azure/api' }, + { url: /.*/, type: 'github' }, + ]; + + var type = typeMapping.filter(function (item) { + return new RegExp(item.url).test(target); + })[0].type; + + const data = await catalogApi.addLocation(type, target); if (!isMounted()) return; From c33fc6c356227a1f5b034350630a879c413e85be Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 00:41:58 +1200 Subject: [PATCH 008/103] Fix linting errors --- .../RegisterComponentPage/RegisterComponentPage.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx index be497e9aa7..1f0eb4fead 100644 --- a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx +++ b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx @@ -79,14 +79,14 @@ const RegisterComponentPage: FC<{}> = () => { setFormState(FormStates.Submitting); const { componentLocation: target } = formData; try { - var typeMapping = [ + const typeMapping = [ { url: /https:\/\/gitlab\.com\/.*/, type: 'gitlab' }, { url: /https:\/\/bitbucket\.org\/.*/, type: 'bitbucket/api' }, { url: /https:\/\/dev\.azure\.com\/.*/, type: 'azure/api' }, { url: /.*/, type: 'github' }, ]; - var type = typeMapping.filter(function (item) { + const type = typeMapping.filter(item => { return new RegExp(item.url).test(target); })[0].type; From 7e454952f7e623fde4ea0437ed98e08c9a6ac0b8 Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 00:59:03 +1200 Subject: [PATCH 009/103] fix broken UI test --- .../RegisterComponentForm/RegisterComponentForm.test.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx index 01402ca19b..fd7a1cc0f8 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx @@ -30,14 +30,14 @@ const setup = (props?: Partial) => { ), }; }; -describe('RegisterComponentForm', () => { +fdescribe('RegisterComponentForm', () => { afterEach(() => cleanup()); it('should initially render a disabled button', async () => { const { rendered } = setup(); expect( await rendered.findByText( - 'Enter the full path to the component.yaml file in GitHub to start tracking your component. It must be in a public repo.', + 'Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.', ), ).toBeInTheDocument(); From 54b6a9cc06d17e3365096c3314c4e78e4bb87cca Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 01:02:32 +1200 Subject: [PATCH 010/103] undo test focus --- .../RegisterComponentForm/RegisterComponentForm.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx index fd7a1cc0f8..d4d2cf4789 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx @@ -30,7 +30,7 @@ const setup = (props?: Partial) => { ), }; }; -fdescribe('RegisterComponentForm', () => { +describe('RegisterComponentForm', () => { afterEach(() => cleanup()); it('should initially render a disabled button', async () => { From f57f6a7402965d7c95eb0b73af2608f779770e99 Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 14:26:39 +1200 Subject: [PATCH 011/103] modify ends with .yaml check to contains .yaml --- .../processors/AzureApiReaderProcessor.test.ts | 2 +- .../ingestion/processors/YamlProcessor.test.ts | 16 ++++++++++++++++ .../src/ingestion/processors/YamlProcessor.ts | 2 +- .../RegisterComponentPage.tsx | 4 +--- .../register-component/src/util/validate.test.ts | 3 ++- plugins/register-component/src/util/validate.ts | 4 ++-- 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts index 35101b2607..0ef111fcb1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts @@ -16,7 +16,7 @@ import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; -describe('BitbucketApiReaderProcessor', () => { +describe('AzureApiReaderProcessor', () => { it('should build raw api', () => { const processor = new AzureApiReaderProcessor(); const tests = [ diff --git a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts index 587793a6de..9f1ede3f9e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts @@ -51,6 +51,22 @@ describe('YamlProcessor', () => { expect(never).not.toBeCalled(); }); + it('should process url that contains yaml', async () => { + const containsYamlLocationSpec = { + type: 'url', + target: 'http://example.com/component?path=test.yaml&c=1&d=2', + }; + + const buffer = Buffer.from([]); + const emit = jest.fn(); + + expect( + await processor.parseData(buffer, containsYamlLocationSpec, emit), + ).toBe(true); + + expect(emit).toBeCalled(); + }); + it('should process entity with yaml', async () => { const entity = { apiVersion: 'backstage.io/v1alpha1', diff --git a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts index 6a2b5cf419..79ae55fae1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts @@ -26,7 +26,7 @@ export class YamlProcessor implements LocationProcessor { location: LocationSpec, emit: LocationProcessorEmit, ): Promise { - if (!location.target.match(/\.ya?ml$/)) { + if (!location.target.match(/\.ya?ml/)) { return false; } diff --git a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx index 1f0eb4fead..e61f904347 100644 --- a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx +++ b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx @@ -86,9 +86,7 @@ const RegisterComponentPage: FC<{}> = () => { { url: /.*/, type: 'github' }, ]; - const type = typeMapping.filter(item => { - return new RegExp(item.url).test(target); - })[0].type; + const type = typeMapping.filter(item => item.url.test(target))[0].type; const data = await catalogApi.addLocation(type, target); diff --git a/plugins/register-component/src/util/validate.test.ts b/plugins/register-component/src/util/validate.test.ts index d062655bd2..e4fc90699b 100644 --- a/plugins/register-component/src/util/validate.test.ts +++ b/plugins/register-component/src/util/validate.test.ts @@ -31,11 +31,12 @@ describe('ComponentIdValidators', () => { }); }); describe('yamlValidator', () => { - const errorMessage = "Must end with '.yaml'."; + const errorMessage = "Must contain '.yaml'."; test.each([ [true, '.yaml'], [true, 'http://example.com/blob/master/service.yaml'], [true, 'https://example.yaml'], + [true, 'https://example.com?path=abc.yaml&c=1'], [errorMessage, '.yml'], [errorMessage, 'http://example.com/blob/master/service'], [errorMessage, undefined], diff --git a/plugins/register-component/src/util/validate.ts b/plugins/register-component/src/util/validate.ts index 78d20995f5..8552872015 100644 --- a/plugins/register-component/src/util/validate.ts +++ b/plugins/register-component/src/util/validate.ts @@ -19,6 +19,6 @@ export const ComponentIdValidators = { (typeof value === 'string' && value.match(/^https:\/\//) !== null) || 'Must start with https://.', yamlValidator: (value: any) => - (typeof value === 'string' && value.match(/.yaml$/) !== null) || - "Must end with '.yaml'.", + (typeof value === 'string' && value.match(/.yaml/) !== null) || + "Must contain '.yaml'.", }; From ba8bef38a637a3544b0aaae56a839fefcb393e56 Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Tue, 25 Aug 2020 18:30:12 +1200 Subject: [PATCH 012/103] update casing --- .../RegisterComponentForm/RegisterComponentForm.test.tsx | 2 +- .../components/RegisterComponentForm/RegisterComponentForm.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx index d4d2cf4789..e1226bbf32 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx @@ -37,7 +37,7 @@ describe('RegisterComponentForm', () => { const { rendered } = setup(); expect( await rendered.findByText( - 'Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.', + 'Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.', ), ).toBeInTheDocument(); diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx index a61469e969..de3c54610d 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx @@ -71,7 +71,7 @@ const RegisterComponentForm: FC = ({ onSubmit, submitting }) => { name="componentLocation" required margin="normal" - helperText="Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config." + helperText="Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config." inputRef={register({ required: true, validate: ComponentIdValidators, From 118e4861be8d791119614ea8acaa7fbc51aed366 Mon Sep 17 00:00:00 2001 From: Tim Jacomb Date: Tue, 25 Aug 2020 11:27:56 +0100 Subject: [PATCH 013/103] Use cookie cutter installed on host --- plugins/scaffolder-backend/package.json | 1 + .../stages/templater/cookiecutter.ts | 43 ++++++++++++------- .../scaffolder/stages/templater/helpers.ts | 43 +++++++++++++++++++ yarn.lock | 5 +++ 4 files changed, 76 insertions(+), 16 deletions(-) diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 7112d0f49d..275da3700b 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -27,6 +27,7 @@ "@octokit/rest": "^18.0.0", "@types/dockerode": "^2.5.32", "@types/express": "^4.17.6", + "command-exists-promise": "^2.0.2", "compression": "^1.7.4", "cors": "^2.8.5", "dockerode": "^3.2.0", diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts index fff349cf82..21b6fc3dde 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts @@ -15,11 +15,13 @@ */ import fs from 'fs-extra'; import { JsonValue } from '@backstage/config'; -import { runDockerContainer } from './helpers'; +import { runDockerContainer, runCommand } from './helpers'; import { TemplaterBase, TemplaterRunOptions } from '.'; import path from 'path'; import { TemplaterRunResult } from './types'; +const commandExists = require('command-exists-promise'); + export class CookieCutter implements TemplaterBase { private async fetchTemplateCookieCutter( directory: string, @@ -51,21 +53,30 @@ export class CookieCutter implements TemplaterBase { const templateDir = options.directory; const resultDir = await fs.promises.mkdtemp(`${options.directory}-result`); - await runDockerContainer({ - imageName: 'spotify/backstage-cookiecutter', - args: [ - 'cookiecutter', - '--no-input', - '-o', - '/result', - '/template', - '--verbose', - ], - templateDir, - resultDir, - logStream: options.logStream, - dockerClient: options.dockerClient, - }); + const cookieCutterInstalled = await commandExists('cookiecutter'); + if (cookieCutterInstalled) { + await runCommand({ + command: 'cookiecutter', + args: ['--no-input', '-o', resultDir, templateDir, '--verbose'], + logStream: options.logStream, + }); + } else { + await runDockerContainer({ + imageName: 'spotify/backstage-cookiecutter', + args: [ + 'cookiecutter', + '--no-input', + '-o', + '/result', + '/template', + '--verbose', + ], + templateDir, + resultDir, + logStream: options.logStream, + dockerClient: options.dockerClient, + }); + } return { resultDir: path.resolve(resultDir, options.values.component_id as string), diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts index a3dea7ea5a..e71b63bfb0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts @@ -18,6 +18,7 @@ import Docker from 'dockerode'; import fs from 'fs'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { InputError } from '@backstage/backend-common'; +import { spawn } from 'child_process'; export type RunDockerContainerOptions = { imageName: string; @@ -29,6 +30,12 @@ export type RunDockerContainerOptions = { createOptions?: Docker.ContainerCreateOptions; }; +export type RunCommandOptions = { + command: string; + args: string[]; + logStream?: Writable; +}; + /** * Gets the templater key to use for templating from the entity * @param entity Template entity @@ -43,6 +50,42 @@ export const getTemplaterKey = (entity: TemplateEntityV1alpha1): string => { return templater; }; +/** + * + * @param options the options object + * @param options.command the command to run + * @param options.args the arguments to pass the command + * @param options.logStream the log streamer to capture log messages + */ +export const runCommand = async ({ + command, + args, + logStream = new PassThrough(), +}: RunCommandOptions) => { + await new Promise((resolve, reject) => { + const process = spawn(command, args); + + process.stdout.on('data', stream => { + logStream.write(stream); + }); + + process.stderr.on('data', stream => { + logStream.write(stream); + }); + + process.on('error', error => { + return reject(error); + }); + + process.on('close', code => { + if (code !== 0) { + return reject(`Command ${command} failed, exit code: ${code}`); + } + return resolve(); + }); + }); +}; + /** * * @param options the options object diff --git a/yarn.lock b/yarn.lock index 857b8b2cc0..6f725fe353 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8104,6 +8104,11 @@ comma-separated-tokens@^1.0.0: resolved "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz#632b80b6117867a158f1080ad498b2fbe7e3f5ea" integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw== +command-exists-promise@^2.0.2: + version "2.0.2" + resolved "https://registry.npmjs.org/command-exists-promise/-/command-exists-promise-2.0.2.tgz#7beecc4b218299f3c61fa69a4047aa0b36a64a99" + integrity sha512-T6PB6vdFrwnHXg/I0kivM3DqaCGZLjjYSOe0a5WgFKcz1sOnmOeIjnhQPXVXX3QjVbLyTJ85lJkX6lUpukTzaA== + commander@^2.11.0, commander@^2.19.0, commander@^2.20.0, commander@^2.20.3, commander@^2.8.1, commander@~2.20.3: version "2.20.3" resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" From 3ca9863f039e9f86c2c567fcf47e50ba49061bd1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 25 Aug 2020 15:24:02 +0200 Subject: [PATCH 014/103] cli: introduce jest transformModules options for providing a list of modules to transform --- package.json | 5 ++ packages/cli/config/jest.js | 87 +++++++++++-------- .../templates/default-app/package.json.hbs | 5 ++ 3 files changed, 60 insertions(+), 37 deletions(-) diff --git a/package.json b/package.json index e4aa18f540..e6ee961701 100644 --- a/package.json +++ b/package.json @@ -60,5 +60,10 @@ "*.{json,md}": [ "prettier --write" ] + }, + "jest": { + "transformModules": [ + "@kyma-project/asyncapi-react" + ] } } diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index cc7d45c139..f69e18190d 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -25,43 +25,6 @@ async function getConfig() { return require(path.resolve('jest.config.ts')); } - const options = { - rootDir: path.resolve('src'), - coverageDirectory: path.resolve('coverage'), - collectCoverageFrom: ['**/*.{js,jsx,ts,tsx}', '!**/*.d.ts'], - moduleNameMapper: { - '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'), - }, - - // We build .esm.js files with plugin:build, so to be able to load these in tests they need to be transformed - // TODO: jest is working on module support, it's possible that we can remove this in the future - transform: { - '\\.esm\\.js$': require.resolve('jest-esm-transformer'), - '\\.(js|jsx|ts|tsx)$': [ - require.resolve('ts-jest'), - { isolatedModules: true }, - ], - '\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg)$': require.resolve( - './jestFileTransform.js', - ), - }, - - // A bit more opinionated - testMatch: ['**/?(*.)test.{js,jsx,mjs,ts,tsx}'], - - // Default behaviour is to not apply transforms for node_modules, but we still want - // to apply the esm-transformer to .esm.js files, since that's what we use in backstage packages. - // The @kyma-project/asyncapi-react library needs to be transformed. - transformIgnorePatterns: [ - '/node_modules/(?!@kyma-project/asyncapi-react/)(?!.*\\.(?:esm\\.js|bmp|gif|jpg|jpeg|png|frag|xml|svg)$)', - ], - }; - - // Use src/setupTests.ts as the default location for configuring test env - if (fs.existsSync('src/setupTests.ts')) { - options.setupFilesAfterEnv = ['/setupTests.ts']; - } - // We read all "jest" config fields in package.json files all the way to the filesystem root. // All configs are merged together to create the final config, with longer paths taking precedence. // The merging of the configs is shallow, meaning e.g. all transforms are replaced if new ones are defined. @@ -92,6 +55,56 @@ async function getConfig() { currentPath = newPath; } + // We add an additional Jest config parameter only known by the Backstage CLI + // called `transformModules`. It's a list of modules that we want to apply + // our configured jest transformations for. + // This is useful when packages are published in untranspiled ESM or TS form. + const transformModules = pkgJsonConfigs + .flatMap(conf => { + const modules = conf.transformModules || []; + delete conf.transformModules; + return modules; + }) + .map(name => `${name}/`) + .join('|'); + const transformModulePattern = transformModules && `(?!${transformModules})`; + + const options = { + rootDir: path.resolve('src'), + coverageDirectory: path.resolve('coverage'), + collectCoverageFrom: ['**/*.{js,jsx,ts,tsx}', '!**/*.d.ts'], + moduleNameMapper: { + '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'), + }, + + // We build .esm.js files with plugin:build, so to be able to load these in tests they need to be transformed + // TODO: jest is working on module support, it's possible that we can remove this in the future + transform: { + '\\.esm\\.js$': require.resolve('jest-esm-transformer'), + '\\.(js|jsx|ts|tsx)$': [ + require.resolve('ts-jest'), + { isolatedModules: true }, + ], + '\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg)$': require.resolve( + './jestFileTransform.js', + ), + }, + + // A bit more opinionated + testMatch: ['**/?(*.)test.{js,jsx,mjs,ts,tsx}'], + + // Default behaviour is to not apply transforms for node_modules, but we still want + // to apply the esm-transformer to .esm.js files, since that's what we use in backstage packages. + transformIgnorePatterns: [ + `/node_modules/${transformModulePattern}(?:(?!\\.esm).)*\\.(?:js|json)$`, + ], + }; + + // Use src/setupTests.ts as the default location for configuring test env + if (fs.existsSync('src/setupTests.ts')) { + options.setupFilesAfterEnv = ['/setupTests.ts']; + } + return Object.assign(options, ...pkgJsonConfigs); } diff --git a/packages/create-app/templates/default-app/package.json.hbs b/packages/create-app/templates/default-app/package.json.hbs index b59ca63fa3..af4384318a 100644 --- a/packages/create-app/templates/default-app/package.json.hbs +++ b/packages/create-app/templates/default-app/package.json.hbs @@ -42,5 +42,10 @@ "*.{json,md}": [ "prettier --write" ] + }, + "jest": { + "transformModules": [ + "@kyma-project/asyncapi-react" + ] } } From 96b8971b69db768054b5ae8fa0f4d69e4d77f07b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2020 08:26:20 +0200 Subject: [PATCH 015/103] chore(deps): bump @kyma-project/asyncapi-react from 0.11.0 to 0.11.2 (#2121) Bumps [@kyma-project/asyncapi-react](https://github.com/asyncapi/asyncapi-react) from 0.11.0 to 0.11.2. - [Release notes](https://github.com/asyncapi/asyncapi-react/releases) - [Commits](https://github.com/asyncapi/asyncapi-react/compare/v0.11.0...v0.11.2) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 959155614a..000bfa9fb9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2172,9 +2172,9 @@ stream "^0.0.2" "@kyma-project/asyncapi-react@^0.11.0": - version "0.11.0" - resolved "https://registry.npmjs.org/@kyma-project/asyncapi-react/-/asyncapi-react-0.11.0.tgz#888fbe9204f120fc04fd8184664e8025d8736c92" - integrity sha512-CJu9vJ4tTjk3oRjYvZxE4CRLE/QvFsyvCnecZSXWKNiytLK/JuQxkosyFvKpCKsB+uGsPFn0sfcLPZ5jVSEFYw== + version "0.11.2" + resolved "https://registry.npmjs.org/@kyma-project/asyncapi-react/-/asyncapi-react-0.11.2.tgz#b523b0843da7c29a0d7569b3f31d7d8cbdf84b47" + integrity sha512-Wo6CgM3pzZaRMCVwXaEIrlXMuFtRNGli1GtFgMHQUB686daSCqSvRZUyitDHTZLofL3OI9JeYh5m6SjPji6bxg== dependencies: "@asyncapi/avro-schema-parser" "^0.1.2" "@asyncapi/openapi-schema-parser" "^2.0.0" From 14323967e244868f31f43c471f22d3c966253f8c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2020 06:35:43 +0000 Subject: [PATCH 016/103] chore(deps): bump inquirer from 7.2.0 to 7.3.3 Bumps [inquirer](https://github.com/SBoudrias/Inquirer.js) from 7.2.0 to 7.3.3. - [Release notes](https://github.com/SBoudrias/Inquirer.js/releases) - [Commits](https://github.com/SBoudrias/Inquirer.js/compare/inquirer@7.2.0...inquirer@7.3.3) Signed-off-by: dependabot-preview[bot] --- yarn.lock | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/yarn.lock b/yarn.lock index 000bfa9fb9..8e24dba10d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7916,6 +7916,11 @@ cli-width@^2.0.0: resolved "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz#ff19ede8a9a5e579324147b0c11f0fbcbabed639" integrity sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk= +cli-width@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz#a2f48437a2caa9a22436e794bf071ec9e61cedf6" + integrity sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw== + clipboard@^2.0.0: version "2.0.6" resolved "https://registry.npmjs.org/clipboard/-/clipboard-2.0.6.tgz#52921296eec0fdf77ead1749421b21c968647376" @@ -13081,20 +13086,20 @@ inquirer@^6.2.0: through "^2.3.6" inquirer@^7.0.0, inquirer@^7.0.4: - version "7.2.0" - resolved "https://registry.npmjs.org/inquirer/-/inquirer-7.2.0.tgz#63ce99d823090de7eb420e4bb05e6f3449aa389a" - integrity sha512-E0c4rPwr9ByePfNlTIB8z51kK1s2n6jrHuJeEHENl/sbq2G/S1auvibgEwNR4uSyiU+PiYHqSwsgGiXjG8p5ZQ== + version "7.3.3" + resolved "https://registry.npmjs.org/inquirer/-/inquirer-7.3.3.tgz#04d176b2af04afc157a83fd7c100e98ee0aad003" + integrity sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA== dependencies: ansi-escapes "^4.2.1" - chalk "^3.0.0" + chalk "^4.1.0" cli-cursor "^3.1.0" - cli-width "^2.0.0" + cli-width "^3.0.0" external-editor "^3.0.3" figures "^3.0.0" - lodash "^4.17.15" + lodash "^4.17.19" mute-stream "0.0.8" run-async "^2.4.0" - rxjs "^6.5.3" + rxjs "^6.6.0" string-width "^4.1.0" strip-ansi "^6.0.0" through "^2.3.6" @@ -20196,10 +20201,10 @@ run-queue@^1.0.0, run-queue@^1.0.3: dependencies: aproba "^1.1.1" -rxjs@^6.3.3, rxjs@^6.4.0, rxjs@^6.5.2, rxjs@^6.5.3, rxjs@^6.5.5: - version "6.6.0" - resolved "https://registry.npmjs.org/rxjs/-/rxjs-6.6.0.tgz#af2901eedf02e3a83ffa7f886240ff9018bbec84" - integrity sha512-3HMA8z/Oz61DUHe+SdOiQyzIf4tOx5oQHmMir7IZEu6TMqCLHT4LRcmNaUS0NwOz8VLvmmBduMsoaUvMaIiqzg== +rxjs@^6.3.3, rxjs@^6.4.0, rxjs@^6.5.2, rxjs@^6.5.3, rxjs@^6.5.5, rxjs@^6.6.0: + version "6.6.2" + resolved "https://registry.npmjs.org/rxjs/-/rxjs-6.6.2.tgz#8096a7ac03f2cc4fe5860ef6e572810d9e01c0d2" + integrity sha512-BHdBMVoWC2sL26w//BCu3YzKT4s2jip/WhwsGEDmeKYBhKDZeYezVUnHatYB7L85v5xs0BAQmg6BEYJEKxBabg== dependencies: tslib "^1.9.0" From 213a1c3c840559c311c0e45d9d255fc8275aeeab Mon Sep 17 00:00:00 2001 From: Twisha Saraiya Date: Sun, 9 Aug 2020 21:53:40 +0530 Subject: [PATCH 017/103] add button to register dummy entities --- .../components/CatalogPage/CatalogPage.tsx | 60 +++++++++++++++++-- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx b/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx index 6a3d7362aa..3eac1b7a30 100644 --- a/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx +++ b/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx @@ -14,28 +14,30 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import { + configApiRef, Content, ContentHeader, identityApiRef, SupportButton, - configApiRef, useApi, } from '@backstage/core'; import { rootRoute as scaffolderRootRoute } from '@backstage/plugin-scaffolder'; import { Button, makeStyles } from '@material-ui/core'; import SettingsIcon from '@material-ui/icons/Settings'; import StarIcon from '@material-ui/icons/Star'; -import React, { useMemo, useState } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { Link as RouterLink } from 'react-router-dom'; +import { catalogApiRef } from '../../api/types'; import { EntityFilterGroupsProvider, useFilteredEntities } from '../../filter'; import { useStarredEntities } from '../../hooks/useStarredEntites'; -import { CatalogFilter, ButtonGroup } from '../CatalogFilter/CatalogFilter'; +import { ButtonGroup, CatalogFilter } from '../CatalogFilter/CatalogFilter'; import { CatalogTable } from '../CatalogTable/CatalogTable'; +import { ResultsFilter } from '../ResultsFilter/ResultsFilter'; import CatalogLayout from './CatalogLayout'; import { CatalogTabs, LabeledComponentType } from './CatalogTabs'; import { WelcomeBanner } from './WelcomeBanner'; -import { ResultsFilter } from '../ResultsFilter/ResultsFilter'; const useStyles = makeStyles(theme => ({ contentWrapper: { @@ -44,6 +46,9 @@ const useStyles = makeStyles(theme => ({ gridTemplateColumns: '250px 1fr', gridColumnGap: theme.spacing(2), }, + mockDataButton: { + marginRight: '20px', + }, })); const CatalogPageContents = () => { @@ -58,9 +63,42 @@ const CatalogPageContents = () => { const userId = useApi(identityApiRef).getUserId(); const [selectedTab, setSelectedTab] = useState(); const [selectedSidebarItem, setSelectedSidebarItem] = useState(); + const [entitiesState, setEntitiesState] = useState([]); + const [errorState, setError] = useState(); + + useEffect(() => { + setError(error); + setEntitiesState(matchingEntities); + }, [error, matchingEntities]); + const catalogApi = useApi(catalogApiRef); const orgName = useApi(configApiRef).getOptionalString('organization.name') ?? 'Company'; + const addMockData = async () => { + try { + const dummyEntities = [ + 'artist-lookup-component.yaml', + 'playback-order-component.yaml', + 'podcast-api-component.yaml', + 'queue-proxy-component.yaml', + 'searcher-component.yaml', + 'playback-lib-component.yaml', + 'www-artist-component.yaml', + 'shuffle-api-component.yaml', + ]; + const _promises = dummyEntities.map(file => + catalogApi.addLocation( + 'github', + `https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/${file}`, + ), + ); + await Promise.all(_promises); + const data: Entity[] = await catalogApi.getEntities(); + setEntitiesState(data); + } catch (err) { + setError(err); + } + }; const tabs = useMemo( () => [ { @@ -129,6 +167,16 @@ const CatalogPageContents = () => { + {entitiesState && entitiesState.length === 0 ? ( + + ) : null} - ) : null} diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx index 1b84b79234..4090b2af9c 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx @@ -49,7 +49,7 @@ const RegisterComponentForm: FC = ({ onSubmit, submitting }) => { }); const classes = useStyles(); const hasErrors = !!errors.componentLocation; - const dirty = formState?.dirty; + const dirty = formState?.isDirty; return submitting ? ( From 6f3b0b8c20e9d1c5309f20470f2c3acc2a1b0ed3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 25 Aug 2020 18:25:56 +0200 Subject: [PATCH 092/103] catalog-backend: initial catalog ingestion rules implementation --- .../src/ingestion/CatalogRules.test.ts | 114 +++++++++++ .../src/ingestion/CatalogRules.ts | 187 ++++++++++++++++++ .../src/ingestion/LocationReaders.ts | 24 ++- 3 files changed, 320 insertions(+), 5 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/CatalogRules.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/CatalogRules.ts diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts new file mode 100644 index 0000000000..7f2ccdc254 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -0,0 +1,114 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec, Entity } from '@backstage/catalog-model'; +import { CatalogRulesEnforcer } from './CatalogRules'; + +const entity = { + user: { + kind: 'User', + } as Entity, + group: { + kind: 'Group', + } as Entity, + component: { + kind: 'component', + } as Entity, +}; + +const location: Record = { + x: { + type: 'github', + target: 'https://github.com/a/b/blob/master/x.yaml', + }, + y: { + type: 'github', + target: 'https://github.com/a/b/blob/master/y.yaml', + }, + z: { + type: 'file', + target: '/root/z.yaml', + }, +}; + +describe('CatalogRulesEnforcer', () => { + it('should allow by default', () => { + const enforcer = new CatalogRulesEnforcer([]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should deny all', () => { + const enforcer = new CatalogRulesEnforcer([{ allow: [] }]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); + + it('should allow all with override', () => { + const enforcer = new CatalogRulesEnforcer([{ allow: [] }, { deny: [] }]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should deny groups', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [], deny: [{ kind: 'Group' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should deny groups from github', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [], deny: [{ kind: 'Group' }], locations: [{ type: 'github' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should override to allow groups from files', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [], deny: [{ kind: 'Group' }] }, + { allow: [{ kind: 'Group' }], deny: [], locations: [{ type: 'file' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should not be sensitive to kind case', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [], deny: [{ kind: 'group' }] }, + { allow: [], deny: [{ kind: 'Component' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts new file mode 100644 index 0000000000..a03d93852a --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -0,0 +1,187 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { LocationSpec, Entity } from '@backstage/catalog-model'; + +/** + * A structure for matching entities to a given rule. + */ +type EntityMatcher = { + kind: string; +}; + +/** + * A structure for matching locations to a given rule. + */ +type LocationMatcher = { + target?: string; + type: string; +}; + +/** + * Rules to apply to catalog entities + * + * An undefined list of matchers means match all, an empty list of matchers means match none + */ +type CatalogRule = { + deny?: EntityMatcher[]; + allow?: EntityMatcher[]; + locations?: LocationMatcher[]; +}; + +export class CatalogRulesEnforcer { + /** + * Default rules used by the catalog. + * + * Denies any location from specifying user or group entities. + */ + static readonly defaultRules: CatalogRule[] = [ + { + deny: [{ kind: 'User' }, { kind: 'Group' }], + allow: [], + }, + ]; + + /** + * Loads catalog rules from config. + * + * This reads `catalog.rules` and defaults to the default rules if no value is present. + * The value of the config should be a list of config objects, each with a single `deny` + * field which in turn is a list of entity kind to deny. + * + * It also reads in rules from `catalog.locations`, where each location can have a list + * of allowed entity for the location, specified in an `allow` field. + * + * For example: + * + * ```yaml + * catalog: + * rules: + * - deny: [User, Group, System] + * + * locations: + * - type: github + * target: https://github.com/org/repo/blob/master/users.yaml + * allow: [User, Group] + * - type: github + * target: https://github.com/org/repo/blob/master/systems.yaml + * allow: [System] + * ``` + */ + static fromConfig(config: Config) { + const rules = new Array(); + + if (config.has('catalog.rules')) { + const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ + deny: sub.getStringArray('deny').map(kind => ({ kind })), + allow: [], + })); + rules.push(...globalRules); + } else { + rules.push(...CatalogRulesEnforcer.defaultRules); + } + + if (config.has('catalog.locations')) { + const locationRules = config + .getConfigArray('catalog.locations') + .flatMap(sub => { + if (!sub.has('allow')) { + return []; + } + + return [ + { + deny: [], + allow: sub.getStringArray('allow').map(kind => ({ kind })), + locations: [ + { + type: sub.getString('type'), + target: sub.getString('target'), + }, + ], + }, + ]; + }); + + rules.push(...locationRules); + } + + return new CatalogRulesEnforcer(rules); + } + + constructor(private readonly rules: CatalogRule[]) {} + + /** + * Checks wether a specific entity/location combination is allowed + * according to the configured rules. + */ + isAllowed(entity: Entity, location: LocationSpec) { + let result = true; + + for (const rule of this.rules) { + if (!this.matchLocation(location, rule.locations)) { + continue; + } + + if (this.matchEntity(entity, rule.allow)) { + result = true; + } + if (this.matchEntity(entity, rule.deny)) { + result = false; + } + } + + return result; + } + + private matchLocation( + location: LocationSpec, + matchers?: LocationMatcher[], + ): boolean { + if (!matchers) { + return true; + } + + for (const matcher of matchers) { + if (matcher.type !== location.type) { + continue; + } + if (matcher.target && matcher.target !== location.target) { + continue; + } + return true; + } + + return false; + } + + private matchEntity(entity: Entity, matchers?: EntityMatcher[]): boolean { + if (!matchers) { + return true; + } + + for (const matcher of matchers) { + if (entity.kind.toLowerCase() !== matcher.kind.toLowerCase()) { + continue; + } + + return true; + } + + return false; + } +} diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 46fa6781f1..bddef6b4de 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -47,6 +47,7 @@ import { } from './processors/types'; import { YamlProcessor } from './processors/YamlProcessor'; import { LocationReader, ReadLocationResult } from './types'; +import { CatalogRulesEnforcer } from './CatalogRules'; // The max amount of nesting depth of generated work items const MAX_DEPTH = 10; @@ -63,6 +64,7 @@ type Options = { export class LocationReaders implements LocationReader { private readonly logger: Logger; private readonly processors: LocationProcessor[]; + private readonly rulesEnforcer: CatalogRulesEnforcer; static defaultProcessors(options: { config?: Config; @@ -96,6 +98,9 @@ export class LocationReaders implements LocationReader { }: Options) { this.logger = logger; this.processors = processors; + this.rulesEnforcer = config + ? CatalogRulesEnforcer.fromConfig(config) + : new CatalogRulesEnforcer(CatalogRulesEnforcer.defaultRules); } async read(location: LocationSpec): Promise { @@ -112,11 +117,20 @@ export class LocationReaders implements LocationReader { } else if (item.type === 'data') { await this.handleData(item, emit); } else if (item.type === 'entity') { - const entity = await this.handleEntity(item, emit); - output.entities.push({ - entity, - location: item.location, - }); + if (this.rulesEnforcer.isAllowed(item.entity, item.location)) { + const entity = await this.handleEntity(item, emit); + output.entities.push({ + entity, + location: item.location, + }); + } else { + output.errors.push({ + location: item.location, + error: new Error( + `Entity of kind ${item.entity.kind} is not allowed from location ${item.location.target}:${item.location.type}`, + ), + }); + } } else if (item.type === 'error') { await this.handleError(item, emit); output.errors.push({ From 8d07b541d7eb037cea5a8a79c01135a02b98cab9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 27 Aug 2020 20:54:56 +0200 Subject: [PATCH 093/103] catalog-backend: switch catalog rules to deny by default --- .../src/ingestion/CatalogRules.test.ts | 42 ++++++++++--------- .../src/ingestion/CatalogRules.ts | 27 +++++------- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 7f2ccdc254..1168db1080 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -45,11 +45,11 @@ const location: Record = { }; describe('CatalogRulesEnforcer', () => { - it('should allow by default', () => { + it('should deny by default', () => { const enforcer = new CatalogRulesEnforcer([]); - expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); - expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); - expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); }); it('should deny all', () => { @@ -59,8 +59,10 @@ describe('CatalogRulesEnforcer', () => { expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); }); - it('should allow all with override', () => { - const enforcer = new CatalogRulesEnforcer([{ allow: [] }, { deny: [] }]); + it('should allow all', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [{ kind: 'User' }, { kind: 'Group' }, { kind: 'Component' }] }, + ]); expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); @@ -68,7 +70,7 @@ describe('CatalogRulesEnforcer', () => { it('should deny groups', () => { const enforcer = new CatalogRulesEnforcer([ - { allow: [], deny: [{ kind: 'Group' }] }, + { allow: [{ kind: 'User' }, { kind: 'Component' }] }, ]); expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); @@ -79,7 +81,8 @@ describe('CatalogRulesEnforcer', () => { it('should deny groups from github', () => { const enforcer = new CatalogRulesEnforcer([ - { allow: [], deny: [{ kind: 'Group' }], locations: [{ type: 'github' }] }, + { allow: [{ kind: 'User' }, { kind: 'Component' }] }, + { allow: [{ kind: 'Group' }], locations: [{ type: 'file' }] }, ]); expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); @@ -88,27 +91,26 @@ describe('CatalogRulesEnforcer', () => { expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); }); - it('should override to allow groups from files', () => { + it('should allow groups from files', () => { const enforcer = new CatalogRulesEnforcer([ - { allow: [], deny: [{ kind: 'Group' }] }, - { allow: [{ kind: 'Group' }], deny: [], locations: [{ type: 'file' }] }, + { allow: [{ kind: 'Group' }], locations: [{ type: 'file' }] }, ]); - expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); - expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); }); it('should not be sensitive to kind case', () => { const enforcer = new CatalogRulesEnforcer([ - { allow: [], deny: [{ kind: 'group' }] }, - { allow: [], deny: [{ kind: 'Component' }] }, + { allow: [{ kind: 'group' }] }, + { allow: [{ kind: 'Component' }] }, ]); - expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); - expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); - expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); - expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); - expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); }); }); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index a03d93852a..eda93b552f 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -38,8 +38,7 @@ type LocationMatcher = { * An undefined list of matchers means match all, an empty list of matchers means match none */ type CatalogRule = { - deny?: EntityMatcher[]; - allow?: EntityMatcher[]; + allow: EntityMatcher[]; locations?: LocationMatcher[]; }; @@ -51,8 +50,7 @@ export class CatalogRulesEnforcer { */ static readonly defaultRules: CatalogRule[] = [ { - deny: [{ kind: 'User' }, { kind: 'Group' }], - allow: [], + allow: [{ kind: 'Component' }, { kind: 'API' }], }, ]; @@ -60,8 +58,10 @@ export class CatalogRulesEnforcer { * Loads catalog rules from config. * * This reads `catalog.rules` and defaults to the default rules if no value is present. - * The value of the config should be a list of config objects, each with a single `deny` - * field which in turn is a list of entity kind to deny. + * The value of the config should be a list of config objects, each with a single `allow` + * field which in turn is a list of entity kinds to allow. + * + * If there is no matching rule to allow an ingested entity, it will be rejected by the catalog. * * It also reads in rules from `catalog.locations`, where each location can have a list * of allowed entity for the location, specified in an `allow` field. @@ -71,7 +71,7 @@ export class CatalogRulesEnforcer { * ```yaml * catalog: * rules: - * - deny: [User, Group, System] + * - allow: [Component, API] * * locations: * - type: github @@ -87,8 +87,7 @@ export class CatalogRulesEnforcer { if (config.has('catalog.rules')) { const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ - deny: sub.getStringArray('deny').map(kind => ({ kind })), - allow: [], + allow: sub.getStringArray('allow').map(kind => ({ kind })), })); rules.push(...globalRules); } else { @@ -105,7 +104,6 @@ export class CatalogRulesEnforcer { return [ { - deny: [], allow: sub.getStringArray('allow').map(kind => ({ kind })), locations: [ { @@ -130,22 +128,17 @@ export class CatalogRulesEnforcer { * according to the configured rules. */ isAllowed(entity: Entity, location: LocationSpec) { - let result = true; - for (const rule of this.rules) { if (!this.matchLocation(location, rule.locations)) { continue; } if (this.matchEntity(entity, rule.allow)) { - result = true; - } - if (this.matchEntity(entity, rule.deny)) { - result = false; + return true; } } - return result; + return false; } private matchLocation( From 903cbdcfb5120f3414662c51b839e89892404af3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 31 Aug 2020 00:24:18 +0200 Subject: [PATCH 094/103] catalog-backend: add tests for config rules --- .../src/ingestion/CatalogRules.test.ts | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 1168db1080..4a403f8dfa 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -16,6 +16,7 @@ import { LocationSpec, Entity } from '@backstage/catalog-model'; import { CatalogRulesEnforcer } from './CatalogRules'; +import { ConfigReader } from '@backstage/config'; const entity = { user: { @@ -113,4 +114,70 @@ describe('CatalogRulesEnforcer', () => { expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); }); + + describe('fromConfig', () => { + it('should allow components by default', () => { + const enforcer = CatalogRulesEnforcer.fromConfig(new ConfigReader({})); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should deny all', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ catalog: { rules: [] } }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); + + it('should allow all', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [{ allow: ['User', 'Group'] }, { allow: ['Component'] }], + }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should deny groups', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { rules: [{ allow: ['User'] }, { allow: ['Component'] }] }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should allow groups from a specific github location', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [{ allow: ['user'] }], + locations: [ + { + type: 'github', + target: 'https://github.com/a/b/blob/master/x.yaml', + allow: ['Group'], + }, + ], + }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); + }); }); From f546bb3862893d85a7f3cfdf84005d2fd603065e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 31 Aug 2020 01:05:54 +0200 Subject: [PATCH 095/103] docs: added catalog configuration docs --- .../software-catalog/configuration.md | 60 +++++++++++++++++++ docs/features/software-catalog/index.md | 16 +++++ mkdocs.yml | 1 + 3 files changed, 77 insertions(+) create mode 100644 docs/features/software-catalog/configuration.md diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md new file mode 100644 index 0000000000..77b630d347 --- /dev/null +++ b/docs/features/software-catalog/configuration.md @@ -0,0 +1,60 @@ +--- +id: software-catalog-configuration +title: Catalog Configuration +--- + +## Static Location Configuration + +To enable declarative catalog setups, it is possible to add locations to the +catalog via [static configuration](../../conf/index.md). Locations are added to +the catalog under the `catalog.locations` key, for example: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +The locations added through static configuration can not be removed through the +catalog locations API. To remove the locations, you have to remove them from the +configuration. + +## Catalog Rules + +By default the catalog will only allow ingestion of entities with the kind +`Component` and `API`. In order to allow entities of other kinds to be added, +you need to add rules to the catalog. Rules are added either in a separate +`catalog.rules` key, or added to statically configured locations. + +For example, given the following configuration: + +```yaml +catalog: + rules: + - allow: [Component, API] + - allow: [System] + locations: + type: github + + locations: + - type: github + target: https://github.com/org/example/blob/master/org-data.yaml + allow: [Group] +``` + +We are able to add entities of kind `Component` or `API` from any location, +entities of kind `System` from any `github` location, and `Group` entities from +the `org-data.yaml`, which will also be read as statically configured location. + +Note that if the `catalog.rules` key is present it will replace the default +value, meaning that you need to add rules for `Component` and `API` kinds if you +want those to be allowed. + +The following configuration will reject any kind of entities from being added to +the catalog: + +```yaml +catalog: + rules: [] +``` diff --git a/docs/features/software-catalog/index.md b/docs/features/software-catalog/index.md index db8c183390..bfc3d53d4d 100644 --- a/docs/features/software-catalog/index.md +++ b/docs/features/software-catalog/index.md @@ -79,6 +79,22 @@ All software created through the [Backstage Software Templates](../software-templates/index.md) are automatically registered in the catalog. +### Static catalog configuration + +In addition to manually registering components, it is also possible to register +components though [static configuration](../../conf/index.md). For example, the +above example can be added using the following configuration: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +More information about catalog configuration can be found +[here](configuration.md). + ### Updating component metadata Teams owning the components are responsible for maintaining the metadata about diff --git a/mkdocs.yml b/mkdocs.yml index 637e45c0f0..b12c8f81c7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,6 +28,7 @@ nav: - Overview: 'features/software-catalog/index.md' - System model: 'features/software-catalog/system-model.md' - YAML File Format: 'features/software-catalog/descriptor-format.md' + - Configuration: 'features/software-catalog/configuration.md' - Extending the model: 'features/software-catalog/extending-the-model.md' - External integrations: 'features/software-catalog/external-integrations.md' - API: 'features/software-catalog/api.md' From 74e55ef32f173e2e89656bbe03802426453df5f2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 31 Aug 2020 13:40:05 +0200 Subject: [PATCH 096/103] catalog-backend: updated docs to not include location in catalog.rules + test --- docs/features/software-catalog/configuration.md | 11 ++++------- .../src/ingestion/CatalogRules.test.ts | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index 77b630d347..d4cc7fc0c2 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -32,10 +32,7 @@ For example, given the following configuration: ```yaml catalog: rules: - - allow: [Component, API] - - allow: [System] - locations: - type: github + - allow: [Component, API, System] locations: - type: github @@ -43,9 +40,9 @@ catalog: allow: [Group] ``` -We are able to add entities of kind `Component` or `API` from any location, -entities of kind `System` from any `github` location, and `Group` entities from -the `org-data.yaml`, which will also be read as statically configured location. +We are able to add entities of kind `Component`, `API`, or `System` from any +location, and `Group` entities from the `org-data.yaml`, which will also be read +as statically configured location. Note that if the `catalog.rules` key is present it will replace the default value, meaning that you need to add rules for `Component` and `API` kinds if you diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 4a403f8dfa..34484f6b1e 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -179,5 +179,20 @@ describe('CatalogRulesEnforcer', () => { expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); }); + + it('should not care about location configuration in catalog.rules', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [{ allow: ['Group'], locations: [{ type: 'github' }] }], + }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); }); }); From 6a22efc31b3994bd04db93fc21f75c82bf6eefe6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 31 Aug 2020 14:06:10 +0200 Subject: [PATCH 097/103] .github: update issue template to label with enhancement instead of help wanted (#2184) --- .github/ISSUE_TEMPLATE/feature_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_template.md b/.github/ISSUE_TEMPLATE/feature_template.md index 012b8d7a06..d70622bf52 100644 --- a/.github/ISSUE_TEMPLATE/feature_template.md +++ b/.github/ISSUE_TEMPLATE/feature_template.md @@ -1,7 +1,7 @@ --- name: 'Feature Request' about: 'Suggest new features and changes' -labels: help wanted +labels: enhancement --- From b284cb38e345c87fcfd5ccf5a468668cc1fd3e36 Mon Sep 17 00:00:00 2001 From: Iain Billett Date: Mon, 31 Aug 2020 13:30:43 +0100 Subject: [PATCH 098/103] Add Plugins Page to microsite (#2144) * Add Plugins Page to microsite * Updates to microsite Plugins Page * Correct plugin authors and documentation links * Add category to UI * Move documentation from README to the docs website * Use external urls for logos * Updates to microsite Plugins Page * trailing whitespace * Run prettier on new markdown * Updates to microsite Plugins Page * Move plugins link * Clarify category field * Updates to microsite Plugins Page * Add authorUrl field to plugin config * Render author as a muted link like those in the site map * Updates to microsite Plugins Page * Add authorUrl field to example in docs * Updates to microsite Plugins Page * Add npmPackageName field - intended for future use * Updates to microsite Plugins Page * Add npmPackageName field - intended for future use * Updates to microsite Plugins Page * Use correct docs link --- docs/plugins/add-to-marketplace.md | 23 ++++++ microsite/data/plugins/rollbar.yaml | 10 +++ microsite/data/plugins/sentry.yaml | 10 +++ microsite/data/plugins/travis-ci.yaml | 10 +++ microsite/i18n/en.json | 4 + microsite/package.json | 3 +- microsite/pages/en/plugins.js | 84 +++++++++++++++++++ microsite/sidebars.json | 2 +- microsite/siteConfig.js | 4 + microsite/static/css/plugins.css | 111 ++++++++++++++++++++++++++ 10 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 docs/plugins/add-to-marketplace.md create mode 100644 microsite/data/plugins/rollbar.yaml create mode 100644 microsite/data/plugins/sentry.yaml create mode 100644 microsite/data/plugins/travis-ci.yaml create mode 100644 microsite/pages/en/plugins.js create mode 100644 microsite/static/css/plugins.css diff --git a/docs/plugins/add-to-marketplace.md b/docs/plugins/add-to-marketplace.md new file mode 100644 index 0000000000..23cfecd5d0 --- /dev/null +++ b/docs/plugins/add-to-marketplace.md @@ -0,0 +1,23 @@ +--- +id: add-to-marketplace +title: Add to Marketplace +--- + +## Adding a Plugin to the Marketplace + +To add a new plugin to the [plugin marketplace](https://backstage.io/plugins) +create a file in `data/plugins` with your plugin's information. Example: + +```yaml +--- +title: Your Plugin +author: Your Name +authorUrl: # A link to information about the author E.g. Company url, github user profile, etc +category: Monitoring # A single category e.g. CI, Machine Learning, Services, Monitoring +description: A brief description of the plugin. # Max 170 characters +documentation: # A link to your documentation E.g. Your github README +iconUrl: # Used as the src attribute for your logo. +# You can provide an external url or add your logo under static/img and provide a path +# relative to static/ e.g. img/my-logo.png +npmPackageName: # Your npm package name E.g. '@backstage/plugin-' quotes are required +``` diff --git a/microsite/data/plugins/rollbar.yaml b/microsite/data/plugins/rollbar.yaml new file mode 100644 index 0000000000..0118fab391 --- /dev/null +++ b/microsite/data/plugins/rollbar.yaml @@ -0,0 +1,10 @@ +--- +title: Rollbar +author: '@andrewthauer' +authorUrl: https://github.com/andrewthauer +category: Monitoring +description: View Rollbar errors for your services in Backstage. +documentation: https://github.com/spotify/backstage/tree/master/plugins/rollbar +iconUrl: https://rollbar.com/assets/media/rollbar-mark-color.png +npmPackageName: '@backstage/plugin-rollbar' + diff --git a/microsite/data/plugins/sentry.yaml b/microsite/data/plugins/sentry.yaml new file mode 100644 index 0000000000..7ca291acea --- /dev/null +++ b/microsite/data/plugins/sentry.yaml @@ -0,0 +1,10 @@ +--- +title: Sentry +author: Spotify +authorUrl: https://www.spotify.com/ +category: Monitoring +description: View Sentry issues in Backstage. +documentation: https://github.com/spotify/backstage/tree/master/plugins/sentry +iconUrl: https://sentry-brand.storage.googleapis.com/sentry-glyph-white.png +npmPackageName: '@backstage/plugin-sentry' + diff --git a/microsite/data/plugins/travis-ci.yaml b/microsite/data/plugins/travis-ci.yaml new file mode 100644 index 0000000000..48c0a4cb86 --- /dev/null +++ b/microsite/data/plugins/travis-ci.yaml @@ -0,0 +1,10 @@ +--- +title: Travis CI +author: roadie.io +authorUrl: https://roadie.io/ +category: CI +description: View Travis CI builds for your service in Backstage. +documentation: https://roadie.io/backstage/plugins/travis-ci +iconUrl: https://roadie.io/static/af2941eaf0af675facb281d566f42e14/45f2b/travis-ci-mascot-200x200.png +npmPackageName: '@roadiehq/backstage-plugin-travis-ci' + diff --git a/microsite/i18n/en.json b/microsite/i18n/en.json index 9d83dfcef9..f61442e938 100644 --- a/microsite/i18n/en.json +++ b/microsite/i18n/en.json @@ -199,6 +199,9 @@ "overview/what-is-backstage": { "title": "What is Backstage?" }, + "plugins/add-to-marketplace": { + "title": "Add to Marketplace" + }, "plugins/backend-plugin": { "title": "Backend plugin" }, @@ -295,6 +298,7 @@ "Docs": "Docs", "Blog": "Blog", "Demos": "Demos", + "Plugins": "Plugins", "Newsletter": "Newsletter" }, "categories": { diff --git a/microsite/package.json b/microsite/package.json index 035c7438a4..baaf9ccaf4 100644 --- a/microsite/package.json +++ b/microsite/package.json @@ -13,6 +13,7 @@ "rename-version": "docusaurus-rename-version" }, "devDependencies": { - "docusaurus": "^2.0.0-alpha.61" + "docusaurus": "^2.0.0-alpha.61", + "js-yaml": "^3.14.0" } } diff --git a/microsite/pages/en/plugins.js b/microsite/pages/en/plugins.js new file mode 100644 index 0000000000..916f34c21e --- /dev/null +++ b/microsite/pages/en/plugins.js @@ -0,0 +1,84 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +const fs = require('fs'); +const yaml = require('js-yaml'); +const React = require('react'); +const Components = require(`${process.cwd()}/core/Components.js`); +const { + Block: { Container }, + BulletLine, +} = Components; + +const pluginsDirectory = require('path').join(process.cwd(), 'data/plugins'); +const pluginMetadata = fs + .readdirSync(pluginsDirectory) + .map(file => + yaml.safeLoad(fs.readFileSync(`./data/plugins/${file}`, 'utf8')), + ); +const truncate = text => + text.length > 170 ? text.substr(0, 170) + '...' : text; + +const addPluginDocsLink = '/docs/plugins/add-to-marketplace'; +const defaultIconUrl = 'img/logo-gradient-on-dark.svg'; + +const Plugins = () => ( + +); + +module.exports = Plugins; diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 72a9388818..5d8952ac41 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -91,7 +91,7 @@ { "type": "subcategory", "label": "Publishing", - "ids": ["plugins/publishing", "plugins/publish-private"] + "ids": ["plugins/publishing", "plugins/publish-private", "plugins/add-to-marketplace"] } ], "Configuration": [ diff --git a/microsite/siteConfig.js b/microsite/siteConfig.js index 73852d6ddd..cc5ad89e09 100644 --- a/microsite/siteConfig.js +++ b/microsite/siteConfig.js @@ -38,6 +38,10 @@ const siteConfig = { href: '/docs', label: 'Docs', }, + { + page: 'plugins', + label: 'Plugins', + }, { page: 'blog', blog: true, diff --git a/microsite/static/css/plugins.css b/microsite/static/css/plugins.css new file mode 100644 index 0000000000..042e2954e1 --- /dev/null +++ b/microsite/static/css/plugins.css @@ -0,0 +1,111 @@ +.PluginCard { + background-color: #272822; + height: 100%; + padding: 16px; + display: flex; + flex-direction: column; +} + +.grid { + display: grid; + grid-gap: 1rem; + grid-template-columns: repeat(4, 1fr); + grid-auto-rows: 1fr; + padding-top: 32px; +} + +@media (max-width: 1200px) { + .grid { + grid-template-columns: repeat(3, 1fr); + } +} + +@media only screen and (max-width: 815px) { + .grid { + grid-template-columns: repeat(2, 1fr); + } +} + +.PluginCard img { + float: left; + margin: 0px 16px 8px 0px; + height: 100px; + width: 100px; +} + +.PluginCardHeader { + max-height: fit-content; + min-height: fit-content; +} + +.PluginCardTitle { + color: white; + vertical-align: top; + margin: 8px 0px 0px 16px; +} + +.PluginAddNewButton { + position: absolute; + bottom: 16px; + right: 0px; +} + +.ButtonFilled { + padding: 4px 8px; + border-radius: 4px; + background-color: #36BAA2; + color: white; + margin-top: 36px; +} + +.ButtonFilled:hover { + border: 1px solid #36BAA2; + background-color: transparent; +} + +.ChipOutlined { + font-size: small; + border-radius: 16px; + padding: 2px 8px; + border: 1px solid #36BAA2; + color: #36BAA2; +} + +.PluginCardLink { + padding: 2px 8px; + position: absolute; + bottom: 0; + right: 0; +} + +.PluginPageLayout { + margin: auto; + max-width: 1430px; + padding: 20px; +} + +.PluginPageHeader { + position: relative; +} + +.PluginPageHeader h2 { + display: inline-block; +} + +.PluginCardBody { + padding-top: 8px; +} + +.PluginCardFooter { + position: relative; + min-height: 2em; +} + +.Author, .Author a { + margin-bottom: 0.25em; + color: rgba(255,255,255, 0.6); +} + + .Author a:hover { + color: white; +} From b602a7b729f8a3a388a8f6b8d61336bcbc0a6e82 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 31 Aug 2020 15:13:42 +0200 Subject: [PATCH 099/103] Remove WIP (#2185) --- packages/techdocs-container/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/techdocs-container/README.md b/packages/techdocs-container/README.md index 2bfa9a1d26..aa1bf15974 100644 --- a/packages/techdocs-container/README.md +++ b/packages/techdocs-container/README.md @@ -2,8 +2,6 @@ This is the Docker container that powers the creation of static documentation sites that are supported by [TechDocs](https://github.com/spotify/backstage/blob/master/plugins/techdocs). -**WIP: This is a work in progress. It is not ready for use yet. Follow our progress on [the Backstage Discord](https://discord.gg/MUpMjP2) under #docs-like-code or on [our GitHub Milestone](https://github.com/spotify/backstage/milestone/15).** - ## Getting Started Using the TechDocs CLI, we can invoke the latest version of `techdocs-container` via Docker Hub: From 0c29e3d410472c40ab939e1b73a934bebb410a3f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 31 Aug 2020 14:51:34 +0200 Subject: [PATCH 100/103] workflows: add discord notification --- .github/workflows/master.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 04b50c3e60..354838037e 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -74,3 +74,11 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" package_root: "packages/core" tag_prefix: "v" + + - name: Discord notification + if: ${{ failure() }} + uses: Ilshidur/action-discord@0.2.0 + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + with: + args: 'Master build failed https://github.com/{{GITHUB_REPOSITORY}}/actions/runs/{{GITHUB_RUN_ID}}' From f14ebd6d186b85311347feda95df1f7ef9cbc7a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20=C3=85lund?= Date: Mon, 31 Aug 2020 17:29:29 +0200 Subject: [PATCH 101/103] Add more plugins to Marketplace (#2192) --- microsite/data/plugins/circleci.yaml | 9 +++++++++ microsite/data/plugins/gitops-cluster.yaml | 14 ++++++++++++++ microsite/data/plugins/graphiql.yaml | 13 +++++++++++++ microsite/data/plugins/lighthouse.yaml | 14 ++++++++++++++ microsite/data/plugins/new-relic.yaml | 14 ++++++++++++++ microsite/data/plugins/tech-radar.yaml | 9 +++++++++ microsite/i18n/en.json | 2 +- 7 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 microsite/data/plugins/circleci.yaml create mode 100644 microsite/data/plugins/gitops-cluster.yaml create mode 100644 microsite/data/plugins/graphiql.yaml create mode 100644 microsite/data/plugins/lighthouse.yaml create mode 100644 microsite/data/plugins/new-relic.yaml create mode 100644 microsite/data/plugins/tech-radar.yaml diff --git a/microsite/data/plugins/circleci.yaml b/microsite/data/plugins/circleci.yaml new file mode 100644 index 0000000000..c8a3677614 --- /dev/null +++ b/microsite/data/plugins/circleci.yaml @@ -0,0 +1,9 @@ +--- +title: CircleCI +author: Spotify +authorUrl: https://www.spotify.com/ +category: CI +description: Automate your development process with CI hosted in the cloud or on a private server. +documentation: https://github.com/spotify/backstage/tree/master/plugins/circleci +iconUrl: https://d3r49iyjzglexf.cloudfront.net/logo-wordmark-26f8eaea9b0f6e13b90d3f4a8fd8fda31490f5af41daab98bbede45037682576.svg +npmPackageName: '@backstage/plugin-circleci' diff --git a/microsite/data/plugins/gitops-cluster.yaml b/microsite/data/plugins/gitops-cluster.yaml new file mode 100644 index 0000000000..6f8ab6b097 --- /dev/null +++ b/microsite/data/plugins/gitops-cluster.yaml @@ -0,0 +1,14 @@ +--- +title: GitOps Clusters +author: Weaveworks +authorUrl: https://www.weave.works/ +category: Kubernetes +description: Create GitOps-managed Kubernetes clusters. Currently, it supports provisioning EKS clusters on GitHub via GitHub Actions. +documentation: https://github.com/spotify/backstage/tree/master/plugins/gitops-profiles +iconUrl: https://res-5.cloudinary.com/crunchbase-production/image/upload/c_lpad,h_256,w_256,f_auto,q_auto:eco/v1462316670/i9d3delzvx1erzjhmcws.png +npmPackageName: '@backstage/plugin-gitops-profiles' +tags: + - kubernetes + - gitops + - github + - eks diff --git a/microsite/data/plugins/graphiql.yaml b/microsite/data/plugins/graphiql.yaml new file mode 100644 index 0000000000..73733c84fd --- /dev/null +++ b/microsite/data/plugins/graphiql.yaml @@ -0,0 +1,13 @@ +--- +title: GraphiQL +author: Spotify +authorUrl: https://www.spotify.com/ +category: Debugging +description: Integrates GraphiQL as a tool to browse GraphiQL endpoints inside Backstage. +documentation: https://github.com/spotify/backstage/tree/master/plugins/lighthouse +iconUrl: https://upload.wikimedia.org/wikipedia/commons/thumb/1/17/GraphQL_Logo.svg/1024px-GraphQL_Logo.svg.png +npmPackageName: '@backstage/plugin-graphiql' +tags: + - graphql + - github + - gitlab diff --git a/microsite/data/plugins/lighthouse.yaml b/microsite/data/plugins/lighthouse.yaml new file mode 100644 index 0000000000..15a650f0bd --- /dev/null +++ b/microsite/data/plugins/lighthouse.yaml @@ -0,0 +1,14 @@ +--- +title: Lighthouse +author: Spotify +authorUrl: https://www.spotify.com/ +category: Accessibility +description: Google's Lighthouse tool is a great resource for benchmarking and improving the accessibility, performance, SEO, and best practices of your website. +documentation: https://github.com/spotify/backstage/tree/master/plugins/lighthouse +iconUrl: https://seeklogo.com/images/G/google-lighthouse-logo-1C7FA08580-seeklogo.com.png +npmPackageName: '@backstage/plugin-lighthouse' +tags: + - web + - seo + - accessibility + - performance diff --git a/microsite/data/plugins/new-relic.yaml b/microsite/data/plugins/new-relic.yaml new file mode 100644 index 0000000000..e3ddf18652 --- /dev/null +++ b/microsite/data/plugins/new-relic.yaml @@ -0,0 +1,14 @@ +--- +title: New Relic +author: '@timwheelercom' +authorUrl: https://github.com/timwheelercom +category: Monitoring +description: Observability platform built to help engineers create and monitor their software. +documentation: https://github.com/spotify/backstage/tree/master/plugins/newrelic +iconUrl: https://www.mulesoft.com/sites/default/files/2018-10/New_relic.png +npmPackageName: '@backstage/plugin-newrelic' +tags: + - performance + - monitoring + - errors + - alerting diff --git a/microsite/data/plugins/tech-radar.yaml b/microsite/data/plugins/tech-radar.yaml new file mode 100644 index 0000000000..a20667e8f0 --- /dev/null +++ b/microsite/data/plugins/tech-radar.yaml @@ -0,0 +1,9 @@ +--- +title: Tech Radar +author: Spotify +authorUrl: https://www.spotify.com/ +category: Discovery +description: Visualize the your company's official guidelines of different areas of software development. +documentation: https://github.com/spotify/backstage/tree/master/plugins/tech-radar +iconUrl: https://github.com/spotify/backstage/raw/master/plugins/tech-radar/docs/screenshot.png +npmPackageName: '@backstage/plugin-tech-radar' diff --git a/microsite/i18n/en.json b/microsite/i18n/en.json index f61442e938..96ffb780dc 100644 --- a/microsite/i18n/en.json +++ b/microsite/i18n/en.json @@ -296,9 +296,9 @@ "links": { "GitHub": "GitHub", "Docs": "Docs", + "Plugins": "Plugins", "Blog": "Blog", "Demos": "Demos", - "Plugins": "Plugins", "Newsletter": "Newsletter" }, "categories": { From 4788ee073ac4ebeef288ffb9d3b16e1fe25d20a3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 31 Aug 2020 20:36:05 +0200 Subject: [PATCH 102/103] cli: use regexps to match paths for rollup plugins --- packages/cli/src/lib/builder/config.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/lib/builder/config.ts b/packages/cli/src/lib/builder/config.ts index e62e1fc6cd..bc689a3eb0 100644 --- a/packages/cli/src/lib/builder/config.ts +++ b/packages/cli/src/lib/builder/config.ts @@ -88,15 +88,25 @@ export const makeConfigs = async ( }), resolve({ mainFields }), commonjs({ - include: ['node_modules/**', '../../node_modules/**'], - exclude: ['**/*.stories.*', '**/*.test.*'], + include: /node_modules/, + exclude: [/\/[^/]+\.(?:stories|test)\.[^/]+$/], }), postcss(), - imageFiles({ exclude: '**/*.icon.svg' }), + imageFiles({ + exclude: /\.icon\.svg$/, + include: [ + /\.css$/, + /\.svg$/, + /\.png$/, + /\.gif$/, + /\.jpg$/, + /\.jpeg$/, + ], + }), json(), yaml(), svgr({ - include: '**/*.icon.svg', + include: /\.icon\.svg$/, template: svgrTemplate, }), esbuild({ From 0ff78005cd23e9d54e313b95548c0b7138b0dd5c Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Mon, 31 Aug 2020 23:07:16 +0200 Subject: [PATCH 103/103] fix(docs): code block background --- microsite/static/css/custom.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microsite/static/css/custom.css b/microsite/static/css/custom.css index 3c0184a07b..5e078582cc 100644 --- a/microsite/static/css/custom.css +++ b/microsite/static/css/custom.css @@ -216,7 +216,7 @@ td { code { font-family: IBM Plex Mono, Menlo, Monaco, Consolas, Courier New, monospace; font-weight: 500; - background-color: #0e0e0e; + background-color: #272822; } /* .stripe {
+
+ + + + {pluginMetadata.map( + ({ + iconUrl, + title, + description, + author, + authorUrl, + documentation, + category, + }) => ( +
+
+ {title} +

{title}

+

+ by {author} +

+ {category} +
+
+

{truncate(description)}

+
+ + + + docs + + + +
+ ), + )} + +
+