diff --git a/.changeset/2798.md b/.changeset/2798.md new file mode 100644 index 0000000000..2b202142ef --- /dev/null +++ b/.changeset/2798.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Use the new `UrlReader` in `PlaceholderProcessor`. +This allows to use the placeholder processor to include API definitions in API entities. +Previously it was only possible to do this if the definition comes from the same location type as the entity itself. diff --git a/.changeset/2803.md b/.changeset/2803.md new file mode 100644 index 0000000000..3a4773757f --- /dev/null +++ b/.changeset/2803.md @@ -0,0 +1,26 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Remove the backstage.io/definition-at-location annotation. +The annotation was superseded by the placeholder processor. + +```yaml +apiVersion: backstage.io/v1alpha1 +kind: API +metadata: + name: spotify + description: The Spotify web API + tags: + - spotify + - rest + annotations: + # Don't use this annotation, but the placeholder $text instead (see below). + backstage.io/definition-at-location: 'url:https://raw.githubusercontent.com/APIs-guru/openapi-directory/master/APIs/spotify.com/v1/swagger.yaml' +spec: + type: openapi + lifecycle: production + owner: spotify@example.com + definition: + $text: https://raw.githubusercontent.com/APIs-guru/openapi-directory/master/APIs/spotify.com/v1/swagger.yaml +``` diff --git a/.changeset/2804.md b/.changeset/2804.md new file mode 100644 index 0000000000..5223a24791 --- /dev/null +++ b/.changeset/2804.md @@ -0,0 +1,15 @@ +--- +'@backstage/plugin-proxy-backend': minor +--- + +Limit the http headers that are forwarded from the request to a safe set of defaults. +A user can configure additional headers that should be forwarded if the specific applications needs that. + +```yaml +proxy: + '/my-api': + target: 'https://my-api.com/get' + allowedHeaders: + # We need to forward the Authorization header that was provided by the caller + - Authorization +``` diff --git a/.changeset/codeowner-processor-url-reader.md b/.changeset/codeowner-processor-url-reader.md new file mode 100644 index 0000000000..af4eb0997f --- /dev/null +++ b/.changeset/codeowner-processor-url-reader.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Use the new `UrlReader` in the `CodeOwnersProcessor`. diff --git a/.changeset/cyan-plants-dress.md b/.changeset/cyan-plants-dress.md new file mode 100644 index 0000000000..19ebc63e53 --- /dev/null +++ b/.changeset/cyan-plants-dress.md @@ -0,0 +1,5 @@ +--- +'example-backend': patch +--- + +Pass GitHub token into Scaffolder GitHub Preparer diff --git a/.changeset/long-ways-count.md b/.changeset/long-ways-count.md new file mode 100644 index 0000000000..e04c1225cc --- /dev/null +++ b/.changeset/long-ways-count.md @@ -0,0 +1,57 @@ +--- +'@backstage/plugin-techdocs-backend': minor +'@backstage/create-app': patch +--- + +Updated naming of environment variables. New pattern [NAME]\_TOKEN for Github, Gitlab, Azure & Github enterprise access tokens. + +### Detail: + +- Previously we have to export same token for both, catalog & scaffolder + +```bash +export GITHUB_ACCESS_TOKEN=foo +export GITHUB_PRIVATE_TOKEN=foo +``` + +with latest changes, only single export is sufficient. + +```bash +export GITHUB_TOKEN=foo +export GITLAB_TOKEN=foo +export GHE_TOKEN=foo +export AZURE_TOKEN=foo +``` + +### list: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Old nameNew name
GITHUB_ACCESS_TOKENGITHUB_TOKEN
GITHUB_PRIVATE_TOKENGITHUB_TOKEN
GITLAB_ACCESS_TOKENGITLAB_TOKEN
GITLAB_PRIVATE_TOKENGITLAB_TOKEN
AZURE_PRIVATE_TOKENAZURE_TOKEN
GHE_PRIVATE_TOKENGHE_TOKEN
diff --git a/.changeset/loud-lamps-visit.md b/.changeset/loud-lamps-visit.md new file mode 100644 index 0000000000..9023061643 --- /dev/null +++ b/.changeset/loud-lamps-visit.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Including source maps with all packages diff --git a/.changeset/ninety-pens-poke.md b/.changeset/ninety-pens-poke.md new file mode 100644 index 0000000000..94bcb06749 --- /dev/null +++ b/.changeset/ninety-pens-poke.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': patch +--- + +Fix for configured templates using 'url' locations even though it's not supported yet diff --git a/.changeset/pretty-cups-joke.md b/.changeset/pretty-cups-joke.md new file mode 100644 index 0000000000..49a1db2a75 --- /dev/null +++ b/.changeset/pretty-cups-joke.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Add authentication token to Scaffolder GitHub Preparer diff --git a/.changeset/remove-location-processor-read.md b/.changeset/remove-location-processor-read.md new file mode 100644 index 0000000000..78aea6d820 --- /dev/null +++ b/.changeset/remove-location-processor-read.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Remove the `read` argument of `LocationProcessor.processEntity`. +Instead, pass the `UrlReader` into the constructor of your `LocationProcessor`. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0437698301..01fc8e70c3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,3 +13,4 @@ - [ ] Prettier run on changed files - [ ] Tests added for new functionality - [ ] Regression tests added for bug fixes +- [ ] Added a changeset ([more info](https://github.com/spotify/backstage/blob/master/CONTRIBUTING.md#creating-changesets)) diff --git a/.codecov.yml b/.github/codecov.yml similarity index 100% rename from .codecov.yml rename to .github/codecov.yml diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 21045f04dd..c9774ae237 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -32,6 +32,7 @@ Codehilite codeowners config Config +configmaps configs const cookiecutter @@ -67,6 +68,7 @@ graphviz Hackathons haproxy heroku +horizontalpodautoscalers Hostname http https @@ -140,6 +142,7 @@ rankdir readme Readme Redash +replicasets repo Repo repos diff --git a/CHANGELOG.md b/CHANGELOG.md index 1860a4de6f..487b94a39d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re > Collect changes for the next release below +### Backend (example-backend, or backends created with @backstage/create-app) + +- A plugin database manager has been created, and plugins can now accept that interface as an argument during initialisation. Notably, the `auth` plugin has a [`createRouter` signature change](./plugins/auth-backend/src/service/router.ts). See [packages/backend/src/index.ts](./packages/backend/src/index.ts) on how to set it up. [#2697](https://github.com/spotify/backstage/pull/2697) + ## v0.1.1-alpha.24 ### Backend (example-backend, or backends created with @backstage/create-app) diff --git a/app-config.yaml b/app-config.yaml index d670771a31..228eafc588 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -67,21 +67,21 @@ integrations: github: - host: github.com token: - $env: GITHUB_PRIVATE_TOKEN + $env: GITHUB_TOKEN ### Example for how to add your GitHub Enterprise instance using the API: # - host: ghe.example.net # apiBaseUrl: https://ghe.example.net/api/v3 # token: - # $env: GHE_PRIVATE_TOKEN + # $env: GHE_TOKEN ### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): # - host: ghe.example.net # rawBaseUrl: https://ghe.example.net/raw # token: - # $env: GHE_PRIVATE_TOKEN + # $env: GHE_TOKEN gitlab: - host: gitlab.com token: - $env: GITLAB_PRIVATE_TOKEN + $env: GITLAB_TOKEN bitbucket: - host: bitbucket.org username: @@ -91,7 +91,7 @@ integrations: azure: - host: dev.azure.com token: - $env: AZURE_PRIVATE_TOKEN + $env: AZURE_TOKEN catalog: rules: @@ -102,12 +102,12 @@ catalog: providers: - target: https://github.com token: - $env: GITHUB_PRIVATE_TOKEN + $env: GITHUB_TOKEN #### Example for how to add your GitHub Enterprise instance using the API: # - target: https://ghe.example.net # apiBaseUrl: https://ghe.example.net/api/v3 # token: - # $env: GHE_PRIVATE_TOKEN + # $env: GHE_TOKEN ldapOrg: ### Example for how to add your enterprise LDAP server # providers: @@ -146,18 +146,18 @@ catalog: scaffolder: github: token: - $env: GITHUB_ACCESS_TOKEN + $env: GITHUB_TOKEN visibility: public # or 'internal' or 'private' gitlab: api: baseUrl: https://gitlab.com token: - $env: GITLAB_ACCESS_TOKEN + $env: GITLAB_TOKEN azure: baseUrl: https://dev.azure.com/{your-organization} api: token: - $env: AZURE_PRIVATE_TOKEN + $env: AZURE_TOKEN auth: providers: diff --git a/backstage_overview.png b/backstage_overview.png deleted file mode 100644 index ae32ae6628..0000000000 Binary files a/backstage_overview.png and /dev/null differ diff --git a/contrib/docker/kubernetes-example-backend/Dockerfile b/contrib/docker/kubernetes-example-backend/Dockerfile new file mode 100644 index 0000000000..df617decf5 --- /dev/null +++ b/contrib/docker/kubernetes-example-backend/Dockerfile @@ -0,0 +1,35 @@ +FROM node:12-buster + +WORKDIR /usr/src/app + +# (workaround) Install cookiecutter and mkdocs to avoid the need to run docker in docker +RUN cd /tmp && curl -O https://www.python.org/ftp/python/3.8.2/Python-3.8.2.tar.xz && \ + tar -xvf Python-3.8.2.tar.xz && \ + cd Python-3.8.2 && \ + ./configure --enable-optimizations && \ + make -j 4 && \ + make altinstall + +RUN apt update +RUN apt install -y mkdocs + +RUN pip3.8 install mkdocs-techdocs-core + +RUN pip3.8 install cookiecutter && \ + apt remove -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev libbz2-dev g++ python-pip python-dev && \ + rm -rf /var/cache/apt/* /tmp/Python-3.8.2 + +# Copy repo skeleton first, to avoid unnecessary docker cache invalidation. +# The skeleton contains the package.json of each package in the monorepo, +# and along with yarn.lock and the root package.json, that's enough to run yarn install. +ADD yarn.lock package.json skeleton.tar ./ + +RUN yarn install --frozen-lockfile --production + +# This will copy the contents of the dist-workspace when running the build-image command. +# Do not use this Dockerfile outside of that command, as it will copy in the source code instead. +COPY . . + +CMD ["node", "packages/backend"] + + diff --git a/contrib/docker/kubernetes-example-backend/README.md b/contrib/docker/kubernetes-example-backend/README.md new file mode 100644 index 0000000000..d0f9d57022 --- /dev/null +++ b/contrib/docker/kubernetes-example-backend/README.md @@ -0,0 +1,13 @@ +# Example backend Dockerfile + +This Dockerfile will build the example backend with certain additional binaries needed to workaround +the docker requirement in the scaffolder and techdocs. + +# Usage + +```bash +yarn docker-build -f --tag +``` + +> The absolute path is necessary as this directory is not copied to the build workspace when building +> the docker image. diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md index 757e0461a9..40a4ca3ea1 100644 --- a/docs/features/software-catalog/well-known-annotations.md +++ b/docs/features/software-catalog/well-known-annotations.md @@ -41,27 +41,6 @@ expecting a two-item array out of it. The format of the target part is type-dependent and could conceivably even be an empty string, but the separator colon is always present. -### backstage.io/definition-at-location - -```yaml -# Example -apiVersion: backstage.io/v1alpha1 -kind: API -metadata: - name: petstore - annotations: - backstage.io/definition-at-location: 'url:https://petstore.swagger.io/v2/swagger.json' -spec: - type: openapi -``` - -This annotation allows to fetch an API definition from another location, instead -of wrapping the API definition inside the definition field. This allows to -easily consume existing API definition. The definition is fetched during -ingestion by a processor and included in the entity. It is updated on every -refresh. The annotation contains a location reference string that contains the -location processor type and the target. - ### backstage.io/techdocs-ref ```yaml @@ -205,6 +184,25 @@ This annotation was used for a while to enable the GitHub Actions feature. This is now instead using the [github.com/project-slug](#github-com-project-slug) annotation, with the same value format. +### backstage.io/definition-at-location + +This annotation allowed to load the API definition from another location. Now +placeholders can be used instead: + +``` +apiVersion: backstage.io/v1alpha1 +kind: API +metadata: + name: petstore + description: The Petstore API +spec: + type: openapi + lifecycle: production + owner: petstore@example.com + definition: + $text: https://petstore.swagger.io/v2/swagger.json +``` + ## Links - [Descriptor Format: annotations](descriptor-format.md#annotations-optional) diff --git a/docs/features/software-templates/extending/create-your-own-publisher.md b/docs/features/software-templates/extending/create-your-own-publisher.md index ecb097f9d9..546980f033 100644 --- a/docs/features/software-templates/extending/create-your-own-publisher.md +++ b/docs/features/software-templates/extending/create-your-own-publisher.md @@ -34,7 +34,7 @@ import { Octokit } from '@octokit/rest'; import type { PluginEnvironment } from '../types'; export default async function createPlugin({ logger }: PluginEnvironment) { - const githubClient = new Octokit({ auth: process.env.GITHUB_ACCESS_TOKEN }); + const githubClient = new Octokit({ auth: process.env.GITHUB_TOKEN }); const publisher = new GithubPublisher({ client: githubClient }); return await createRouter({ diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index 60f73ad0c3..838c5e2d11 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -232,13 +232,13 @@ instance: scaffolder: github: token: - $env: GITHUB_ACCESS_TOKEN + $env: GITHUB_TOKEN visibility: public # or 'internal' or 'private' gitlab: api: baseUrl: https://gitlab.com token: - $env: SCAFFOLDER_GITLAB_PRIVATE_TOKEN + $env: GITLAB_TOKEN ``` #### Azure DevOps @@ -255,7 +255,7 @@ scaffolder: baseUrl: https://dev.azure.com/{your-organization} api: token: - $env: AZURE_PRIVATE_TOKEN + $env: AZURE_TOKEN ``` ### Running the Backend @@ -265,7 +265,7 @@ backend with the new configuration: ```bash cd packages/backend -GITHUB_ACCESS_TOKEN= yarn start +GITHUB_TOKEN= yarn start ``` If you've also set up the frontend plugin, so you should be ready to go browse diff --git a/docs/plugins/proxying.md b/docs/plugins/proxying.md index 3d9aa90447..f55b4d9229 100644 --- a/docs/plugins/proxying.md +++ b/docs/plugins/proxying.md @@ -57,6 +57,15 @@ is also possible to limit the forwarded HTTP methods with the configuration `allowedMethods`, for example `allowedMethods: ['GET']` to enforce read-only access. +By default, the proxy will only forward safe HTTP request headers to the target. +Those are based on the headers that are considered safe for CORS and includes +headers like `content-type` or `last-modified`, as well as all headers that are +set by the proxy. If the proxy should forward other headers like +`authorization`, this must be enabled by the `allowedHeaders` config, for +example `allowedHeaders: ['Authorization']`. This should help to not +accidentally forward confidential headers (`cookie`, `X-Auth-Request-User`) to +third-parties. + If the value is a string, it is assumed to correspond to: ```yaml diff --git a/packages/app/package.json b/packages/app/package.json index 20b9e24701..76e20bc0dc 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -28,6 +28,7 @@ "@backstage/plugin-tech-radar": "^0.1.1-alpha.24", "@backstage/plugin-techdocs": "^0.1.1-alpha.24", "@backstage/plugin-welcome": "^0.1.1-alpha.24", + "@backstage/plugin-user-settings": "^0.1.1-alpha.24", "@backstage/test-utils": "^0.1.1-alpha.24", "@backstage/theme": "^0.1.1-alpha.24", "@material-ui/core": "^4.11.0", diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index 667b14366f..81d08f58a9 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -33,6 +33,7 @@ import { Router as GraphiQLRouter } from '@backstage/plugin-graphiql'; import { Router as TechRadarRouter } from '@backstage/plugin-tech-radar'; import { Router as LighthouseRouter } from '@backstage/plugin-lighthouse'; import { Router as RegisterComponentRouter } from '@backstage/plugin-register-component'; +import { Router as SettingsRouter } from '@backstage/plugin-user-settings'; import { Route, Routes, Navigate } from 'react-router'; import { EntityPage } from './components/catalog/EntityPage'; @@ -81,6 +82,7 @@ const AppRoutes = () => ( path="/register-component" element={} /> + } /> {...deprecatedAppRoutes} ); diff --git a/packages/app/src/components/Root/Root.tsx b/packages/app/src/components/Root/Root.tsx index 45ffca7801..e947e5c91d 100644 --- a/packages/app/src/components/Root/Root.tsx +++ b/packages/app/src/components/Root/Root.tsx @@ -35,11 +35,10 @@ import { SidebarDivider, SidebarSearchField, SidebarSpace, - SidebarUserSettings, - DefaultProviderSettings, } from '@backstage/core'; import { NavLink } from 'react-router-dom'; import { graphiQLRouteRef } from '@backstage/plugin-graphiql'; +import { Settings as SidebarSettings } from '@backstage/plugin-user-settings'; const useSidebarLogoStyles = makeStyles({ root: { @@ -103,7 +102,7 @@ const Root: FC<{}> = ({ children }) => ( /> - } /> + {children} diff --git a/packages/app/src/components/catalog/EntityPage.tsx b/packages/app/src/components/catalog/EntityPage.tsx index c2483b40ee..393ffe2631 100644 --- a/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/app/src/components/catalog/EntityPage.tsx @@ -49,6 +49,11 @@ import { import { Entity } from '@backstage/catalog-model'; import { Button, Grid } from '@material-ui/core'; import { EmptyState } from '@backstage/core'; +import { + EmbeddedRouter as LighthouseRouter, + LastLighthouseAuditCard, + isPluginApplicableToEntity as isLighthouseAvailable, +} from '@backstage/plugin-lighthouse/'; const CICDSwitcher = ({ entity }: { entity: Entity }) => { // This component is just an example of how you can implement your company's logic in entity page. @@ -115,6 +120,11 @@ const OverviewContent = ({ entity }: { entity: Entity }) => ( + {isLighthouseAvailable(entity) && ( + + + + )} ); @@ -165,6 +175,11 @@ const WebsiteEntityPage = ({ entity }: { entity: Entity }) => ( title="CI/CD" element={} /> + } + /> { + const createConfig = (data: any) => + ConfigReader.fromConfigs([ + { + context: '', + data, + }, + ]); + + const defaultConfigOptions = { + backend: { + database: { + client: 'pg', + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }, + }, + }; + const defaultConfig = () => createConfig(defaultConfigOptions); + + // This is similar to the ts-jest `mocked` helper. + const mocked = (f: Function) => f as jest.Mock; + + afterEach(() => jest.resetAllMocks()); + + describe('SingleConnectionDatabaseManager.fromConfig', () => { + it('accesses the backend.database key', () => { + const getConfig = jest.fn(); + const config = defaultConfig(); + config.getConfig = getConfig; + + SingleConnectionDatabaseManager.fromConfig(config); + + expect(getConfig.mock.calls[0][0]).toEqual('backend.database'); + }); + }); + + describe('SingleConnectionDatabaseManager.forPlugin', () => { + const manager = SingleConnectionDatabaseManager.fromConfig(defaultConfig()); + + it('connects to a database scoped to the plugin', async () => { + const pluginId = 'test1'; + await manager.forPlugin(pluginId).getClient(); + + expect(mocked(createDatabaseClient)).toHaveBeenCalledTimes(1); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const callArgs = mockCalls[0]; + expect(callArgs[0].get()).toEqual(defaultConfigOptions.backend.database); + expect(callArgs[1].connection.database).toEqual( + `backstage_plugin_${pluginId}`, + ); + }); + + it('provides different plugins different databases', async () => { + const plugin1Id = 'test1'; + const plugin2Id = 'test2'; + await manager.forPlugin(plugin1Id).getClient(); + await manager.forPlugin(plugin2Id).getClient(); + + expect(mocked(createDatabaseClient)).toHaveBeenCalledTimes(2); + + const mockCalls = mocked(createDatabaseClient).mock.calls; + const plugin1CallArgs = mockCalls[0]; + const plugin2CallArgs = mockCalls[1]; + expect(plugin1CallArgs[1].connection.database).not.toEqual( + plugin2CallArgs[1].connection.database, + ); + }); + }); +}); diff --git a/packages/backend-common/src/database/SingleConnection.ts b/packages/backend-common/src/database/SingleConnection.ts new file mode 100644 index 0000000000..777ced54a0 --- /dev/null +++ b/packages/backend-common/src/database/SingleConnection.ts @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Knex from 'knex'; +import { Config } from '@backstage/config'; +import { createDatabaseClient, ensureDatabaseExists } from './connection'; +import { PluginDatabaseManager } from './types'; + +/** + * Implements a Database Manager which will automatically create new databases + * for plugins when requested. All requested databases are created with the + * credentials provided; if the database already exists no attempt to create + * the database will be made. + */ +export class SingleConnectionDatabaseManager { + /** + * Creates a new SingleConnectionDatabaseManager instance by reading from the `backend` + * config section, specifically the `.database` key for discovering the management + * database configuration. + * + * @param config The loaded application configuration. + */ + static fromConfig(config: Config): SingleConnectionDatabaseManager { + return new SingleConnectionDatabaseManager( + config.getConfig('backend.database'), + ); + } + + private constructor(private readonly config: Config) {} + + /** + * Generates a PluginDatabaseManager for consumption by plugins. + * + * @param pluginId The plugin that the database manager should be created for. Plugin names should be unique. + */ + forPlugin(pluginId: string): PluginDatabaseManager { + const _this = this; + + return { + getClient(): Promise { + return _this.getDatabase(pluginId); + }, + }; + } + + private async getDatabase(pluginId: string): Promise { + const config = this.config; + const overrides = SingleConnectionDatabaseManager.getDatabaseOverrides( + pluginId, + ); + const overrideConfig = overrides.connection as Knex.ConnectionConfig; + await this.ensureDatabase(overrideConfig.database); + + return createDatabaseClient(config, overrides); + } + + private static getDatabaseOverrides(pluginId: string): Knex.Config { + return { + connection: { + database: `backstage_plugin_${pluginId}`, + }, + }; + } + + private async ensureDatabase(database: string) { + const config = this.config; + await ensureDatabaseExists(config, database); + } +} diff --git a/packages/backend-common/src/database/config.test.ts b/packages/backend-common/src/database/config.test.ts index ab29bc19ba..c9a4107686 100644 --- a/packages/backend-common/src/database/config.test.ts +++ b/packages/backend-common/src/database/config.test.ts @@ -18,6 +18,18 @@ import { mergeDatabaseConfig } from './config'; describe('config', () => { describe(mergeDatabaseConfig, () => { + it('does not mutate the input object', () => { + const input = { + original: 'key', + }; + const override = { + added: 'value', + }; + + mergeDatabaseConfig(input, override); + expect(input).not.toHaveProperty('added'); + }); + it('does not require overrides', () => { expect( mergeDatabaseConfig({ diff --git a/packages/backend-common/src/database/config.ts b/packages/backend-common/src/database/config.ts index 80abc06aa2..af32556c28 100644 --- a/packages/backend-common/src/database/config.ts +++ b/packages/backend-common/src/database/config.ts @@ -19,9 +19,9 @@ import { merge } from 'lodash'; /** * Merges database objects together * - * @param config The base config + * @param config The base config. The input is not modified * @param overrides Any additional overrides */ export function mergeDatabaseConfig(config: any, ...overrides: any[]) { - return merge(config, ...overrides); + return merge({}, config, ...overrides); } diff --git a/packages/backend-common/src/database/index.ts b/packages/backend-common/src/database/index.ts index 38d3d6224b..c81153aa62 100644 --- a/packages/backend-common/src/database/index.ts +++ b/packages/backend-common/src/database/index.ts @@ -15,3 +15,5 @@ */ export * from './connection'; +export * from './types'; +export * from './SingleConnection'; diff --git a/packages/backend-common/src/database/types.ts b/packages/backend-common/src/database/types.ts new file mode 100644 index 0000000000..7eaa05f173 --- /dev/null +++ b/packages/backend-common/src/database/types.ts @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import knex from 'knex'; + +/** + * The PluginDatabaseManager manages access to databases that Plugins get. + */ +export interface PluginDatabaseManager { + /** + * getClient provides backend plugins database connections for itself. + * + * The purpose of this method is to allow plugins to get isolated data + * stores so that plugins are discouraged from database integration. + */ + getClient(): Promise; +} diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 959af71427..18fd44f089 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -24,17 +24,16 @@ import Router from 'express-promise-router'; import { - ensureDatabaseExists, - createDatabaseClient, createServiceBuilder, loadBackendConfig, getRootLogger, useHotMemoize, notFoundHandler, + SingleConnectionDatabaseManager, SingleHostDiscovery, UrlReaders, } from '@backstage/backend-common'; -import { ConfigReader, AppConfig } from '@backstage/config'; +import { ConfigReader } from '@backstage/config'; import healthcheck from './plugins/healthcheck'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; @@ -48,24 +47,18 @@ import graphql from './plugins/graphql'; import app from './plugins/app'; import { PluginEnvironment } from './types'; -function makeCreateEnv(loadedConfigs: AppConfig[]) { - const config = ConfigReader.fromConfigs(loadedConfigs); +function makeCreateEnv(config: ConfigReader) { const root = getRootLogger(); const reader = UrlReaders.default({ logger: root, config }); const discovery = SingleHostDiscovery.fromConfig(config); root.info(`Created UrlReader ${reader}`); + const databaseManager = SingleConnectionDatabaseManager.fromConfig(config); + return (plugin: string): PluginEnvironment => { const logger = root.child({ type: 'plugin', plugin }); - const database = createDatabaseClient( - config.getConfig('backend.database'), - { - connection: { - database: `backstage_plugin_${plugin}`, - }, - }, - ); + const database = databaseManager.forPlugin(plugin); return { logger, database, config, reader, discovery }; }; } @@ -73,12 +66,7 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { async function main() { const configs = await loadBackendConfig(); const configReader = ConfigReader.fromConfigs(configs); - const createEnv = makeCreateEnv(configs); - await ensureDatabaseExists( - configReader.getConfig('backend.database'), - 'backstage_plugin_catalog', - 'backstage_plugin_auth', - ); + const createEnv = makeCreateEnv(configReader); const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck')); const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index c1d390f8cc..b8e99e6398 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -34,7 +34,9 @@ export default async function createPlugin({ }: PluginEnvironment) { const locationReader = new LocationReaders({ logger, reader, config }); - const db = await DatabaseManager.createDatabase(database, { logger }); + const db = await DatabaseManager.createDatabase(await database.getClient(), { + logger, + }); const entitiesCatalog = new DatabaseEntitiesCatalog(db); const locationsCatalog = new DatabaseLocationsCatalog(db); const higherOrderOperation = new HigherOrderOperations( diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index c5b36ab8da..90667bfb16 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -47,13 +47,12 @@ export default async function createPlugin({ templaters.register('cra', craTemplater); const filePreparer = new FilePreparer(); - const githubPreparer = new GithubPreparer(); + const gitlabPreparer = new GitlabPreparer(config); const azurePreparer = new AzurePreparer(config); const preparers = new Preparers(); preparers.register('file', filePreparer); - preparers.register('github', githubPreparer); preparers.register('gitlab', gitlabPreparer); preparers.register('gitlab/api', gitlabPreparer); preparers.register('azure/api', azurePreparer); @@ -75,6 +74,10 @@ export default async function createPlugin({ token: githubToken, repoVisibility, }); + + const githubPreparer = new GithubPreparer({ token: githubToken }); + + preparers.register('github', githubPreparer); publishers.register('file', githubPublisher); publishers.register('github', githubPublisher); } catch (e) { diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index e04d2a2732..6e36c26f39 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -23,6 +23,7 @@ import { TechdocsGenerator, GithubPreparer, GitlabPreparer, + AzurePreparer, } from '@backstage/plugin-techdocs-backend'; import { PluginEnvironment } from '../types'; import Docker from 'dockerode'; @@ -39,10 +40,12 @@ export default async function createPlugin({ const preparers = new Preparers(); const githubPreparer = new GithubPreparer(logger); const gitlabPreparer = new GitlabPreparer(logger); + const azurePreparer = new AzurePreparer(logger); const directoryPreparer = new DirectoryPreparer(logger); preparers.register('dir', directoryPreparer); preparers.register('github', githubPreparer); preparers.register('gitlab', gitlabPreparer); + preparers.register('azure/api', azurePreparer); const publisher = new LocalPublish(logger); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 9257fcc9cf..f63b9dd780 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -14,14 +14,17 @@ * limitations under the License. */ -import Knex from 'knex'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common'; +import { + PluginDatabaseManager, + PluginEndpointDiscovery, + UrlReader, +} from '@backstage/backend-common'; export type PluginEnvironment = { logger: Logger; - database: Knex; + database: PluginDatabaseManager; config: Config; reader: UrlReader; discovery: PluginEndpointDiscovery; diff --git a/packages/catalog-model/examples/spotify-api.yaml b/packages/catalog-model/examples/spotify-api.yaml index 30524dfdd4..14deb17abe 100644 --- a/packages/catalog-model/examples/spotify-api.yaml +++ b/packages/catalog-model/examples/spotify-api.yaml @@ -7,8 +7,11 @@ metadata: - spotify - rest annotations: + # The annotation is deprecated, we use placeholders (see below) instead, remove it later. backstage.io/definition-at-location: 'url:https://raw.githubusercontent.com/APIs-guru/openapi-directory/master/APIs/spotify.com/v1/swagger.yaml' spec: type: openapi lifecycle: production owner: spotify@example.com + definition: + $text: https://github.com/APIs-guru/openapi-directory/blob/master/APIs/spotify.com/v1/swagger.yaml diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index f69e18190d..b9f709241c 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -96,7 +96,7 @@ async function getConfig() { // Default behaviour is to not apply transforms for node_modules, but we still want // to apply the esm-transformer to .esm.js files, since that's what we use in backstage packages. transformIgnorePatterns: [ - `/node_modules/${transformModulePattern}(?:(?!\\.esm).)*\\.(?:js|json)$`, + `/node_modules/${transformModulePattern}.*\\.(?:(?; }; /** diff --git a/packages/core-api/src/apis/definitions/FeatureFlagsApi.ts b/packages/core-api/src/apis/definitions/FeatureFlagsApi.ts index b6aac75620..dd9657d7d8 100644 --- a/packages/core-api/src/apis/definitions/FeatureFlagsApi.ts +++ b/packages/core-api/src/apis/definitions/FeatureFlagsApi.ts @@ -15,11 +15,8 @@ */ import { createApiRef } from '../ApiRef'; -import { - UserFlags, - FeatureFlagsRegistry, - FeatureFlagsRegistryItem, -} from '../../app/FeatureFlags'; +import { UserFlags, FeatureFlagsRegistry } from '../../app/FeatureFlags'; +import { FeatureFlagName } from '../../plugin'; /** * The feature flags API is used to toggle functionality to users across plugins and Backstage. @@ -55,6 +52,11 @@ export interface FeatureFlagsApi { getRegisteredFlags(): FeatureFlagsRegistry; } +export interface FeatureFlagsRegistryItem { + pluginId: string; + name: FeatureFlagName; +} + export const featureFlagsApiRef = createApiRef({ id: 'core.featureflags', description: 'Used to toggle functionality in features across Backstage', diff --git a/packages/core-api/src/app/App.tsx b/packages/core-api/src/app/App.tsx index 9dd2fb138f..c222c3c420 100644 --- a/packages/core-api/src/app/App.tsx +++ b/packages/core-api/src/app/App.tsx @@ -30,12 +30,12 @@ import { SignInPageProps, } from './types'; import { BackstagePlugin } from '../plugin'; -import { FeatureFlagsRegistryItem } from './FeatureFlags'; import { featureFlagsApiRef, AppThemeApi, ConfigApi, identityApiRef, + FeatureFlagsRegistryItem, } from '../apis/definitions'; import { AppThemeProvider } from './AppThemeProvider'; diff --git a/packages/core-api/src/app/FeatureFlags.tsx b/packages/core-api/src/app/FeatureFlags.tsx index 3db2a18a02..7a69fa5456 100644 --- a/packages/core-api/src/app/FeatureFlags.tsx +++ b/packages/core-api/src/app/FeatureFlags.tsx @@ -15,7 +15,11 @@ */ import { FeatureFlagName } from '../plugin/types'; -import { FeatureFlagState, FeatureFlagsApi } from '../apis/definitions'; +import { + FeatureFlagState, + FeatureFlagsApi, + FeatureFlagsRegistryItem, +} from '../apis/definitions'; /** * Helper method for validating compatibility and flag name. @@ -129,10 +133,6 @@ export class UserFlags extends Map { * This acts as a holding data structure for feature flags * that plugins wish to register for use in Backstage. */ -export interface FeatureFlagsRegistryItem { - pluginId: string; - name: FeatureFlagName; -} export class FeatureFlagsRegistry extends Array { static from(entries: FeatureFlagsRegistryItem[]) { diff --git a/packages/core-api/src/app/types.ts b/packages/core-api/src/app/types.ts index 882ecd3d8b..8a1eb92e8e 100644 --- a/packages/core-api/src/app/types.ts +++ b/packages/core-api/src/app/types.ts @@ -112,11 +112,13 @@ export type AppOptions = { * title: 'Light Theme', * variant: 'light', * theme: lightTheme, + * icon: , * }, { * id: 'dark', * title: 'Dark Theme', * variant: 'dark', * theme: darkTheme, + * icon: , * }] * ``` */ diff --git a/packages/core/package.json b/packages/core/package.json index f125c9d67e..b665006fd2 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -39,8 +39,9 @@ "@types/react-sparklines": "^1.7.0", "classnames": "^2.2.6", "clsx": "^1.1.0", + "immer": "^7.0.9", "lodash": "^4.17.15", - "material-table": "1.68.0", + "material-table": "^1.69.1", "prop-types": "^15.7.2", "rc-progress": "^3.0.0", "react": "^16.12.0", diff --git a/packages/core/src/api-wrappers/createApp.tsx b/packages/core/src/api-wrappers/createApp.tsx index 7fe77423e2..c7d0f5ce01 100644 --- a/packages/core/src/api-wrappers/createApp.tsx +++ b/packages/core/src/api-wrappers/createApp.tsx @@ -22,7 +22,8 @@ import privateExports, { AppConfigLoader, } from '@backstage/core-api'; import { BrowserRouter, MemoryRouter } from 'react-router-dom'; - +import LightIcon from '@material-ui/icons/WbSunny'; +import DarkIcon from '@material-ui/icons/Brightness2'; import { ErrorPage } from '../layout/ErrorPage'; import { Progress } from '../components/Progress'; import { defaultApis } from './defaultApis'; @@ -110,12 +111,14 @@ export function createApp(options?: AppOptions) { title: 'Light Theme', variant: 'light', theme: lightTheme, + icon: , }, { id: 'dark', title: 'Dark Theme', variant: 'dark', theme: darkTheme, + icon: , }, ]; const configLoader = options?.configLoader ?? defaultConfigLoader; diff --git a/packages/core/src/components/Button/Button.stories.tsx b/packages/core/src/components/Button/Button.stories.tsx index aaca389f82..a5005574fe 100644 --- a/packages/core/src/components/Button/Button.stories.tsx +++ b/packages/core/src/components/Button/Button.stories.tsx @@ -24,7 +24,7 @@ const Location = () => { }; export default { - title: 'Button', + title: 'Inputs/Button', component: Button, decorators: [ (storyFn: FunctionComponentFactory<{}>) => ( diff --git a/packages/core/src/components/CheckboxTree/CheckboxTree.stories.tsx b/packages/core/src/components/CheckboxTree/CheckboxTree.stories.tsx new file mode 100644 index 0000000000..12f19b17a1 --- /dev/null +++ b/packages/core/src/components/CheckboxTree/CheckboxTree.stories.tsx @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { CheckboxTree } from '.'; + +const CHECKBOX_TREE_ITEMS = [ + { + label: 'Genereic subcategory name 1', + options: [ + { + label: 'Option 1', + value: 1, + }, + { + label: 'Option 2', + value: 2, + }, + ], + }, + { + label: 'Genereic subcategory name 2', + options: [ + { + label: 'Option 1', + value: 1, + }, + { + label: 'Option 2', + value: 2, + }, + ], + }, + { + label: 'Genereic subcategory name 3', + options: [ + { + label: 'Option 1', + value: 1, + }, + { + label: 'Option 2', + value: 2, + }, + ], + }, +]; + +export default { + title: 'CheckboxTree', + component: CheckboxTree, +}; + +export const Default = () => ( + {}} + label="default" + subCategories={CHECKBOX_TREE_ITEMS} + /> +); diff --git a/packages/core/src/components/CheckboxTree/CheckboxTree.test.tsx b/packages/core/src/components/CheckboxTree/CheckboxTree.test.tsx new file mode 100644 index 0000000000..f5eaa0e58b --- /dev/null +++ b/packages/core/src/components/CheckboxTree/CheckboxTree.test.tsx @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { render, fireEvent } from '@testing-library/react'; + +import { CheckboxTree } from '.'; + +const CHECKBOX_TREE_ITEMS = [ + { + label: 'Genereic subcategory name 1', + options: [ + { + label: 'Option 1', + value: 1, + }, + { + label: 'Option 2', + value: 2, + }, + ], + }, +]; + +const minProps = { + onChange: jest.fn(), + label: 'Default', + subCategories: CHECKBOX_TREE_ITEMS, +}; + +describe('', () => { + it('renders without exploding', async () => { + const { getByText, getByTestId } = render(); + + expect(getByText('Genereic subcategory name 1')).toBeInTheDocument(); + const checkbox = await getByTestId('expandable'); + + // Simulate click on expandable arrow + fireEvent.click(checkbox); + + // Simulate click on option + const option = getByText('Option 1'); + expect(getByText('Option 1')).toBeInTheDocument(); + fireEvent.click(option); + expect(minProps.onChange).toHaveBeenCalled(); + }); +}); diff --git a/packages/core/src/components/CheckboxTree/CheckboxTree.tsx b/packages/core/src/components/CheckboxTree/CheckboxTree.tsx new file mode 100644 index 0000000000..6825704971 --- /dev/null +++ b/packages/core/src/components/CheckboxTree/CheckboxTree.tsx @@ -0,0 +1,294 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* eslint-disable guard-for-in */ +import React, { useEffect, useReducer } from 'react'; +import { createStyles, makeStyles, Theme } from '@material-ui/core/styles'; +import { + List, + ListItem, + ListItemIcon, + Checkbox, + ListItemText, + Collapse, + Typography, +} from '@material-ui/core'; +import ExpandLess from '@material-ui/icons/ExpandLess'; +import ExpandMore from '@material-ui/icons/ExpandMore'; +import produce from 'immer'; + +type IndexedObject = { + [key: string]: T; +}; + +const useStyles = makeStyles((theme: Theme) => + createStyles({ + root: { + width: '100%', + minWidth: 10, + maxWidth: 360, + backgroundColor: 'transparent', + '&:hover': { + backgroundColor: 'transparent', + }, + '&:active': { + animation: 'none', + transform: 'none', + }, + }, + nested: { + paddingLeft: theme.spacing(5), + height: '32px', + '&:hover': { + backgroundColor: 'transparent', + }, + }, + listItemIcon: { + minWidth: 10, + }, + listItem: { + '&:hover': { + backgroundColor: 'transparent', + }, + }, + text: { + '& span, & svg': { + fontWeight: 'normal', + fontSize: 14, + }, + }, + }), +); + +/* SUB_CATEGORY */ + +type SubCategory = { + label: string; + isChecked?: boolean; + isOpen?: boolean; + options?: Option[]; +}; + +type SubCategoryWithIndexedOptions = { + label: string; + isChecked?: boolean; + isOpen?: boolean; + options: IndexedObject