Merge branch 'master' of github.com:spotify/backstage into mob/union-types

* 'master' of github.com:spotify/backstage: (656 commits)
  Update playback-order-component.yaml (#2495)
  feat: rework Api Explorer components
  Add GitLab integration for scaffolder
  feat(catalog-backend): support GHE and other GitHub providers
  Add scaffolder config to monorepo (#2492)
  Simplify code
  prettified md
  Added PlantUML diagrams in backstage.io docs (#2413)
  chore(deps): bump nodegit from 0.26.5 to 0.27.0 (#2454)
  grammar correction
  added description header field
  added description header field
  added description header field
  added description header field
  added description header field
  Condense if to else if
  Update docs
  Reword
  Make access optional
  eslintignore: add template dirs
  ...

packages/backend/Dockerfile

@@ -2,10 +2,15 @@ FROM node:12
 
 WORKDIR /usr/src/app
 
+# Copy repo skeleton first, to avoid unnecessary docker cache invalidation.
+# The skeleton contains the package.json of each package in the monorepo,
+# and along with yarn.lock and the root package.json, that's enough to run yarn install.
+ADD yarn.lock package.json skeleton.tar ./
+
+RUN yarn install --frozen-lockfile --production
+
 # This will copy the contents of the dist-workspace when running the build-image command.
 # Do not use this Dockerfile outside of that command, as it will copy in the source code instead.
 COPY . .
 
-RUN yarn install --frozen-lockfile --production
-
 CMD ["node", "packages/backend"]

packages/backend/README.md

@@ -37,26 +37,19 @@ yarn start
 
 Substitute `x` for actual values, or leave them as
 dummy values just to try out the backend without using the auth or sentry features.
+You can also, instead of using dummy values for a huge number of environment variables, remove those config directly from app-config.yaml file located in the root folder.
 
 The backend starts up on port 7000 per default.
 
 ## Populating The Catalog
 
-If you want to use the catalog functionality, you need to add so called locations
-to the backend. These are places where the backend can find some entity descriptor
-data to consume and serve.
+If you want to use the catalog functionality, you need to add so called
+locations to the backend. These are places where the backend can find some
+entity descriptor data to consume and serve. For more information, see
+[Software Catalog Overview - Adding Components to the Catalog](https://backstage.io/docs/features/software-catalog/software-catalog-overview#adding-components-to-the-catalog).
 
-To get started, you can issue the following after starting the backend, from inside
-the `plugins/catalog-backend` directory:
-
-```bash
-yarn mock-data
-```
-
-You should then start seeing data on `localhost:7000/catalog/entities`.
-
-The catalog currently runs in-memory only, so feel free to try it out, but it will
-need to be re-populated on next startup.
+For convenience we already include some statically configured example locations
+in `app-config.yaml` under `catalog.locations`. For local development you can override these in your own `app-config.local.yaml`.
 
 ## Authentication
 

packages/backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "example-backend",
-  "version": "0.1.1-alpha.18",
+  "version": "0.1.1-alpha.21",
   "main": "dist/index.cjs.js",
   "types": "src/index.ts",
   "private": true,
@@ -10,7 +10,7 @@
   },
   "scripts": {
     "build": "backstage-cli backend:build",
-    "build-image": "backstage-cli backend:build-image example-backend",
+    "build-image": "backstage-cli backend:build-image --build --tag example-backend",
     "start": "backstage-cli backend:dev",
     "lint": "backstage-cli lint",
     "test": "backstage-cli test",
@@ -18,31 +18,35 @@
     "migrate:create": "knex migrate:make -x ts"
   },
   "dependencies": {
-    "@backstage/backend-common": "^0.1.1-alpha.18",
-    "@backstage/catalog-model": "^0.1.1-alpha.18",
-    "@backstage/config": "^0.1.1-alpha.18",
-    "@backstage/plugin-auth-backend": "^0.1.1-alpha.18",
-    "@backstage/plugin-catalog-backend": "^0.1.1-alpha.18",
-    "@backstage/plugin-graphql-backend": "^0.1.1-alpha.18",
-    "@backstage/plugin-identity-backend": "^0.1.1-alpha.18",
-    "@backstage/plugin-proxy-backend": "^0.1.1-alpha.18",
-    "@backstage/plugin-rollbar-backend": "^0.1.1-alpha.18",
-    "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.18",
-    "@backstage/plugin-sentry-backend": "^0.1.1-alpha.18",
-    "@backstage/plugin-techdocs-backend": "^0.1.1-alpha.18",
+    "@backstage/backend-common": "^0.1.1-alpha.21",
+    "@backstage/catalog-model": "^0.1.1-alpha.21",
+    "@backstage/config": "^0.1.1-alpha.21",
+    "@backstage/plugin-app-backend": "^0.1.1-alpha.21",
+    "@backstage/plugin-auth-backend": "^0.1.1-alpha.21",
+    "@backstage/plugin-catalog-backend": "^0.1.1-alpha.21",
+    "@backstage/plugin-graphql-backend": "^0.1.1-alpha.21",
+    "@backstage/plugin-identity-backend": "^0.1.1-alpha.21",
+    "@backstage/plugin-proxy-backend": "^0.1.1-alpha.21",
+    "@backstage/plugin-rollbar-backend": "^0.1.1-alpha.21",
+    "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.21",
+    "@backstage/plugin-sentry-backend": "^0.1.1-alpha.21",
+    "@backstage/plugin-techdocs-backend": "^0.1.1-alpha.21",
+    "@gitbeaker/node": "^23.5.0",
     "@octokit/rest": "^18.0.0",
     "dockerode": "^3.2.0",
+    "example-app": "^0.1.1-alpha.21",
     "express": "^4.17.1",
     "knex": "^0.21.1",
     "pg": "^8.3.0",
+    "pg-connection-string": "^2.3.0",
     "sqlite3": "^4.2.0",
     "winston": "^3.2.1"
   },
   "devDependencies": {
-    "@backstage/cli": "^0.1.1-alpha.18",
+    "@backstage/cli": "^0.1.1-alpha.21",
     "@types/dockerode": "^2.5.32",
     "@types/express": "^4.17.6",
     "@types/express-serve-static-core": "^4.17.5",
-    "@types/helmet": "^0.0.47"
+    "@types/helmet": "^0.0.48"
   }
 }

packages/backend/src/index.ts

@@ -23,13 +23,13 @@
  */
 
 import {
+  createDatabaseClient,
   createServiceBuilder,
   loadBackendConfig,
   getRootLogger,
   useHotMemoize,
 } from '@backstage/backend-common';
 import { ConfigReader, AppConfig } from '@backstage/config';
-import knex, { PgConnectionConfig } from 'knex';
 import healthcheck from './plugins/healthcheck';
 import auth from './plugins/auth';
 import catalog from './plugins/catalog';
@@ -40,6 +40,7 @@ import sentry from './plugins/sentry';
 import proxy from './plugins/proxy';
 import techdocs from './plugins/techdocs';
 import graphql from './plugins/graphql';
+import app from './plugins/app';
 import { PluginEnvironment } from './types';
 
 function makeCreateEnv(loadedConfigs: AppConfig[]) {
@@ -47,39 +48,14 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) {
 
   return (plugin: string): PluginEnvironment => {
     const logger = getRootLogger().child({ type: 'plugin', plugin });
-    // Supported DBs are sqlite and postgres
-    const isPg = [
-      'POSTGRES_USER',
-      'POSTGRES_HOST',
-      'POSTGRES_PASSWORD',
-    ].every(key => config.getOptional(`backend.${key}`));
-
-    let knexConfig;
-
-    if (isPg) {
-      knexConfig = {
-        client: 'pg',
-        useNullAsDefault: true,
+    const database = createDatabaseClient(
+      config.getConfig('backend.database'),
+      {
         connection: {
-          port: config.getOptionalNumber('backend.POSTGRES_PORT'),
-          host: config.getString('backend.POSTGRES_HOST'),
-          user: config.getString('backend.POSTGRES_USER'),
-          password: config.getString('backend.POSTGRES_PASSWORD'),
           database: `backstage_plugin_${plugin}`,
-        } as PgConnectionConfig,
-      };
-    } else {
-      knexConfig = {
-        client: 'sqlite3',
-        connection: ':memory:',
-        useNullAsDefault: true,
-      };
-    }
-
-    const database = knex(knexConfig);
-    database.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
-      resource.run('PRAGMA foreign_keys = ON', () => {});
-    });
+        },
+      },
+    );
     return { logger, database, config };
   };
 }
@@ -98,6 +74,7 @@ async function main() {
   const sentryEnv = useHotMemoize(module, () => createEnv('sentry'));
   const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs'));
   const graphqlEnv = useHotMemoize(module, () => createEnv('graphql'));
+  const appEnv = useHotMemoize(module, () => createEnv('app'));
 
   const service = createServiceBuilder(module)
     .loadConfig(configReader)
@@ -109,8 +86,9 @@ async function main() {
     .addRouter('/auth', await auth(authEnv))
     .addRouter('/identity', await identity(identityEnv))
     .addRouter('/techdocs', await techdocs(techdocsEnv))
-    .addRouter('/proxy', await proxy(proxyEnv))
-    .addRouter('/graphql', await graphql(graphqlEnv));
+    .addRouter('/proxy', await proxy(proxyEnv, '/proxy'))
+    .addRouter('/graphql', await graphql(graphqlEnv))
+    .addRouter('', await app(appEnv));
 
   await service.start().catch(err => {
     console.log(err);
@@ -120,6 +98,6 @@ async function main() {
 
 module.hot?.accept();
 main().catch(error => {
-  console.error(`Backend failed to start up, ${error}`);
+  console.error('Backend failed to start up', error);
   process.exit(1);
 });

packages/backend/src/plugins/app.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createRouter } from '@backstage/plugin-app-backend';
+import { PluginEnvironment } from '../types';
+
+export default async function createPlugin({ logger }: PluginEnvironment) {
+  return await createRouter({
+    logger,
+    appPackageName: 'example-app',
+  });
+}

packages/backend/src/plugins/proxy.ts

@@ -13,13 +13,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 // @ts-ignore
 import { createRouter } from '@backstage/plugin-proxy-backend';
 import { PluginEnvironment } from '../types';
 
-export default async function createPlugin({
-  logger,
-  config,
-}: PluginEnvironment) {
-  return await createRouter({ logger, config });
+export default async function createPlugin(
+  { logger, config }: PluginEnvironment,
+  pathPrefix: string,
+) {
+  return await createRouter({ logger, config, pathPrefix });
 }

packages/backend/src/plugins/scaffolder.ts

@@ -19,16 +19,24 @@ import {
   createRouter,
   FilePreparer,
   GithubPreparer,
+  GitlabPreparer,
   Preparers,
+  Publishers,
   GithubPublisher,
+  GitlabPublisher,
   CreateReactAppTemplater,
   Templaters,
+  RepoVisilityOptions,
 } from '@backstage/plugin-scaffolder-backend';
 import { Octokit } from '@octokit/rest';
+import { Gitlab } from '@gitbeaker/node';
 import type { PluginEnvironment } from '../types';
 import Docker from 'dockerode';
 
-export default async function createPlugin({ logger }: PluginEnvironment) {
+export default async function createPlugin({
+  logger,
+  config,
+}: PluginEnvironment) {
   const cookiecutterTemplater = new CookieCutter();
   const craTemplater = new CreateReactAppTemplater();
   const templaters = new Templaters();
@@ -37,19 +45,48 @@ export default async function createPlugin({ logger }: PluginEnvironment) {
 
   const filePreparer = new FilePreparer();
   const githubPreparer = new GithubPreparer();
+  const gitlabPreparer = new GitlabPreparer(config);
   const preparers = new Preparers();
 
   preparers.register('file', filePreparer);
   preparers.register('github', githubPreparer);
+  preparers.register('gitlab', gitlabPreparer);
+  preparers.register('gitlab/api', gitlabPreparer);
 
-  const githubClient = new Octokit({ auth: process.env.GITHUB_ACCESS_TOKEN });
-  const publisher = new GithubPublisher({ client: githubClient });
+  const publishers = new Publishers();
+
+  const githubToken = config.getString('scaffolder.github.token');
+  const repoVisibility = config.getString(
+    'scaffolder.github.visibility',
+  ) as RepoVisilityOptions;
+
+  const githubClient = new Octokit({ auth: githubToken });
+  const githubPublisher = new GithubPublisher({
+    client: githubClient,
+    token: githubToken,
+    repoVisibility,
+  });
+  publishers.register('file', githubPublisher);
+  publishers.register('github', githubPublisher);
+
+  const gitLabConfig = config.getOptionalConfig('scaffolder.gitlab.api');
+
+  if (gitLabConfig) {
+    const gitLabToken = gitLabConfig.getString('token');
+    const gitLabClient = new Gitlab({
+      host: gitLabConfig.getOptionalString('baseUrl'),
+      token: gitLabToken,
+    });
+    const gitLabPublisher = new GitlabPublisher(gitLabClient, gitLabToken);
+    publishers.register('gitlab', gitLabPublisher);
+    publishers.register('gitlab/api', gitLabPublisher);
+  }
 
   const dockerClient = new Docker();
   return await createRouter({
     preparers,
     templaters,
-    publisher,
+    publishers,
     logger,
     dockerClient,
   });

packages/backend/src/plugins/techdocs.ts

@@ -20,22 +20,27 @@ import {
   Preparers,
   Generators,
   LocalPublish,
-  TechdocsGenerator
+  TechdocsGenerator,
+  GithubPreparer,
 } from '@backstage/plugin-techdocs-backend';
 import { PluginEnvironment } from '../types';
 import Docker from 'dockerode';
 
-export default async function createPlugin({ logger, config }: PluginEnvironment) {
+export default async function createPlugin({
+  logger,
+  config,
+}: PluginEnvironment) {
   const generators = new Generators();
-  const techdocsGenerator = new TechdocsGenerator();
+  const techdocsGenerator = new TechdocsGenerator(logger);
   generators.register('techdocs', techdocsGenerator);
 
-  const directoryPreparer = new DirectoryPreparer();
   const preparers = new Preparers();
-
+  const githubPreparer = new GithubPreparer(logger);
+  const directoryPreparer = new DirectoryPreparer(logger);
   preparers.register('dir', directoryPreparer);
+  preparers.register('github', githubPreparer);
 
-  const publisher = new LocalPublish();
+  const publisher = new LocalPublish(logger);
 
   const dockerClient = new Docker();