Merge remote-tracking branch 'origin/master' into erikengervall/plugin-release-manager-as-a-service

.changeset/dry-elephants-doubt.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-github-deployments': patch
+---
+
+Adds extraColumns field to GitHub Deployments card

.changeset/fair-carrots-tell.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Add `config:docs` command that opens up reference documentation for the local configuration schema in a browser.

.changeset/fluffy-suns-repair.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder': patch
+---
+
+Respect top-level UI schema keys in scaffolder forms. Allows more advanced RJSF features such as explicit field ordering.

.changeset/fresh-cheetahs-rush.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+No longer add newly created plugins to `plugins.ts` in the app, as it is no longer needed.

.changeset/good-glasses-build.md

@@ -0,0 +1,6 @@
+---
+'@backstage/core-api': patch
+'@backstage/core': patch
+---
+
+Add support for discovering plugins through the app element tree, removing the need to register them explicitly.

.changeset/quiet-badgers-cheer.md

@@ -0,0 +1,11 @@
+---
+'@backstage/create-app': patch
+---
+
+Removed `plugins.ts` from the app, as plugins are now discovered through the react tree.
+
+To apply this change to an existing app, simply delete `packages/app/src/plugins.ts` along with the import and usage in `packages/app/src/App.tsx`.
+
+Note that there are a few plugins that require explicit registration, in which case you would need to keep them in `plugins.ts`. The set of plugins that need explicit registration is any plugin that doesn't have a component extension that gets rendered as part of the app element tree. An example of such a plugin in the main Backstage repo is `@backstage/plugin-badges`. In the case of the badges plugin this is because there is not yet a component-based API for adding context menu items to the entity layout.
+
+If you have plugins that still rely on route registration through the `register` method of `createPlugin`, these need to be kept in `plugins.ts` as well. However, it is recommended to migrate these to export an extensions component instead.

.changeset/real-apples-visit.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+GithubDiscoveryProcessor now excludes archived repositories so they won't be added to Backstage.

.changeset/rude-items-bow.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-kubernetes-backend': patch
+---
+
+Kubernetes client TLS verification is now configurable and defaults to true

.changeset/six-turtles-sip.md

@@ -0,0 +1,6 @@
+---
+'@backstage/backend-common': patch
+---
+
+Support configuration of file storage for SQLite databases. Every plugin has its
+own database file at the specified path.

.changeset/sour-plums-enjoy.md

@@ -0,0 +1,7 @@
+---
+'@backstage/create-app': patch
+---
+
+Fix system diagram card to be on the system page
+
+To apply the same fix to an existing application, in `EntityPage.tsx` simply move the `<EntityLayout.route>` for the `/diagram` path from the `groupPage` down into the `systemPage` element.

.changeset/stale-carpets-poke.md

@@ -0,0 +1,30 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Externalize repository processing for BitbucketDiscoveryProcessor.
+
+Add an extension point where you can customize how a matched Bitbucket repository should
+be processed. This can for example be used if you want to generate the catalog-info.yaml
+automatically based on other files in a repository, while taking advantage of the
+build-in repository crawling functionality.
+
+`BitbucketDiscoveryProcessor.fromConfig` now takes an optional parameter `options.parser` where
+you can customize the logic for each repository found. The default parser has the same
+behaviour as before, where it emits an optional location for the matched repository
+and lets the other processors take care of further processing.
+
+```typescript
+const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({
+  client,
+  repository,
+}) {
+  // Custom logic for interpret the matching repository.
+  // See defaultRepositoryParser for an example
+};
+
+const processor = BitbucketDiscoveryProcessor.fromConfig(env.config, {
+  parser: customRepositoryParser,
+  logger: env.logger,
+});
+```

.changeset/stale-chefs-retire.md

@@ -0,0 +1,5 @@
+---
+'@backstage/core': patch
+---
+
+Adding close button on support menu

contrib/docs/tutorials/aws-deployment.md

@@ -41,12 +41,9 @@ documentation to build a new Backstage Docker image:
 
 ```shell
 $ yarn build
-$ docker image build . -f packages/backend/Dockerfile --tag backstage
+$ yarn build-image --tag backstage
 ```
 
-This command builds a backend-only image, but you can similarly build a frontend
-or combined Docker image.
-
 Next, configure the [AWS CLI](https://aws.amazon.com/cli/) to use the
 `ecr-publisher` user you created:
 
@@ -90,65 +87,37 @@ document, but it can be as easy as `eksctl create cluster` documented in the
 guide](https://docs.aws.amazon.com/eks/latest/userguide/getting-started-eksctl.html),
 which uses a Cloudformation template to create the necessary resources.
 
-To deploy the Docker image to EKS, create a `kubernetes` folder in your
-Backstage source folder and add a Kubernetes `deployment.yaml`:
+To deploy the Docker image to EKS, follow the [Kubernetes
+guide](https://backstage.io/docs/deployment/k8s#creating-the-backstage-instance)
+but set the Backstage deployment `image` to the ECR repository URL:
 
 ```yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: backstage-backend
-  labels:
-    app: backstage-backend
-  namespace: default
+  name: backstage
+  namespace: backstage
 spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: backstage-backend
-  strategy:
-    rollingUpdate:
-      maxSurge: 25%
-      maxUnavailable: 25%
-    type: RollingUpdate
+  ...
   template:
     metadata:
       labels:
-        app: backstage-backend
+        app: backstage
     spec:
       containers:
         - image: <repo_url>/backstage:1.0.0
           imagePullPolicy: Always
-          name: backstage-backend
-          ports:
-            - containerPort: 7000
-              protocol: TCP
+          ...
 ```
 
-Note the `image` key in the container spec referencing the ECR repository.
-
-Now create a simple `service.yaml` to map the container ports:
-
-```yaml
-apiVersion: v1
-kind: Service
-metadata:
-  name: backstage-backend
-spec:
-  selector:
-    app: backstage-backend
-  ports:
-    - protocol: TCP
-      port: 80
-      targetPort: 7000
-```
-
-Apply these Kubernetes definitions to the EKS cluster to complete the Backstage
-deployment:
+Create the [Service
+descriptor](https://backstage.io/docs/deployment/k8s#creating-a-backstage-service)
+as well, and apply these Kubernetes definitions to the EKS cluster to complete
+the Backstage deployment:
 
 ```shell
-$ kubectl apply -f deployment.yaml
-$ kubectl apply -f service.yaml
+$ kubectl apply -f kubernetes/backstage.yaml
+$ kubectl apply -f kubernetes/backstage-service.yaml
 ```
 
 Now you can see your Backstage workload running from the [EKS
@@ -158,14 +127,15 @@ console](https://console.aws.amazon.com/eks/home).
 
 ### Exposing Backstage with a load balancer
 
-Backstage users need to query the backend, which means we need to expose
-the workload with a load balancer. Follow the [Application load balancing on
+To make the service useful, we need to expose the workload with a load balancer.
+Follow the [Application load balancing on
 EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) guide to
 set up a Load Balancer controller and Kubernetes ingress to your application.
 
 This is ultimately a `kubectl apply` with an ingress definition:
 
 ```yaml
+# kubernetes/backstage-ingress.yaml
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -187,9 +157,3 @@ spec:
                 port:
                   number: 80
 ```
-
-### Updating the deployment
-
-To update the Kubernetes deployment to a newly published version of your
-Backstage Docker image, update the image tag reference in `deployment.yaml` and
-then apply the changes to EKS with `kubectl apply -f deployment.yaml`.

contrib/kubernetes/plain_single_backend_deployment/README.md

@@ -1,44 +0,0 @@
-# Plain Kubernetes Deployment
-
-This directory contains an example of a simple Kubernetes deployment of Backstage. It is not intended to serve as a complete production deployment, but as a starting point for setting one up.
-
-## Usage
-
-You can try the deployment out as is. The easiest way is to use [Docker Desktop](https://www.docker.com/products/docker-desktop) with [Kubernetes](https://docs.docker.com/get-started/kube-deploy/).
-
-You can now follow the documentation here to build the Backend Container [Docker Build](https://backstage.io/docs/getting-started/deployment-docker)
-
-From a fresh clone of this repo, run the following in the root:
-
-```bash
-yarn install
-
-yarn docker-build
-
-kubectl apply -f contrib/kubernetes/plain_single_backend_deployment/deployment.yaml
-```
-
-You can use the following commands to monitor the deployment:
-
-```bash
-# List all resources in the backstage namespace
-kubectl -n backstage get all
-
-# Inspect the status of the deployment resource
-kubectl -n backstage describe deployment backstage-backend
-
-# Inspect the status of the pod running the backstage backend
-kubectl -n backstage describe pod -l app=backstage,component=backend
-```
-
-Once the deployment is up and running, you can use the following to set up a proxy to reach the backend locally:
-
-```bash
-kubectl proxy
-```
-
-With the proxy up and running, you should be able to navigate to [http://localhost:8001/api/v1/namespaces/backstage/services/backstage-backend:http/proxy](http://localhost:8001/api/v1/namespaces/backstage/services/backstage-backend:http/proxy) and see Backstage. Note that you'll end up on a 404 page, but hitting the home icon in the sidebar should take you to the catalog page where you can see a few example services.
-
-## Caveats
-
-This deployment is for demonstration purposes only, for a production deployment you will need to set up at least a persistent database and some form of ingress. If your organization doesn't already have established patterns for these, you could look at options of managed PostgreSQL instances from cloud providers, or something like Zalando's [postgres-operator](https://github.com/zalando/postgres-operator). For ingress there are also [plenty of options](https://ramitsurana.gitbook.io/awesome-kubernetes/docs/projects/projects#load-balancing), where `nginx` is a popular choice to get started.

contrib/kubernetes/plain_single_backend_deployment/deployment.yaml

@@ -1,107 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: backstage
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: backstage-backend
-  namespace: backstage
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: backstage
-      component: backend
-  template:
-    metadata:
-      labels:
-        app: backstage
-        component: backend
-    spec:
-      containers:
-        - name: backend
-          # This image is built with `yarn docker-build` in the repo root.
-          # Replace this with your own image to deploy your own Backstage app.
-          image: example-backend:latest
-          imagePullPolicy: Never
-
-          command: [node, packages/backend]
-          args: [--config, app-config.yaml, --config, k8s-config.yaml]
-
-          env:
-            # We set this to development to make the backend start with incomplete configuration. In a production
-            # deployment you will want to make sure that you have a full configuration, and remove any plugins that
-            # you are not using.
-            - name: NODE_ENV
-              value: development
-
-            # This makes it possible for the app to reach the backend when serving through `kubectl proxy`
-            # If you expose the service using for example an ingress controller, you should
-            # switch this out or remove it.
-            #
-            # Note that we're not setting app.baseUrl here, as setting the base path is not working at the moment.
-            # Further work is needed around the routing in the frontend or react-router before we can support that.
-            - name: APP_CONFIG_backend_baseUrl
-              value: http://localhost:8001/api/v1/namespaces/backstage/services/backstage-backend:http/proxy
-
-          ports:
-            - name: http
-              containerPort: 7000
-
-          volumeMounts:
-            - name: config-volume
-              mountPath: /app/k8s-config.yaml
-              subPath: k8s-config.yaml
-
-          resources:
-            limits:
-              cpu: 1
-              memory: 0.5Gi
-
-          readinessProbe:
-            httpGet:
-              port: 7000
-              path: /healthcheck
-          livenessProbe:
-            httpGet:
-              port: 7000
-              path: /healthcheck
-
-      volumes:
-        - name: config-volume
-          configMap:
-            name: backstage-config
-            items:
-              - key: app-config
-                path: k8s-config.yaml
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: backstage-config
-  namespace: backstage
-data:
-  # Note that the config here is only applied to the backend. The frontend config is applied at build time.
-  # To override frontend config in this deployment, use `APP_CONFIG_` env vars.
-  app-config: |
-    app:
-      baseUrl: http://localhost:8001/api/v1/namespaces/backstage/services/backstage-backend:http/proxy
-    backend:
-      baseUrl: http://localhost:8001/api/v1/namespaces/backstage/services/backstage-backend:http/proxy
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: backstage-backend
-  namespace: backstage
-spec:
-  selector:
-    app: backstage
-    component: backend
-  ports:
-    - name: http
-      port: 80
-      targetPort: http

docs/cli/commands.md

@@ -44,6 +44,7 @@ clean                    Delete cache directories
 create-plugin            Creates a new plugin in the current repository
 remove-plugin            Removes plugin in the current repository
 
+config:docs              Browse the configuration reference documentation
 config:print             Print the app configuration for the current package
 config:check             Validate that the given configuration loads and matches schema
 config:schema            Dump the app configuration schema
@@ -447,6 +448,25 @@ Options:
   --backstage-cli-help    display help for command
 ```
 
+## config:docs
+
+Scope: `root`
+
+This commands opens up the reference documentation of your apps local
+configuration schema in the browser. This is useful to get an overview of what
+configuration values are available to use, a description of what they do and
+their format, and where they get sent.
+
+```text
+Usage: backstage-cli config:docs [options]
+
+Browse the configuration reference documentation
+
+Options:
+  --package <name>  Only include the schema that applies to the given package
+  -h, --help        display help for command
+```
+
 ## config:print
 
 Scope: `root`

docs/features/kubernetes/configuration.md

@@ -25,6 +25,7 @@ kubernetes:
         - url: http://127.0.0.1:9999
           name: minikube
           authProvider: 'serviceAccount'
+          skipTLSVerify: false
           serviceAccountToken: ${K8S_MINIKUBE_TOKEN}
         - url: http://127.0.0.2:9999
           name: aws-cluster-1
@@ -78,6 +79,11 @@ cluster. Valid values are:
 | `serviceAccount` | This will use a Kubernetes [service account](https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/) to access the Kubernetes API. When this is used the `serviceAccountToken` field should also be set. |
 | `google`         | This will use a user's Google auth token from the [Google auth plugin](https://backstage.io/docs/auth/) to access the Kubernetes API.                                                                                             |
 
+##### `clusters.\*.skipTLSVerify`
+
+This determines whether or not the Kubernetes client verifies the TLS
+certificate presented by the API server. Defaults to `false`.
+
 ##### `clusters.\*.serviceAccountToken` (optional)
 
 The service account token to be used when using the `serviceAccount` auth

docs/integrations/bitbucket/discovery.md

@@ -39,3 +39,29 @@ The target is composed of four parts:
 - The path within each repository to find the catalog YAML file. This will
   usually be `/catalog-info.yaml` or a similar variation for catalog files
   stored in the root directory of each repository.
+
+## Custom repository processing
+
+The Bitbucket Discovery Processor will by default emit a location for each
+matching repository for further processing by other processors. However, it is
+possible to override this functionality and take full control of how each
+matching repository is processed.
+
+`BitbucketDiscoveryProcessor.fromConfig` takes an optional parameter
+`options.parser` where you can set your own parser to be used for each matched
+repository.
+
+```typescript
+const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({
+  client,
+  repository,
+}) {
+  // Custom logic for interpret the matching repository.
+  // See defaultRepositoryParser for an example
+};
+
+const processor = BitbucketDiscoveryProcessor.fromConfig(env.config, {
+  parser: customRepositoryParser,
+  logger: env.logger,
+});
+```

packages/app/package.json

@@ -19,8 +19,6 @@
     "@backstage/plugin-explore": "^0.3.2",
     "@backstage/plugin-gcp-projects": "^0.2.5",
     "@backstage/plugin-github-actions": "^0.4.2",
-    "@backstage/plugin-github-deployments": "^0.1.2",
-    "@backstage/plugin-gitops-profiles": "^0.2.6",
     "@backstage/plugin-graphiql": "^0.2.9",
     "@backstage/plugin-jenkins": "^0.4.1",
     "@backstage/plugin-kafka": "^0.2.6",
@@ -29,7 +27,6 @@
     "@backstage/plugin-newrelic": "^0.2.6",
     "@backstage/plugin-org": "^0.3.12",
     "@backstage/plugin-pagerduty": "0.3.2",
-    "@backstage/plugin-register-component": "^0.2.12",
     "@backstage/plugin-github-release-manager": "^0.1.1",
     "@backstage/plugin-rollbar": "^0.3.3",
     "@backstage/plugin-scaffolder": "^0.9.0",

packages/app/src/components/catalog/EntityPage.tsx

@@ -441,10 +441,6 @@ const groupPage = (
         </Grid>
       </Grid>
     </EntityLayout.Route>
-
-    <EntityLayout.Route path="/diagram" title="Diagram">
-      <EntitySystemDiagramCard />
-    </EntityLayout.Route>
   </EntityLayoutWrapper>
 );
 
@@ -463,6 +459,9 @@ const systemPage = (
         </Grid>
       </Grid>
     </EntityLayout.Route>
+    <EntityLayout.Route path="/diagram" title="Diagram">
+      <EntitySystemDiagramCard />
+    </EntityLayout.Route>
   </EntityLayoutWrapper>
 );
 

packages/app/src/plugins.ts

@@ -13,37 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export { plugin as LighthousePlugin } from '@backstage/plugin-lighthouse';
-export { catalogPlugin } from '@backstage/plugin-catalog';
-export { scaffolderPlugin } from '@backstage/plugin-scaffolder';
-export { plugin as TechRadar } from '@backstage/plugin-tech-radar';
-export { explorePlugin } from '@backstage/plugin-explore';
-export { plugin as Circleci } from '@backstage/plugin-circleci';
-export { plugin as RegisterComponent } from '@backstage/plugin-register-component';
-export { plugin as Sentry } from '@backstage/plugin-sentry';
-export { plugin as GitopsProfiles } from '@backstage/plugin-gitops-profiles';
-export { plugin as TechDocs } from '@backstage/plugin-techdocs';
-export { plugin as GraphiQL } from '@backstage/plugin-graphiql';
-export { plugin as GithubActions } from '@backstage/plugin-github-actions';
-export { plugin as Rollbar } from '@backstage/plugin-rollbar';
-export { plugin as Newrelic } from '@backstage/plugin-newrelic';
-export { travisciPlugin } from '@roadiehq/backstage-plugin-travis-ci';
-export { plugin as Jenkins } from '@backstage/plugin-jenkins';
-export { plugin as ApiDocs } from '@backstage/plugin-api-docs';
-export { githubPullRequestsPlugin } from '@roadiehq/backstage-plugin-github-pull-requests';
-export { plugin as GcpProjects } from '@backstage/plugin-gcp-projects';
-export { plugin as Kubernetes } from '@backstage/plugin-kubernetes';
-export { plugin as Cloudbuild } from '@backstage/plugin-cloudbuild';
-export { plugin as CostInsights } from '@backstage/plugin-cost-insights';
-export { githubInsightsPlugin } from '@roadiehq/backstage-plugin-github-insights';
-export { plugin as CatalogImport } from '@backstage/plugin-catalog-import';
-export { plugin as UserSettings } from '@backstage/plugin-user-settings';
-export { plugin as PagerDuty } from '@backstage/plugin-pagerduty';
-export { buildkitePlugin } from '@roadiehq/backstage-plugin-buildkite';
-export { plugin as Search } from '@backstage/plugin-search';
-export { plugin as Org } from '@backstage/plugin-org';
-export { plugin as Kafka } from '@backstage/plugin-kafka';
-export { todoPlugin } from '@backstage/plugin-todo';
+
+// TODO(Rugvip): This plugin is currently not part of the app element tree,
+//               ideally we have an API for the context menu that permits that.
 export { badgesPlugin } from '@backstage/plugin-badges';
-export { githubDeploymentsPlugin } from '@backstage/plugin-github-deployments';
-export { gitHubReleaseManagerPlugin } from '@backstage/plugin-github-release-manager';

packages/backend-common/config.d.ts

@@ -57,7 +57,7 @@ export interface Config {
     database:
       | {
           client: 'sqlite3';
-          connection: ':memory:' | string;
+          connection: ':memory:' | string | { filename: string };
         }
       | {
           client: 'pg';

packages/backend-common/src/database/connection.ts

@@ -37,7 +37,7 @@ export function createDatabaseClient(
   if (client === 'pg') {
     return createPgDatabaseClient(dbConfig, overrides);
   } else if (client === 'sqlite3') {
-    return createSqliteDatabaseClient(dbConfig);
+    return createSqliteDatabaseClient(dbConfig, overrides);
   }
 
   return knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides));

packages/backend-common/src/database/sqlite3.test.ts

@@ -15,6 +15,7 @@
  */
 
 import { ConfigReader } from '@backstage/config';
+import path from 'path';
 import {
   buildSqliteDatabaseConfig,
   createSqliteDatabaseClient,
@@ -25,25 +26,70 @@ describe('sqlite3', () => {
     new ConfigReader({ client: 'sqlite3', connection });
 
   describe('buildSqliteDatabaseConfig', () => {
-    it('buidls a string connection', () => {
+    it('builds an in-memory connection', () => {
       expect(buildSqliteDatabaseConfig(createConfig(':memory:'))).toEqual({
         client: 'sqlite3',
-        connection: ':memory:',
+        connection: { filename: ':memory:' },
         useNullAsDefault: true,
       });
     });
 
-    it('builds a filename connection', () => {
+    it('builds an in-memory connection by override with filename', () => {
+      expect(
+        buildSqliteDatabaseConfig(
+          createConfig(path.join('path', 'to', 'foo')),
+          { connection: ':memory:' },
+        ),
+      ).toEqual({
+        client: 'sqlite3',
+        connection: { filename: ':memory:' },
+        useNullAsDefault: true,
+      });
+    });
+
+    it('builds a persistent connection, normalize config with filename', () => {
+      expect(
+        buildSqliteDatabaseConfig(createConfig(path.join('path', 'to', 'foo'))),
+      ).toEqual({
+        client: 'sqlite3',
+        connection: { filename: path.join('path', 'to', 'foo') },
+        useNullAsDefault: true,
+      });
+    });
+
+    it('builds a persistent connection', () => {
       expect(
         buildSqliteDatabaseConfig(
           createConfig({
-            filename: '/path/to/foo',
+            filename: path.join('path', 'to', 'foo'),
           }),
         ),
       ).toEqual({
         client: 'sqlite3',
         connection: {
-          filename: '/path/to/foo',
+          filename: path.join('path', 'to', 'foo'),
+        },
+        useNullAsDefault: true,
+      });
+    });
+
+    it('builds a persistent connection per database', () => {
+      expect(
+        buildSqliteDatabaseConfig(
+          createConfig({
+            filename: path.join('path', 'to', 'foo'),
+          }),
+          {
+            connection: {
+              database: 'my-database',
+            },
+          },
+        ),
+      ).toEqual({
+        client: 'sqlite3',
+        connection: {
+          filename: path.join('path', 'to', 'foo', 'my-database.sqlite'),
+          database: 'my-database',
         },
         useNullAsDefault: true,
       });
@@ -52,12 +98,12 @@ describe('sqlite3', () => {
     it('replaces the connection with an override', () => {
       expect(
         buildSqliteDatabaseConfig(createConfig(':memory:'), {
-          connection: { filename: '/path/to/foo' },
+          connection: { filename: path.join('path', 'to', 'foo') },
         }),
       ).toEqual({
         client: 'sqlite3',
         connection: {
-          filename: '/path/to/foo',
+          filename: path.join('path', 'to', 'foo'),
         },
         useNullAsDefault: true,
       });

packages/backend-common/src/database/sqlite3.ts

@@ -14,8 +14,10 @@
  * limitations under the License.
  */
 
-import knexFactory, { Knex } from 'knex';
 import { Config } from '@backstage/config';
+import { ensureDirSync } from 'fs-extra';
+import knexFactory, { Knex } from 'knex';
+import path from 'path';
 import { mergeDatabaseConfig } from './config';
 
 /**
@@ -29,6 +31,19 @@ export function createSqliteDatabaseClient(
   overrides?: Knex.Config,
 ) {
   const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides);
+
+  // If storage on disk is used, ensure that the directory exists
+  if (
+    (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename &&
+    (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename !==
+      ':memory:'
+  ) {
+    const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig;
+    const directory = path.dirname(filename);
+
+    ensureDirSync(directory);
+  }
+
   const database = knexFactory(knexConfig);
 
   database.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
@@ -47,12 +62,40 @@ export function createSqliteDatabaseClient(
 export function buildSqliteDatabaseConfig(
   dbConfig: Config,
   overrides?: Knex.Config,
-) {
-  return mergeDatabaseConfig(
-    dbConfig.get(),
+): Knex.Config {
+  const baseConfig = dbConfig.get<Knex.Config>();
+
+  // Normalize config to always contain a connection object
+  if (typeof baseConfig.connection === 'string') {
+    baseConfig.connection = { filename: baseConfig.connection };
+  }
+  if (overrides && typeof overrides.connection === 'string') {
+    overrides.connection = { filename: overrides.connection };
+  }
+
+  const config: Knex.Config = mergeDatabaseConfig(
+    {
+      connection: {},
+    },
+    baseConfig,
     {
       useNullAsDefault: true,
     },
     overrides,
   );
+
+  // If we don't create an in-memory database, interpret the connection string
+  // as a directory that contains multiple sqlite files based on the database
+  // name.
+  const database = (config.connection as Knex.ConnectionConfig).database;
+  const sqliteConnection = config.connection as Knex.Sqlite3ConnectionConfig;
+
+  if (database && sqliteConnection.filename !== ':memory:') {
+    sqliteConnection.filename = path.join(
+      sqliteConnection.filename,
+      `${database}.sqlite`,
+    );
+  }
+
+  return config;
 }

packages/cli/package.json

@@ -62,7 +62,7 @@
     "commander": "^6.1.0",
     "css-loader": "^3.5.3",
     "dashify": "^2.0.0",
-    "diff": "^4.0.2",
+    "diff": "^5.0.0",
     "esbuild": "^0.8.56",
     "eslint": "^7.1.0",
     "eslint-config-prettier": "^6.0.0",

packages/cli/src/commands/config/docs.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { JsonObject } from '@backstage/config';
+import { mergeConfigSchemas } from '@backstage/config-loader';
+import { Command } from 'commander';
+import { JSONSchema7 as JSONSchema } from 'json-schema';
+import openBrowser from 'react-dev-utils/openBrowser';
+import { loadCliConfig } from '../../lib/config';
+
+const DOCS_URL = 'https://config.backstage.io';
+
+export default async (cmd: Command) => {
+  const { schema: appSchemas } = await loadCliConfig({
+    args: [],
+    fromPackage: cmd.package,
+    mockEnv: true,
+  });
+
+  const schema = mergeConfigSchemas(
+    (appSchemas.serialize().schemas as JsonObject[]).map(
+      _ => _.value as JSONSchema,
+    ),
+  );
+
+  openBrowser(`${DOCS_URL}#schema=${JSON.stringify(schema)}`);
+};

packages/cli/src/commands/create-plugin/createPlugin.ts

@@ -106,24 +106,6 @@ export async function addPluginDependencyToApp(
   });
 }
 
-export async function addPluginImportToApp(
-  rootDir: string,
-  pluginVar: string,
-  pluginPackage: string,
-) {
-  const pluginExport = `export { ${pluginVar} } from '${pluginPackage}';`;
-  const pluginsFilePath = 'packages/app/src/plugins.ts';
-  const pluginsFile = resolvePath(rootDir, pluginsFilePath);
-
-  await Task.forItem('processing', pluginsFilePath, async () => {
-    await addExportStatement(pluginsFile, pluginExport).catch(error => {
-      throw new Error(
-        `Failed to import plugin in app: ${pluginsFile}: ${error.message}`,
-      );
-    });
-  });
-}
-
 export async function addPluginExtensionToApp(
   pluginId: string,
   extensionName: string,
@@ -320,7 +302,6 @@ export default async (cmd: Command) => {
       await addPluginDependencyToApp(paths.targetRoot, name, pluginVersion);
 
       Task.section('Import plugin in app');
-      await addPluginImportToApp(paths.targetRoot, pluginVar, name);
       await addPluginExtensionToApp(pluginId, extensionName, name);
     }
 

packages/cli/src/commands/index.ts

@@ -140,6 +140,15 @@ export function registerCommands(program: CommanderStatic) {
     .description('Run tests, forwarding args to Jest, defaulting to watch mode')
     .action(lazy(() => import('./testCommand').then(m => m.default)));
 
+  program
+    .command('config:docs')
+    .option(
+      '--package <name>',
+      'Only include the schema that applies to the given package',
+    )
+    .description('Browse the configuration reference documentation')
+    .action(lazy(() => import('./config/docs').then(m => m.default)));
+
   program
     .command('config:print')
     .option(

packages/core-api/src/app/App.test.tsx

@@ -212,6 +212,9 @@ describe('Integration Test', () => {
     expect(screen.getByText('extLink2: /foo/a')).toBeInTheDocument();
     expect(screen.getByText('extLink3: /sub1')).toBeInTheDocument();
     expect(screen.getByText('extLink4: /foo/b')).toBeInTheDocument();
+
+    // Plugins should be discovered through element tree
+    expect(app.getPlugins()).toEqual([plugin1, plugin2]);
   });
 
   it('runs happy paths without optional routes', async () => {

packages/core-api/src/app/App.tsx

@@ -52,6 +52,7 @@ import {
 } from '../extensions/traversal';
 import { IconComponent, IconComponentMap, IconKey } from '../icons';
 import { BackstagePlugin } from '../plugin';
+import { pluginCollector } from '../plugin/collectors';
 import { AnyRoutes } from '../plugin/types';
 import { RouteRef, ExternalRouteRef, SubRouteRef } from '../routing';
 import {
@@ -189,7 +190,7 @@ export class PrivateAppImpl implements BackstageApp {
 
   private readonly apis: Iterable<AnyApiFactory>;
   private readonly icons: IconComponentMap;
-  private readonly plugins: BackstagePlugin<any, any>[];
+  private readonly plugins: Set<BackstagePlugin<any, any>>;
   private readonly components: AppComponents;
   private readonly themes: AppTheme[];
   private readonly configLoader?: AppConfigLoader;
@@ -201,7 +202,7 @@ export class PrivateAppImpl implements BackstageApp {
   constructor(options: FullAppOptions) {
     this.apis = options.apis;
     this.icons = options.icons;
-    this.plugins = options.plugins;
+    this.plugins = new Set(options.plugins);
     this.components = options.components;
     this.themes = options.themes;
     this.configLoader = options.configLoader;
@@ -210,7 +211,7 @@ export class PrivateAppImpl implements BackstageApp {
   }
 
   getPlugins(): BackstagePlugin<any, any>[] {
-    return this.plugins;
+    return Array.from(this.plugins);
   }
 
   getSystemIcon(key: IconKey): IconComponent | undefined {
@@ -276,7 +277,6 @@ export class PrivateAppImpl implements BackstageApp {
 
   getProvider(): ComponentType<{}> {
     const appContext = new AppContextImpl(this);
-    const apiHolder = this.getApiHolder();
 
     const Provider = ({ children }: PropsWithChildren<{}>) => {
       const appThemeApi = useMemo(
@@ -292,11 +292,25 @@ export class PrivateAppImpl implements BackstageApp {
             routePaths: routePathCollector,
             routeParents: routeParentCollector,
             routeObjects: routeObjectCollector,
+            collectedPlugins: pluginCollector,
           },
         });
 
         validateRoutes(result.routePaths, result.routeParents);
 
+        // TODO(Rugvip): Restructure the public API so that we can get an immediate view of
+        //               the app, rather than having to wait for the provider to render.
+        //               For now we need to push the additional plugins we find during
+        //               collection and then make sure we initialize things afterwards.
+        result.collectedPlugins.forEach(plugin => this.plugins.add(plugin));
+        this.verifyPlugins(this.plugins);
+
+        // Initialize APIs once all plugins are available
+        if (this.apiHolder) {
+          throw new Error('Plugin holder was initialized too soon');
+        }
+        this.getApiHolder();
+
         return result;
       }, [children]);
 
@@ -340,7 +354,7 @@ export class PrivateAppImpl implements BackstageApp {
       }
 
       return (
-        <ApiProvider apis={apiHolder}>
+        <ApiProvider apis={this.getApiHolder()}>
           <AppContextProvider appContext={appContext}>
             <AppThemeProvider>
               <RoutingProvider
@@ -495,10 +509,15 @@ export class PrivateAppImpl implements BackstageApp {
     return this.apiHolder;
   }
 
-  verify() {
+  /**
+   * @deprecated
+   */
+  verify() {}
+
+  private verifyPlugins(plugins: Iterable<BackstagePlugin>) {
     const pluginIds = new Set<string>();
 
-    for (const plugin of this.plugins) {
+    for (const plugin of plugins) {
       const id = plugin.getId();
       if (pluginIds.has(id)) {
         throw new Error(`Duplicate plugin found '${id}'`);

packages/core/src/components/SupportButton/SupportButton.tsx

@@ -17,6 +17,7 @@
 import { HelpIcon, useApp } from '@backstage/core-api';
 import {
   Button,
+  DialogActions,
   List,
   ListItem,
   ListItemIcon,
@@ -127,6 +128,11 @@ export const SupportButton = ({ children }: PropsWithChildren<Props>) => {
           {items &&
             items.map((item, i) => <SupportListItem item={item} key={i} />)}
         </List>
+        <DialogActions>
+          <Button color="primary" onClick={popoverCloseHandler}>
+            Close
+          </Button>
+        </DialogActions>
       </Popover>
     </Fragment>
   );

packages/create-app/templates/default-app/packages/app/src/App.tsx

@@ -21,11 +21,9 @@ import { UserSettingsPage } from '@backstage/plugin-user-settings';
 import { apis } from './apis';
 import { entityPage } from './components/catalog/EntityPage';
 import { Root } from './components/Root';
-import * as plugins from './plugins';
 
 const app = createApp({
   apis,
-  plugins: Object.values(plugins),
   bindRoutes({ bind }) {
     bind(catalogPlugin.externalRoutes, {
       createComponent: scaffolderPlugin.routes.root,

packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx

@@ -208,10 +208,6 @@ const groupPage = (
         </Grid>
       </Grid>
     </EntityLayout.Route>
-
-    <EntityLayout.Route path="/diagram" title="Diagram">
-      <EntitySystemDiagramCard />
-    </EntityLayout.Route>
   </EntityLayout>
 );
 
@@ -230,6 +226,9 @@ const systemPage = (
         </Grid>
       </Grid>
     </EntityLayout.Route>
+    <EntityLayout.Route path="/diagram" title="Diagram">
+      <EntitySystemDiagramCard />
+    </EntityLayout.Route>
   </EntityLayout>
 );
 

packages/create-app/templates/default-app/packages/app/src/plugins.ts

@@ -1,9 +0,0 @@
-export { plugin as ApiDocs } from '@backstage/plugin-api-docs';
-export { plugin as CatalogPlugin } from '@backstage/plugin-catalog';
-export { plugin as CatalogImport } from '@backstage/plugin-catalog-import';
-export { plugin as GithubActions } from '@backstage/plugin-github-actions';
-export { plugin as ScaffolderPlugin } from '@backstage/plugin-scaffolder';
-export { plugin as TechDocsPlugin } from '@backstage/plugin-techdocs';
-export { plugin as TechRadar } from '@backstage/plugin-tech-radar';
-export { plugin as UserSettings } from '@backstage/plugin-user-settings';
-

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts

@@ -14,13 +14,15 @@
  * limitations under the License.
  */
 import { getVoidLogger } from '@backstage/backend-common';
-import {
-  BitbucketDiscoveryProcessor,
-  readBitbucketOrg,
-} from './BitbucketDiscoveryProcessor';
+import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 import { ConfigReader } from '@backstage/config';
 import { LocationSpec } from '@backstage/catalog-model';
-import { BitbucketClient, PagedResponse } from './bitbucket';
+import {
+  BitbucketClient,
+  BitbucketRepositoryParser,
+  PagedResponse,
+} from './bitbucket';
+import { results } from './index';
 
 function pagedResponse(values: any): PagedResponse<any> {
   return {
@@ -30,11 +32,6 @@ function pagedResponse(values: any): PagedResponse<any> {
 }
 
 describe('BitbucketDiscoveryProcessor', () => {
-  const client: jest.Mocked<BitbucketClient> = {
-    listProjects: jest.fn(),
-    listRepositories: jest.fn(),
-  } as any;
-
   afterEach(() => jest.resetAllMocks());
 
   describe('reject unrelated entries', () => {
@@ -81,137 +78,236 @@ describe('BitbucketDiscoveryProcessor', () => {
   });
 
   describe('handles repositories', () => {
+    const processor = BitbucketDiscoveryProcessor.fromConfig(
+      new ConfigReader({
+        integrations: {
+          bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }],
+        },
+      }),
+      { logger: getVoidLogger() },
+    );
+
     it('output all repositories', async () => {
-      const target =
-        'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml';
-
-      client.listProjects.mockResolvedValue(
-        pagedResponse([{ key: 'backstage' }, { key: 'demo' }]),
-      );
-      client.listRepositories.mockResolvedValueOnce(
-        pagedResponse([
-          {
-            slug: 'backstage',
-            links: {
-              self: [
-                {
-                  href:
-                    'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse',
-                },
-              ],
-            },
-          },
-        ]),
-      );
-      client.listRepositories.mockResolvedValueOnce(
-        pagedResponse([
-          {
-            slug: 'demo',
-            links: {
-              self: [
-                {
-                  href:
-                    'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse',
-                },
-              ],
-            },
-          },
-        ]),
-      );
-
-      const actual = await readBitbucketOrg(client, target);
-      expect(actual.scanned).toBe(2);
-      expect(actual.matches).toContainEqual({
-        type: 'url',
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
         target:
-          'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml',
+          'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml',
+      };
+
+      jest
+        .spyOn(BitbucketClient.prototype, 'listProjects')
+        .mockResolvedValue(
+          pagedResponse([{ key: 'backstage' }, { key: 'demo' }]),
+        );
+      jest
+        .spyOn(BitbucketClient.prototype, 'listRepositories')
+        .mockResolvedValueOnce(
+          pagedResponse([
+            {
+              slug: 'backstage',
+              links: {
+                self: [
+                  {
+                    href:
+                      'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse',
+                  },
+                ],
+              },
+            },
+          ]),
+        );
+      jest
+        .spyOn(BitbucketClient.prototype, 'listRepositories')
+        .mockResolvedValueOnce(
+          pagedResponse([
+            {
+              slug: 'demo',
+              links: {
+                self: [
+                  {
+                    href:
+                      'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse',
+                  },
+                ],
+              },
+            },
+          ]),
+        );
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml',
+        },
+        optional: true,
       });
-      expect(actual.matches).toContainEqual({
-        type: 'url',
-        target:
-          'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml',
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml',
+        },
+        optional: true,
       });
     });
 
     it('output repositories with wildcards', async () => {
-      const target =
-        'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml';
-
-      client.listProjects.mockResolvedValue(
-        pagedResponse([{ key: 'backstage' }]),
-      );
-      client.listRepositories.mockResolvedValueOnce(
-        pagedResponse([
-          { slug: 'backstage' },
-          {
-            slug: 'techdocs-cli',
-            links: {
-              self: [
-                {
-                  href:
-                    'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse',
-                },
-              ],
-            },
-          },
-          {
-            slug: 'techdocs-container',
-            links: {
-              self: [
-                {
-                  href:
-                    'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse',
-                },
-              ],
-            },
-          },
-        ]),
-      );
-
-      const actual = await readBitbucketOrg(client, target);
-      expect(actual.scanned).toBe(3);
-      expect(actual.matches).toContainEqual({
-        type: 'url',
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
         target:
-          'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
+          'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml',
+      };
+
+      jest
+        .spyOn(BitbucketClient.prototype, 'listProjects')
+        .mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
+      jest
+        .spyOn(BitbucketClient.prototype, 'listRepositories')
+        .mockResolvedValueOnce(
+          pagedResponse([
+            { slug: 'backstage' },
+            {
+              slug: 'techdocs-cli',
+              links: {
+                self: [
+                  {
+                    href:
+                      'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse',
+                  },
+                ],
+              },
+            },
+            {
+              slug: 'techdocs-container',
+              links: {
+                self: [
+                  {
+                    href:
+                      'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse',
+                  },
+                ],
+              },
+            },
+          ]),
+        );
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
+        },
+        optional: true,
       });
-      expect(actual.matches).toContainEqual({
-        type: 'url',
-        target:
-          'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
+        },
+        optional: true,
       });
     });
     it('filter unrelated repositories', async () => {
-      const target =
-        'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml';
-
-      client.listProjects.mockResolvedValue(
-        pagedResponse([{ key: 'backstage' }]),
-      );
-      client.listRepositories.mockResolvedValue(
-        pagedResponse([
-          { slug: 'abstest' },
-          { slug: 'testxyz' },
-          {
-            slug: 'test',
-            links: {
-              self: [
-                {
-                  href:
-                    'https://bitbucket.mycompany.com/projects/backstage/repos/test',
-                },
-              ],
-            },
-          },
-        ]),
-      );
-
-      const actual = await readBitbucketOrg(client, target);
-      expect(actual.scanned).toBe(3);
-      expect(actual.matches).toContainEqual({
-        type: 'url',
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
         target:
           'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
+      };
+
+      jest
+        .spyOn(BitbucketClient.prototype, 'listProjects')
+        .mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
+      jest
+        .spyOn(BitbucketClient.prototype, 'listRepositories')
+        .mockResolvedValue(
+          pagedResponse([
+            { slug: 'abstest' },
+            { slug: 'testxyz' },
+            {
+              slug: 'test',
+              links: {
+                self: [
+                  {
+                    href:
+                      'https://bitbucket.mycompany.com/projects/backstage/repos/test',
+                  },
+                ],
+              },
+            },
+          ]),
+        );
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
+        },
+        optional: true,
+      });
+    });
+  });
+
+  describe('Custom repository parser', () => {
+    const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({}) {
+      yield results.location(
+        {
+          type: 'custom-location-type',
+          target: 'custom-target',
+        },
+        true,
+      );
+    };
+
+    const processor = BitbucketDiscoveryProcessor.fromConfig(
+      new ConfigReader({
+        integrations: {
+          bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }],
+        },
+      }),
+      { parser: customRepositoryParser, logger: getVoidLogger() },
+    );
+
+    it('use custom repository parser', async () => {
+      const location: LocationSpec = {
+        type: 'bitbucket-discovery',
+        target:
+          'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
+      };
+
+      jest
+        .spyOn(BitbucketClient.prototype, 'listProjects')
+        .mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
+      jest
+        .spyOn(BitbucketClient.prototype, 'listRepositories')
+        .mockResolvedValue(pagedResponse([{ slug: 'test' }]));
+
+      const emitter = jest.fn();
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledTimes(1);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'custom-location-type',
+          target: 'custom-target',
+        },
+        optional: true,
       });
     });
   });

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts

@@ -21,15 +21,24 @@ import {
   ScmIntegrations,
 } from '@backstage/integration';
 import { LocationSpec } from '@backstage/catalog-model';
-import { BitbucketClient, paginated } from './bitbucket';
+import {
+  Repository,
+  BitbucketRepositoryParser,
+  BitbucketClient,
+  defaultRepositoryParser,
+  paginated,
+} from './bitbucket';
 import { CatalogProcessor, CatalogProcessorEmit } from './types';
-import { results } from './index';
 
 export class BitbucketDiscoveryProcessor implements CatalogProcessor {
   private readonly integrations: ScmIntegrationRegistry;
+  private readonly parser: BitbucketRepositoryParser;
   private readonly logger: Logger;
 
-  static fromConfig(config: Config, options: { logger: Logger }) {
+  static fromConfig(
+    config: Config,
+    options: { parser?: BitbucketRepositoryParser; logger: Logger },
+  ) {
     const integrations = ScmIntegrations.fromConfig(config);
 
     return new BitbucketDiscoveryProcessor({
@@ -40,9 +49,11 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
 
   constructor(options: {
     integrations: ScmIntegrationRegistry;
+    parser?: BitbucketRepositoryParser;
     logger: Logger;
   }) {
     this.integrations = options.integrations;
+    this.parser = options.parser || defaultRepositoryParser;
     this.logger = options.logger;
   }
 
@@ -73,18 +84,18 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
     const startTimestamp = Date.now();
     this.logger.info(`Reading Bitbucket repositories from ${location.target}`);
 
+    const { catalogPath } = parseUrl(location.target);
+
     const result = await readBitbucketOrg(client, location.target);
 
     for (const repository of result.matches) {
-      emit(
-        results.location(
-          repository,
-          // Not all locations may actually exist, since the user defined them as a wildcard pattern.
-          // Thus, we emit them as optional and let the downstream processor find them while not outputting
-          // an error if it couldn't.
-          true,
-        ),
-      );
+      for await (const entity of this.parser({
+        client: client,
+        repository: repository,
+        path: catalogPath,
+      })) {
+        emit(entity);
+      }
     }
 
     const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
@@ -100,7 +111,7 @@ export async function readBitbucketOrg(
   client: BitbucketClient,
   target: string,
 ): Promise<Result> {
-  const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target);
+  const { projectSearchPath, repoSearchPath } = parseUrl(target);
   const projects = paginated(options => client.listProjects(options));
   const result: Result = {
     scanned: 0,
@@ -116,12 +127,8 @@ export async function readBitbucketOrg(
     );
     for await (const repository of repositories) {
       result.scanned++;
-
       if (repoSearchPath.test(repository.slug)) {
-        result.matches.push({
-          type: 'url',
-          target: `${repository.links.self[0].href}${catalogPath}`,
-        });
+        result.matches.push(repository);
       }
     }
   }
@@ -152,5 +159,5 @@ function escapeRegExp(str: string): RegExp {
 
 type Result = {
   scanned: number;
-  matches: LocationSpec[];
+  matches: Repository[];
 };

plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts

@@ -118,8 +118,16 @@ describe('GithubDiscoveryProcessor', () => {
       };
       mockGetOrganizationRepositories.mockResolvedValueOnce({
         repositories: [
-          { name: 'backstage', url: 'https://github.com/backstage/backstage' },
-          { name: 'demo', url: 'https://github.com/backstage/demo' },
+          {
+            name: 'backstage',
+            url: 'https://github.com/backstage/backstage',
+            isArchived: false,
+          },
+          {
+            name: 'demo',
+            url: 'https://github.com/backstage/demo',
+            isArchived: false,
+          },
         ],
       });
       const emitter = jest.fn();
@@ -153,14 +161,20 @@ describe('GithubDiscoveryProcessor', () => {
       };
       mockGetOrganizationRepositories.mockResolvedValueOnce({
         repositories: [
-          { name: 'backstage', url: 'https://github.com/backstage/backstage' },
+          {
+            name: 'backstage',
+            url: 'https://github.com/backstage/backstage',
+            isArchived: false,
+          },
           {
             name: 'techdocs-cli',
             url: 'https://github.com/backstage/techdocs-cli',
+            isArchived: false,
           },
           {
             name: 'techdocs-container',
             url: 'https://github.com/backstage/techdocs-container',
+            isArchived: false,
           },
         ],
       });
@@ -187,21 +201,32 @@ describe('GithubDiscoveryProcessor', () => {
         optional: true,
       });
     });
-    it('filter unrelated repositories', async () => {
+    it('filter unrelated and archived repositories', async () => {
       const location: LocationSpec = {
         type: 'github-discovery',
         target: 'https://github.com/backstage/test/blob/master/catalog.yaml',
       };
       mockGetOrganizationRepositories.mockResolvedValueOnce({
         repositories: [
-          { name: 'abstest', url: 'https://github.com/backstage/abctest' },
+          {
+            name: 'abstest',
+            url: 'https://github.com/backstage/abctest',
+            isArchived: false,
+          },
           {
             name: 'test',
             url: 'https://github.com/backstage/test',
+            isArchived: false,
+          },
+          {
+            name: 'test-archived',
+            url: 'https://github.com/backstage/test',
+            isArchived: true,
           },
           {
             name: 'testxyz',
             url: 'https://github.com/backstage/testxyz',
+            isArchived: false,
           },
         ],
       });

plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts

@@ -78,7 +78,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
     this.logger.info(`Reading GitHub repositories from ${location.target}`);
 
     const { repositories } = await getOrganizationRepositories(client, org);
-    const matching = repositories.filter(r => repoSearchPath.test(r.name));
+    const matching = repositories.filter(
+      r => !r.isArchived && repoSearchPath.test(r.name),
+    );
 
     const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
     this.logger.debug(

plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { defaultRepositoryParser } from './BitbucketRepositoryParser';
+import { Project, Repository } from './types';
+import { BitbucketClient } from './client';
+import { results } from '../index';
+
+describe('BitbucketRepositoryParser', () => {
+  describe('defaultRepositoryParser', () => {
+    it('emits location', async () => {
+      const browseUrl =
+        'https://bitbucket.mycompany.com/projects/project-key/repos/repo-slug/browse';
+      const path = '/catalog-info.yaml';
+      const expected = [
+        results.location(
+          {
+            type: 'url',
+            target: `${browseUrl}${path}`,
+          },
+          true,
+        ),
+      ];
+      const actual = await defaultRepositoryParser({
+        client: {} as BitbucketClient,
+        repository: {
+          project: {} as Project,
+          slug: 'repo-slug',
+          links: {
+            self: [{ href: browseUrl }],
+          },
+        } as Repository,
+        path: path,
+      });
+
+      let i = 0;
+      for await (const entity of actual) {
+        expect(entity).toStrictEqual(expected[i]);
+        i++;
+      }
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Repository } from './types';
+import { CatalogProcessorResult } from '../types';
+import { results } from '../index';
+import { BitbucketClient } from './client';
+
+export type BitbucketRepositoryParser = (options: {
+  client: BitbucketClient;
+  repository: Repository;
+  path: string;
+}) => AsyncIterable<CatalogProcessorResult>;
+
+export const defaultRepositoryParser: BitbucketRepositoryParser = async function* defaultRepositoryParser({
+  repository,
+  path,
+}) {
+  yield results.location(
+    {
+      type: 'url',
+      target: `${repository.links.self[0].href}${path}`,
+    },
+    // Not all locations may actually exist, since the user defined them as a wildcard pattern.
+    // Thus, we emit them as optional and let the downstream processor find them while not outputting
+    // an error if it couldn't.
+    true,
+  );
+};

plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts

@@ -41,6 +41,15 @@ export class BitbucketClient {
     );
   }
 
+  async getRaw(
+    projectKey: string,
+    repo: string,
+    path: string,
+  ): Promise<Response> {
+    const request = `${this.config.apiBaseUrl}/projects/${projectKey}/repos/${repo}/raw/${path}`;
+    return fetch(request, getBitbucketRequestOptions(this.config));
+  }
+
   private async pagedRequest(
     endpoint: string,
     options?: ListOptions,

plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts

@@ -15,3 +15,6 @@
  */
 export { BitbucketClient, paginated } from './client';
 export type { PagedResponse } from './client';
+export * from './types';
+export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser';
+export { defaultRepositoryParser } from './BitbucketRepositoryParser';

plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export type Project = {
+  key: string;
+};
+
+export type Repository = {
+  project: Project;
+  slug: string;
+  links: Record<string, Link[]>;
+};
+
+export type Link = {
+  href: string;
+};

plugins/catalog-backend/src/ingestion/processors/github/github.test.ts

@@ -162,10 +162,12 @@ describe('github', () => {
               {
                 name: 'backstage',
                 url: 'https://github.com/backstage/backstage',
+                isArchived: false,
               },
               {
                 name: 'demo',
                 url: 'https://github.com/backstage/demo',
+                isArchived: true,
               },
             ],
             pageInfo: {
@@ -177,10 +179,15 @@ describe('github', () => {
 
       const output = {
         repositories: [
-          { name: 'backstage', url: 'https://github.com/backstage/backstage' },
+          {
+            name: 'backstage',
+            url: 'https://github.com/backstage/backstage',
+            isArchived: false,
+          },
           {
             name: 'demo',
             url: 'https://github.com/backstage/demo',
+            isArchived: true,
           },
         ],
       };

plugins/catalog-backend/src/ingestion/processors/github/github.ts

@@ -56,6 +56,7 @@ export type Team = {
 export type Repository = {
   name: string;
   url: string;
+  isArchived: boolean;
 };
 
 export type Connection<T> = {
@@ -234,6 +235,7 @@ export async function getOrganizationRepositories(
         nodes {
           name
           url
+          isArchived
         }
         pageInfo {
           hasNextPage

plugins/github-deployments/src/api/index.ts

@@ -24,6 +24,10 @@ export type GithubDeployment = {
     abbreviatedOid: string;
     commitUrl: string;
   };
+  creator: {
+    login: string;
+  };
+  payload: string;
 };
 
 export interface GithubDeploymentsApi {
@@ -55,6 +59,10 @@ query deployments($owner: String!, $repo: String!, $last: Int) {
           abbreviatedOid
           commitUrl
         }
+        creator {
+          login
+        }
+        payload
       }
     }
   }

plugins/github-deployments/src/components/GithubDeploymentsCard.test.tsx

@@ -26,7 +26,11 @@ import {
 
 import { fireEvent } from '@testing-library/react';
 import { msw, renderInTestApp } from '@backstage/test-utils';
-import { GithubDeploymentsApiClient, githubDeploymentsApiRef } from '../api';
+import {
+  GithubDeployment,
+  GithubDeploymentsApiClient,
+  githubDeploymentsApiRef,
+} from '../api';
 import { githubDeploymentsPlugin } from '../plugin';
 import { GithubDeploymentsCard } from './GithubDeploymentsCard';
 
@@ -39,6 +43,8 @@ import {
 
 import { setupServer } from 'msw/node';
 import { graphql } from 'msw';
+import { GithubDeploymentsTable } from './GithubDeploymentsTable';
+import { Box } from '@material-ui/core';
 
 jest.mock('@backstage/plugin-catalog-react', () => ({
   useEntity: () => {
@@ -159,5 +165,39 @@ describe('github-deployments', () => {
       ).toBeInTheDocument();
       expect(await rendered.findByText('failure')).toBeInTheDocument();
     });
+
+    it('should display extra columns', async () => {
+      worker.use(
+        graphql.query('deployments', (_, res, ctx) =>
+          res(ctx.data(responseStub)),
+        ),
+      );
+
+      const renderTargetFromPayload = (payload: string) => {
+        const parsedPayload = JSON.parse(payload);
+        return parsedPayload?.target || 'unknown';
+      };
+
+      const extraColumn = {
+        title: 'Target',
+        render: (row: GithubDeployment): JSX.Element => (
+          <Box>{renderTargetFromPayload(row.payload)}</Box>
+        ),
+      };
+
+      const columns = [
+        ...GithubDeploymentsTable.defaultDeploymentColumns,
+        extraColumn,
+      ];
+
+      const rendered = await renderInTestApp(
+        <ApiProvider apis={apis}>
+          <GithubDeploymentsCard columns={columns} />
+        </ApiProvider>,
+      );
+
+      expect(await rendered.findByText('moon')).toBeInTheDocument();
+      expect(await rendered.findByText('sun')).toBeInTheDocument();
+    });
   });
 });

plugins/github-deployments/src/components/GithubDeploymentsCard.tsx

@@ -17,23 +17,26 @@ import React from 'react';
 import {
   MissingAnnotationEmptyState,
   ResponseErrorPanel,
+  TableColumn,
   useApi,
 } from '@backstage/core';
 import { useAsyncRetry } from 'react-use';
-import { githubDeploymentsApiRef } from '../api';
+import { GithubDeployment, githubDeploymentsApiRef } from '../api';
 import { useEntity } from '@backstage/plugin-catalog-react';
 import {
   GITHUB_PROJECT_SLUG_ANNOTATION,
   isGithubDeploymentsAvailable,
 } from '../Router';
-import GithubDeploymentsTable from './GithubDeploymentsTable/GithubDeploymentsTable';
+import { GithubDeploymentsTable } from './GithubDeploymentsTable/GithubDeploymentsTable';
 
 const GithubDeploymentsComponent = ({
   projectSlug,
   last,
+  columns,
 }: {
   projectSlug: string;
   last: number;
+  columns: TableColumn<GithubDeployment>[];
 }) => {
   const api = useApi(githubDeploymentsApiRef);
   const [owner, repo] = projectSlug.split('/');
@@ -51,11 +54,18 @@ const GithubDeploymentsComponent = ({
       deployments={value || []}
       isLoading={loading}
       reload={reload}
+      columns={columns}
     />
   );
 };
 
-export const GithubDeploymentsCard = ({ last }: { last?: number }) => {
+export const GithubDeploymentsCard = ({
+  last,
+  columns,
+}: {
+  last?: number;
+  columns?: TableColumn<GithubDeployment>[];
+}) => {
   const { entity } = useEntity();
 
   return !isGithubDeploymentsAvailable(entity) ? (
@@ -66,6 +76,7 @@ export const GithubDeploymentsCard = ({ last }: { last?: number }) => {
         entity?.metadata.annotations?.[GITHUB_PROJECT_SLUG_ANNOTATION] || ''
       }
       last={last || 10}
+      columns={columns || GithubDeploymentsTable.defaultDeploymentColumns}
     />
   );
 };

plugins/github-deployments/src/components/GithubDeploymentsTable/GithubDeploymentsTable.tsx

@@ -14,19 +14,12 @@
  * limitations under the License.
  */
 import React from 'react';
-import {
-  StatusPending,
-  StatusRunning,
-  StatusOK,
-  Table,
-  TableColumn,
-  StatusAborted,
-  StatusError,
-} from '@backstage/core';
+import { Table, TableColumn } from '@backstage/core';
 import { GithubDeployment } from '../../api';
-import { DateTime } from 'luxon';
-import { Box, Typography, Link, makeStyles } from '@material-ui/core';
+import { Typography, makeStyles } from '@material-ui/core';
 import SyncIcon from '@material-ui/icons/Sync';
+import * as columnFactories from './columns';
+import { defaultDeploymentColumns } from './presets';
 
 const useStyles = makeStyles(theme => ({
   empty: {
@@ -36,63 +29,19 @@ const useStyles = makeStyles(theme => ({
   },
 }));
 
-const statusIndicator = (value: string): React.ReactNode => {
-  switch (value) {
-    case 'PENDING':
-      return <StatusPending />;
-    case 'IN_PROGRESS':
-      return <StatusRunning />;
-    case 'ACTIVE':
-      return <StatusOK />;
-    case 'ERROR':
-    case 'FAILURE':
-      return <StatusError />;
-    default:
-      return <StatusAborted />;
-  }
-};
-
-const columns: TableColumn<GithubDeployment>[] = [
-  {
-    title: 'Environment',
-    field: 'environment',
-    highlight: true,
-  },
-  {
-    title: 'Status',
-    render: (row: GithubDeployment): React.ReactNode => (
-      <Box display="flex" alignItems="center">
-        {statusIndicator(row.state)}
-        <Typography variant="caption">{row.state}</Typography>
-      </Box>
-    ),
-  },
-  {
-    title: 'Commit',
-    render: (row: GithubDeployment): React.ReactNode => (
-      <Link href={row.commit.commitUrl} target="_blank" rel="noopener">
-        {row.commit.abbreviatedOid}
-      </Link>
-    ),
-  },
-  {
-    title: 'Last Updated',
-    render: (row: GithubDeployment): React.ReactNode =>
-      DateTime.fromISO(row.updatedAt).toRelative({ locale: 'en' }),
-  },
-];
-
 type GithubDeploymentsTableProps = {
   deployments: GithubDeployment[];
   isLoading: boolean;
   reload: () => void;
+  columns: TableColumn<GithubDeployment>[];
 };
 
-const GithubDeploymentsTable = ({
+export function GithubDeploymentsTable({
   deployments,
   isLoading,
   reload,
-}: GithubDeploymentsTableProps) => {
+  columns,
+}: GithubDeploymentsTableProps) {
   const classes = useStyles();
 
   return (
@@ -119,6 +68,8 @@ const GithubDeploymentsTable = ({
       }
     />
   );
-};
+}
 
-export default GithubDeploymentsTable;
+GithubDeploymentsTable.columns = columnFactories;
+
+GithubDeploymentsTable.defaultDeploymentColumns = defaultDeploymentColumns;

plugins/github-deployments/src/components/GithubDeploymentsTable/columns.tsx

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React from 'react';
+import {
+  StatusPending,
+  StatusRunning,
+  StatusOK,
+  TableColumn,
+  StatusAborted,
+  StatusError,
+  Link,
+} from '@backstage/core';
+import { GithubDeployment } from '../../api';
+import { DateTime } from 'luxon';
+import { Box, Typography } from '@material-ui/core';
+
+export const GithubStateIndicator = ({ state }: { state: string }) => {
+  switch (state) {
+    case 'PENDING':
+      return <StatusPending />;
+    case 'IN_PROGRESS':
+      return <StatusRunning />;
+    case 'ACTIVE':
+      return <StatusOK />;
+    case 'ERROR':
+    case 'FAILURE':
+      return <StatusError />;
+    default:
+      return <StatusAborted />;
+  }
+};
+
+export function createEnvironmentColumn(): TableColumn<GithubDeployment> {
+  return {
+    title: 'Environment',
+    field: 'environment',
+    highlight: true,
+  };
+}
+
+export function createStatusColumn(): TableColumn<GithubDeployment> {
+  return {
+    title: 'Status',
+    render: (row: GithubDeployment): JSX.Element => (
+      <Box display="flex" alignItems="center">
+        <GithubStateIndicator state={row.state} />
+        <Typography variant="caption">{row.state}</Typography>
+      </Box>
+    ),
+  };
+}
+
+export function createCommitColumn(): TableColumn<GithubDeployment> {
+  return {
+    title: 'Commit',
+    render: (row: GithubDeployment): JSX.Element => (
+      <Link to={row.commit.commitUrl} target="_blank" rel="noopener">
+        {row.commit.abbreviatedOid}
+      </Link>
+    ),
+  };
+}
+
+export function createCreatorColumn(): TableColumn<GithubDeployment> {
+  return {
+    title: 'Creator',
+    field: 'creator.login',
+  };
+}
+
+export function createLastUpdatedColumn(): TableColumn<GithubDeployment> {
+  return {
+    title: 'Last Updated',
+    render: (row: GithubDeployment): JSX.Element => (
+      <Box>{DateTime.fromISO(row.updatedAt).toRelative({ locale: 'en' })}</Box>
+    ),
+  };
+}

plugins/github-deployments/src/components/GithubDeploymentsTable/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { GithubDeploymentsTable } from './GithubDeploymentsTable';

plugins/github-deployments/src/components/GithubDeploymentsTable/presets.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { TableColumn } from '@backstage/core';
+import { GithubDeployment } from '../../api';
+import {
+  createEnvironmentColumn,
+  createStatusColumn,
+  createCommitColumn,
+  createLastUpdatedColumn,
+  createCreatorColumn,
+} from './columns';
+
+export const defaultDeploymentColumns: TableColumn<GithubDeployment>[] = [
+  createEnvironmentColumn(),
+  createStatusColumn(),
+  createCommitColumn(),
+  createCreatorColumn(),
+  createLastUpdatedColumn(),
+];

plugins/github-deployments/src/index.ts

@@ -14,4 +14,5 @@
  * limitations under the License.
  */
 export { githubDeploymentsPlugin, EntityGithubDeploymentsCard } from './plugin';
+export { GithubDeploymentsTable } from './components/GithubDeploymentsTable';
 export { isGithubDeploymentsAvailable } from './Router';

plugins/github-deployments/src/mocks/mocks.ts

@@ -49,6 +49,10 @@ export const responseStub: QueryResponse = {
             commitUrl: 'https://exampleapi.com/123456789',
             abbreviatedOid: '12345',
           },
+          creator: {
+            login: 'robot-user-001',
+          },
+          payload: '{"target":"moon"}',
         },
         {
           state: 'pending',
@@ -58,6 +62,10 @@ export const responseStub: QueryResponse = {
             commitUrl: 'https://exampleapi.com/543212345',
             abbreviatedOid: '54321',
           },
+          creator: {
+            login: 'robot-user-002',
+          },
+          payload: '{"target":"sun"}',
         },
       ],
     },
@@ -76,6 +84,10 @@ export const refreshedResponseStub: QueryResponse = {
             commitUrl: 'https://exampleapi.com/123456789',
             abbreviatedOid: '12345',
           },
+          creator: {
+            login: 'robot-user-001',
+          },
+          payload: '',
         },
         {
           state: 'failure',
@@ -85,6 +97,10 @@ export const refreshedResponseStub: QueryResponse = {
             commitUrl: 'https://exampleapi.com/543212345',
             abbreviatedOid: '54321',
           },
+          creator: {
+            login: 'robot-user-002',
+          },
+          payload: '',
         },
       ],
     },

plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts

@@ -52,6 +52,7 @@ describe('ConfigClusterLocator', () => {
         serviceAccountToken: undefined,
         url: 'http://localhost:8080',
         authProvider: 'serviceAccount',
+        skipTLSVerify: false,
       },
     ]);
   });
@@ -64,11 +65,13 @@ describe('ConfigClusterLocator', () => {
           serviceAccountToken: 'token',
           url: 'http://localhost:8080',
           authProvider: 'serviceAccount',
+          skipTLSVerify: false,
         },
         {
           name: 'cluster2',
           url: 'http://localhost:8081',
           authProvider: 'google',
+          skipTLSVerify: true,
         },
       ],
     });
@@ -83,12 +86,14 @@ describe('ConfigClusterLocator', () => {
         serviceAccountToken: 'token',
         url: 'http://localhost:8080',
         authProvider: 'serviceAccount',
+        skipTLSVerify: false,
       },
       {
         name: 'cluster2',
         serviceAccountToken: undefined,
         url: 'http://localhost:8081',
         authProvider: 'google',
+        skipTLSVerify: true,
       },
     ]);
   });

plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts

@@ -33,6 +33,7 @@ export class ConfigClusterLocator implements KubernetesClustersSupplier {
           name: c.getString('name'),
           url: c.getString('url'),
           serviceAccountToken: c.getOptionalString('serviceAccountToken'),
+          skipTLSVerify: c.getOptionalBoolean('skipTLSVerify') ?? false,
           authProvider: c.getString('authProvider'),
         };
       }),

plugins/kubernetes-backend/src/cluster-locator/index.test.ts

@@ -53,12 +53,14 @@ describe('getCombinedClusterDetails', () => {
         serviceAccountToken: 'token',
         url: 'http://localhost:8080',
         authProvider: 'serviceAccount',
+        skipTLSVerify: false,
       },
       {
         name: 'cluster2',
         serviceAccountToken: undefined,
         url: 'http://localhost:8081',
         authProvider: 'google',
+        skipTLSVerify: false,
       },
     ]);
   });

plugins/kubernetes-backend/src/service/KubernetesClientProvider.test.ts

@@ -34,6 +34,7 @@ describe('KubernetesClientProvider', () => {
       url: 'http://localhost:9999',
       serviceAccountToken: 'TOKEN',
       authProvider: 'serviceAccount',
+      skipTLSVerify: false,
     });
 
     expect(result.basePath).toBe('http://localhost:9999');
@@ -41,6 +42,7 @@ describe('KubernetesClientProvider', () => {
     const auth = (result as any).authentications.default;
     expect(auth.users[0].token).toBe('TOKEN');
     expect(auth.clusters[0].name).toBe('cluster-name');
+    expect(auth.clusters[0].skipTLSVerify).toBe(false);
 
     expect(mockGetKubeConfig.mock.calls.length).toBe(1);
   });
@@ -57,6 +59,7 @@ describe('KubernetesClientProvider', () => {
       url: 'http://localhost:9999',
       serviceAccountToken: 'TOKEN',
       authProvider: 'serviceAccount',
+      skipTLSVerify: false,
     });
 
     expect(result.basePath).toBe('http://localhost:9999');

plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts

@@ -30,8 +30,7 @@ export class KubernetesClientProvider {
     const cluster = {
       name: clusterDetails.name,
       server: clusterDetails.url,
-      // TODO configure this
-      skipTLSVerify: true,
+      skipTLSVerify: clusterDetails.skipTLSVerify,
     };
 
     // TODO configure

plugins/kubernetes-backend/src/types/types.ts

@@ -30,6 +30,7 @@ export interface ClusterDetails {
   url: string;
   authProvider: string;
   serviceAccountToken?: string | undefined;
+  skipTLSVerify?: boolean;
 }
 
 export interface KubernetesRequestBody {

plugins/scaffolder/src/components/MultistepJsonForm/schema.test.ts

@@ -20,6 +20,7 @@ describe('transformSchemaToProps', () => {
   it('transforms deep schema', () => {
     const inputSchema = {
       type: 'object',
+      'ui:welp': 'warp',
       properties: {
         field1: {
           type: 'string',
@@ -53,6 +54,7 @@ describe('transformSchemaToProps', () => {
       },
     };
     const expectedUiSchema = {
+      'ui:welp': 'warp',
       field1: {
         'ui:derp': 'herp',
       },

plugins/scaffolder/src/components/MultistepJsonForm/schema.ts

@@ -22,41 +22,39 @@ function isObject(value: unknown): value is JsonObject {
 }
 
 function extractUiSchema(schema: JsonObject, uiSchema: JsonObject) {
+  if (!isObject(schema)) {
+    return;
+  }
+
   const { properties } = schema;
+
+  for (const propName in schema) {
+    if (!schema.hasOwnProperty(propName)) {
+      continue;
+    }
+
+    if (propName.startsWith('ui:')) {
+      uiSchema[propName] = schema[propName];
+      delete schema[propName];
+    }
+  }
+
   if (!isObject(properties)) {
     return;
   }
+
   for (const propName in properties) {
     if (!properties.hasOwnProperty(propName)) {
       continue;
     }
+
     const schemaNode = properties[propName];
     if (!isObject(schemaNode)) {
       continue;
     }
-
-    if (schemaNode.type === 'object') {
-      const innerUiSchema = {};
-      uiSchema[propName] = innerUiSchema;
-      extractUiSchema(schemaNode, innerUiSchema);
-    } else {
-      for (const innerKey in schemaNode) {
-        if (!schemaNode.hasOwnProperty(innerKey)) {
-          continue;
-        }
-        const innerValue = schemaNode[innerKey];
-        if (innerKey.startsWith('ui:')) {
-          const innerUiSchema = uiSchema[propName] || {};
-          if (!isObject(innerUiSchema)) {
-            throw new TypeError('Unexpected non-object in uiSchema');
-          }
-          uiSchema[propName] = innerUiSchema;
-
-          innerUiSchema[innerKey] = innerValue;
-          delete schemaNode[innerKey];
-        }
-      }
-    }
+    const innerUiSchema = {};
+    uiSchema[propName] = innerUiSchema;
+    extractUiSchema(schemaNode, innerUiSchema);
   }
 }
 

yarn.lock

@@ -6857,11 +6857,11 @@
   integrity sha512-Fx+NpfOO0CpeYX2g9bkvX8O5qh9wrU1sOF4g8sft4Mu7z+qfe387YlyY8w8daDyDsKY5vUxM0yxkAYnbkRbZEw==
 
 "@types/webpack-node-externals@^2.5.0":
-  version "2.5.0"
-  resolved "https://registry.npmjs.org/@types/webpack-node-externals/-/webpack-node-externals-2.5.0.tgz#bcd161af84a4960416e5850e06931b35321c6654"
-  integrity sha512-KaWfhUQlpWknM/CMBKhV7i0vxX/N2xEy3WeaE500s4ZNxC4nLnKB+0F3gD3Fg+5octPq0nn8ZlfFR/P3dSkXpw==
+  version "2.5.1"
+  resolved "https://registry.npmjs.org/@types/webpack-node-externals/-/webpack-node-externals-2.5.1.tgz#0f00036bce0f405ceabc092e415b734059fe5505"
+  integrity sha512-Cwg6+FQogkImRMF5nu5bKsLoZlwNCzpEyvxIzJM0ZgkkuKP7TrmQ3suOvNKKG1O4luxXZroKGo0mMC5EN5gPBA==
   dependencies:
-    "@types/webpack" "*"
+    "@types/webpack" "^4"
 
 "@types/webpack-sources@*":
   version "0.1.6"
@@ -11527,11 +11527,16 @@ diff@1.4.0:
   resolved "https://registry.npmjs.org/diff/-/diff-1.4.0.tgz#7f28d2eb9ee7b15a97efd89ce63dcfdaa3ccbabf"
   integrity sha1-fyjS657nsVqX79ic5j3P2qPMur8=
 
-diff@^4.0.1, diff@^4.0.2:
+diff@^4.0.1:
   version "4.0.2"
   resolved "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
   integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==
 
+diff@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz#7ed6ad76d859d030787ec35855f5b1daf31d852b"
+  integrity sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==
+
 diffie-hellman@^5.0.0:
   version "5.0.3"
   resolved "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz#40e8ee98f55a2149607146921c63e1ae5f3d2875"