diff --git a/.changeset/gentle-days-return.md b/.changeset/gentle-days-return.md deleted file mode 100644 index 96552173bd..0000000000 --- a/.changeset/gentle-days-return.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -'@backstage/plugin-api-docs': patch -'@backstage/plugin-bitrise': patch -'@backstage/plugin-catalog-import': patch -'@backstage/plugin-circleci': patch -'@backstage/plugin-cost-insights': patch -'@backstage/plugin-explore': patch -'@backstage/plugin-fossa': patch -'@backstage/plugin-github-actions': patch -'@backstage/plugin-github-deployments': patch -'@backstage/plugin-graphiql': patch -'@backstage/plugin-jenkins': patch -'@backstage/plugin-lighthouse': patch -'@backstage/plugin-pagerduty': patch -'@backstage/plugin-register-component': patch -'@backstage/plugin-rollbar': patch -'@backstage/plugin-sentry': patch -'@backstage/plugin-sonarqube': patch ---- - -Updated README to have up-to-date install instructions. diff --git a/.changeset/healthy-phones-press.md b/.changeset/healthy-phones-press.md deleted file mode 100644 index c771caefc5..0000000000 --- a/.changeset/healthy-phones-press.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/core': patch ---- - -Allow passing NavLinkProps to SidebarItem component to use in NavLink diff --git a/.changeset/kind-suns-melt.md b/.changeset/kind-suns-melt.md deleted file mode 100644 index 7cbec28fea..0000000000 --- a/.changeset/kind-suns-melt.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-api-docs': patch ---- - -Added fields filtering in get API entities to avoid the requesting of unused data diff --git a/.changeset/little-trees-fold.md b/.changeset/little-trees-fold.md deleted file mode 100644 index 4963dd5aa1..0000000000 --- a/.changeset/little-trees-fold.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/create-app': patch ---- - -Updates the end to end test in the app to match the new catalog index page title. To apply this change to an existing app, update `packages/app/cypress/integration/app.js` to search for `"My Company Catalog"` instead of `"My Company Service Catalog"`. diff --git a/.changeset/pink-islands-help.md b/.changeset/pink-islands-help.md deleted file mode 100644 index c7672f484f..0000000000 --- a/.changeset/pink-islands-help.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-code-coverage-backend': patch ---- - -Include migrations diff --git a/.changeset/rare-parrots-share.md b/.changeset/rare-parrots-share.md deleted file mode 100644 index 917ec5a18e..0000000000 --- a/.changeset/rare-parrots-share.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@backstage/plugin-catalog': patch -'@backstage/plugin-scaffolder': patch ---- - -Removed unused `swr` dependency. diff --git a/.changeset/search-odd-ads-invite.md b/.changeset/search-odd-ads-invite.md deleted file mode 100644 index 0dde3d0c1f..0000000000 --- a/.changeset/search-odd-ads-invite.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-search-backend-node': patch ---- - -Introduced Scheduler which is responsible for adding new tasks to a schedule together with it's interval timer as well as starting and stopping the scheduler processes. diff --git a/.changeset/six-rocks-allow.md b/.changeset/six-rocks-allow.md deleted file mode 100644 index f207654f45..0000000000 --- a/.changeset/six-rocks-allow.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/techdocs-common': patch ---- - -Gracefully handle HTTP request failures in download method of AzureBlobStorage publisher. diff --git a/.changeset/sixty-dragons-retire.md b/.changeset/sixty-dragons-retire.md deleted file mode 100644 index 62965c6e88..0000000000 --- a/.changeset/sixty-dragons-retire.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-search': patch ---- - -Use `RouteRef` to generate path to search page. diff --git a/.changeset/spotty-pigs-bathe.md b/.changeset/spotty-pigs-bathe.md new file mode 100644 index 0000000000..0585f335dc --- /dev/null +++ b/.changeset/spotty-pigs-bathe.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-tech-radar': patch +--- + +Update README for composability diff --git a/.changeset/techdocs-lovely-wombats-relate.md b/.changeset/techdocs-lovely-wombats-relate.md deleted file mode 100644 index 33ed205c89..0000000000 --- a/.changeset/techdocs-lovely-wombats-relate.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@backstage/plugin-techdocs': patch ---- - -Use `EntityRefLink` in header and use relations to reference the owner of the -document. diff --git a/.changeset/techdocs-mean-humans-hammer.md b/.changeset/techdocs-mean-humans-hammer.md deleted file mode 100644 index 7697650250..0000000000 --- a/.changeset/techdocs-mean-humans-hammer.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-techdocs': patch ---- - -Fix TechDocs landing page table wrong copied link diff --git a/.changeset/techdocs-quiet-pens-prove.md b/.changeset/techdocs-quiet-pens-prove.md deleted file mode 100644 index 379c20e72a..0000000000 --- a/.changeset/techdocs-quiet-pens-prove.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-techdocs': patch ---- - -Add customization and exportable components for TechDocs landing page diff --git a/.changeset/techdocs-red-donkeys-share.md b/.changeset/techdocs-red-donkeys-share.md deleted file mode 100644 index 29d932d0b8..0000000000 --- a/.changeset/techdocs-red-donkeys-share.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-techdocs': patch ---- - -Disable color transitions on links to avoid issues in dark mode. diff --git a/.changeset/techdocs-swift-waves-sleep.md b/.changeset/techdocs-swift-waves-sleep.md deleted file mode 100644 index 79d10931cd..0000000000 --- a/.changeset/techdocs-swift-waves-sleep.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -'@backstage/plugin-techdocs': minor ---- - -Make `techdocsStorageApiRef` and `techdocsApiRef` use interfaces instead of the -actual implementation classes. - -This renames the classes `TechDocsApi` to `TechDocsClient` and `TechDocsStorageApi` -to `TechDocsStorageClient` and renames the interfaces `TechDocs` to `TechDocsApi` -and `TechDocsStorage` to `TechDocsStorageApi` to comply the pattern elsewhere in -the project. This also fixes the types returned by some methods on those -interfaces. diff --git a/.changeset/techdocs-young-walls-decide.md b/.changeset/techdocs-young-walls-decide.md deleted file mode 100644 index 8dc0373234..0000000000 --- a/.changeset/techdocs-young-walls-decide.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-techdocs': patch ---- - -Rework state management to avoid rendering multiple while navigating between pages. diff --git a/.changeset/thick-cobras-switch.md b/.changeset/thick-cobras-switch.md new file mode 100644 index 0000000000..002c724421 --- /dev/null +++ b/.changeset/thick-cobras-switch.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-proxy-backend': patch +--- + +Prefix proxy routes with `/` if not present in configuration diff --git a/.changeset/thin-coins-compete.md b/.changeset/thin-coins-compete.md deleted file mode 100644 index 36511ec1cb..0000000000 --- a/.changeset/thin-coins-compete.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-scaffolder-backend': patch ---- - -Added the `nebula-preview` preview to `Octokit` for repository visibility. diff --git a/.changeset/thirty-humans-knock.md b/.changeset/thirty-humans-knock.md deleted file mode 100644 index 47545af282..0000000000 --- a/.changeset/thirty-humans-knock.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/core': patch ---- - -Improve owner example value in `MissingAnnotationEmptyState`. diff --git a/.changeset/tough-walls-wash.md b/.changeset/tough-walls-wash.md new file mode 100644 index 0000000000..a45adcfbf6 --- /dev/null +++ b/.changeset/tough-walls-wash.md @@ -0,0 +1,9 @@ +--- +'@backstage/catalog-model': patch +'@backstage/cli': patch +'@backstage/config-loader': patch +'@backstage/config': patch +'@backstage/plugin-scaffolder': patch +--- + +Bump `json-schema` dependency from `0.2.5` to `0.3.0`. diff --git a/.changeset/witty-lies-shave.md b/.changeset/witty-lies-shave.md deleted file mode 100644 index 68fd4b1137..0000000000 --- a/.changeset/witty-lies-shave.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog': patch ---- - -Switch out the time-based personal greeting for a plain title on the catalog index page, and remove the clocks for different timezones. diff --git a/contrib/chart/backstage/README.md b/contrib/chart/backstage/README.md index 7d04a2cf1e..40bb2666bc 100644 --- a/contrib/chart/backstage/README.md +++ b/contrib/chart/backstage/README.md @@ -129,6 +129,16 @@ For the CA, create a `configMap` named `--postgres-ca` kubectl create configmap my-company-backstage-postgres-ca --from-file=ca.crt" ``` +or disable CA mount + +```yaml +backend: + postgresCertMountEnabled: false + +lighthouse: + postgresCertMountEnabled: false +``` + > Where the release name contains the chart name "backstage" then only the release name will be used. Now install the helm chart: diff --git a/contrib/chart/backstage/templates/_helpers.tpl b/contrib/chart/backstage/templates/_helpers.tpl index 5d49ba6975..123d4dba5b 100644 --- a/contrib/chart/backstage/templates/_helpers.tpl +++ b/contrib/chart/backstage/templates/_helpers.tpl @@ -214,7 +214,7 @@ Postgres port for the backend {{- .Values.postgresql.service.port }} {{- else if .Values.appConfig.backend.database.connection.port -}} {{- .Values.appConfig.backend.database.connection.port }} -{{ else }} +{{- else -}} 5432 {{- end -}} {{- end -}} diff --git a/contrib/chart/backstage/templates/backend-deployment.yaml b/contrib/chart/backstage/templates/backend-deployment.yaml index 144fc93e62..0fa6b7fbe2 100644 --- a/contrib/chart/backstage/templates/backend-deployment.yaml +++ b/contrib/chart/backstage/templates/backend-deployment.yaml @@ -26,6 +26,13 @@ spec: {{- end}} containers: - name: {{ .Chart.Name }}-backend + command: ["node"] + args: + - "packages/backend" + - "--config" + - "app-config.yaml" + - "--config" + - {{ printf "/usr/src/app/%s" (include "backstage.appConfigFilename" .) | quote }} image: {{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag }} imagePullPolicy: {{ .Values.backend.image.pullPolicy }} ports: @@ -49,16 +56,20 @@ spec: name: {{ include "backend.postgresql.passwordSecret" .}} key: postgresql-password volumeMounts: + {{- if .Values.backend.postgresCertMountEnabled }} - name: postgres-ca mountPath: {{ include "backstage.backend.postgresCaDir" . }} + {{- end }} - name: app-config mountPath: {{ printf "/usr/src/app/%s" (include "backstage.appConfigFilename" .) }} subPath: {{ include "backstage.appConfigFilename" . }} volumes: + {{- if .Values.backend.postgresCertMountEnabled }} - name: postgres-ca configMap: name: {{ include "backstage.fullname" . }}-postgres-ca + {{- end }} - name: app-config configMap: name: {{ include "backstage.fullname" . }}-app-config @@ -83,5 +94,5 @@ spec: app: backstage component: backend - type: ClusterIP + type: {{ .Values.backend.serviceType }} {{- end }} diff --git a/contrib/chart/backstage/templates/frontend-deployment.yaml b/contrib/chart/backstage/templates/frontend-deployment.yaml index 9c674cc160..04983513a6 100644 --- a/contrib/chart/backstage/templates/frontend-deployment.yaml +++ b/contrib/chart/backstage/templates/frontend-deployment.yaml @@ -1,3 +1,4 @@ +{{- if .Values.frontend.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -46,7 +47,6 @@ spec: {{- if .Values.global.nodeSelector }} nodeSelector: {{- toYaml .Values.global.nodeSelector | nindent 8 }} {{- end }} -{{- if .Values.frontend.enabled }} --- apiVersion: v1 kind: Service @@ -62,5 +62,5 @@ spec: app: backstage component: frontend - type: ClusterIP + type: {{ .Values.frontend.serviceType }} {{- end }} diff --git a/contrib/chart/backstage/templates/lighthouse-config.yaml b/contrib/chart/backstage/templates/lighthouse-config.yaml index fdf4bd8b06..7b8cad42dc 100644 --- a/contrib/chart/backstage/templates/lighthouse-config.yaml +++ b/contrib/chart/backstage/templates/lighthouse-config.yaml @@ -1,3 +1,4 @@ +{{- if .Values.lighthouse.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -7,4 +8,5 @@ data: PGUSER: {{ include "lighthouse.postgresql.user" . | quote }} PGPORT: {{ include "lighthouse.postgresql.port" . | quote }} PGHOST: {{ include "lighthouse.postgresql.host" . | quote }} - PGPATH_TO_CA: {{ include "backstage.lighthouse.postgresCaFilename" . | quote }} \ No newline at end of file + PGPATH_TO_CA: {{ include "backstage.lighthouse.postgresCaFilename" . | quote }} +{{- end }} \ No newline at end of file diff --git a/contrib/chart/backstage/templates/lighthouse-deployment.yaml b/contrib/chart/backstage/templates/lighthouse-deployment.yaml index d6e1ce19dd..849c48d7a6 100644 --- a/contrib/chart/backstage/templates/lighthouse-deployment.yaml +++ b/contrib/chart/backstage/templates/lighthouse-deployment.yaml @@ -51,14 +51,18 @@ spec: name: {{ include "lighthouse.postgresql.passwordSecret" . }} key: postgresql-password + {{- if .Values.lighthouse.postgresCertMountEnabled }} volumeMounts: - name: postgres-ca mountPath: {{ include "backstage.lighthouse.postgresCaDir" . }} + {{- end }} + {{- if .Values.lighthouse.postgresCertMountEnabled }} volumes: - name: postgres-ca configMap: name: {{ include "backstage.fullname" . }}-postgres-ca + {{- end }} {{- if .Values.global.nodeSelector }} nodeSelector: {{- toYaml .Values.global.nodeSelector | nindent 8 }} @@ -78,5 +82,5 @@ spec: app: backstage component: lighthouse-audit-service - type: ClusterIP + type: {{ .Values.lighthouse.serviceType }} {{- end }} diff --git a/contrib/chart/backstage/templates/postgresql-password-backend-secret.yaml b/contrib/chart/backstage/templates/postgresql-password-backend-secret.yaml new file mode 100644 index 0000000000..a71cef7b21 --- /dev/null +++ b/contrib/chart/backstage/templates/postgresql-password-backend-secret.yaml @@ -0,0 +1,15 @@ +{{- if not .Values.postgresql.enabled }} +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: {{ include "backend.postgresql.passwordSecret" . }} + labels: + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-delete-policy": "before-hook-creation" +data: + postgresql-password: {{ .Values.appConfig.backend.database.connection.password | b64enc }} +{{- end }} diff --git a/contrib/chart/backstage/templates/postgresql-password-secret.yaml b/contrib/chart/backstage/templates/postgresql-password-lighthouse-secret.yaml similarity index 51% rename from contrib/chart/backstage/templates/postgresql-password-secret.yaml rename to contrib/chart/backstage/templates/postgresql-password-lighthouse-secret.yaml index e53369adb9..e7374e70db 100644 --- a/contrib/chart/backstage/templates/postgresql-password-secret.yaml +++ b/contrib/chart/backstage/templates/postgresql-password-lighthouse-secret.yaml @@ -1,18 +1,4 @@ -{{- if not .Values.postgresql.enabled }} ---- -apiVersion: v1 -kind: Secret -type: Opaque -metadata: - name: {{ include "backend.postgresql.passwordSecret" . }} - labels: - release: {{ .Release.Name }} - annotations: - "helm.sh/hook": "pre-install,pre-upgrade" - "helm.sh/hook-delete-policy": "before-hook-creation" -data: - postgresql-password: {{ .Values.appConfig.backend.database.connection.password | b64enc }} -{{- end }} +{{- if .Values.lighthouse.enabled }} {{- if not .Values.postgresql.enabled }} --- apiVersion: v1 @@ -28,3 +14,4 @@ metadata: data: postgresql-password: {{ .Values.lighthouse.database.connection.password | b64enc }} {{- end }} +{{- end }} diff --git a/contrib/chart/backstage/values.yaml b/contrib/chart/backstage/values.yaml index bd80bc22b6..9f70dac6dc 100644 --- a/contrib/chart/backstage/values.yaml +++ b/contrib/chart/backstage/values.yaml @@ -3,13 +3,14 @@ # Declare variables to be passed into your templates. frontend: - enabled: true + enabled: false replicaCount: 1 image: repository: martinaif/backstage-k8s-demo-frontend tag: test1 pullPolicy: IfNotPresent containerPort: 80 + serviceType: ClusterIP resources: requests: memory: 128Mi @@ -23,9 +24,11 @@ backend: replicaCount: 1 image: repository: martinaif/backstage-k8s-demo-backend - tag: test1 + tag: 20210423T1550 pullPolicy: IfNotPresent containerPort: 7000 + serviceType: ClusterIP + postgresCertMountEnabled: true resources: requests: memory: 512Mi @@ -40,6 +43,8 @@ lighthouse: tag: latest pullPolicy: IfNotPresent containerPort: 3003 + serviceType: ClusterIP + postgresCertMountEnabled: true resources: requests: memory: 128Mi diff --git a/contrib/docker/frontend-with-nginx/Dockerfile.dockerbuild b/contrib/docker/frontend-with-nginx/Dockerfile.dockerbuild index dda9779eb0..41f931544f 100644 --- a/contrib/docker/frontend-with-nginx/Dockerfile.dockerbuild +++ b/contrib/docker/frontend-with-nginx/Dockerfile.dockerbuild @@ -51,9 +51,8 @@ FROM nginx:mainline RUN apt-get update && apt-get -y install jq && rm -rf /var/lib/apt/lists/* COPY --from=build /app/packages/app/dist /usr/share/nginx/html -COPY docker/default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker/run.sh /usr/local/bin/run.sh +COPY docker/default.conf.template /etc/nginx/templates/default.conf.template -CMD run.sh +COPY docker/inject-config.sh /docker-entrypoint.d/40-inject-config.sh ENV PORT 80 diff --git a/contrib/docker/frontend-with-nginx/Dockerfile.hostbuild b/contrib/docker/frontend-with-nginx/Dockerfile.hostbuild index 6e5912dd67..1e0134017f 100644 --- a/contrib/docker/frontend-with-nginx/Dockerfile.hostbuild +++ b/contrib/docker/frontend-with-nginx/Dockerfile.hostbuild @@ -33,9 +33,9 @@ FROM nginx:mainline RUN apt-get update && apt-get -y install jq && rm -rf /var/lib/apt/lists/* COPY packages/app/dist /usr/share/nginx/html -COPY docker/default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker/run.sh /usr/local/bin/run.sh +COPY docker/default.conf.template /etc/nginx/templates/default.conf.template -CMD run.sh +COPY docker/inject-config.sh /docker-entrypoint.d/40-inject-config.sh ENV PORT 80 + diff --git a/contrib/docker/frontend-with-nginx/docker/run.sh b/contrib/docker/frontend-with-nginx/docker/inject-config.sh similarity index 80% rename from contrib/docker/frontend-with-nginx/docker/run.sh rename to contrib/docker/frontend-with-nginx/docker/inject-config.sh index fdb1742a07..a0cea96ff3 100755 --- a/contrib/docker/frontend-with-nginx/docker/run.sh +++ b/contrib/docker/frontend-with-nginx/docker/inject-config.sh @@ -2,13 +2,6 @@ set -Eeuo pipefail -# Run nginx as root -sed -i 's/user nginx.*$//' /etc/nginx/nginx.conf - -# Write selected env vars to nginx config -envsubst '$PORT' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf - -# Inject runtime config into the client function inject_config() { # Read runtime config from env in the same way as the @backstage/config-loader package local config @@ -41,4 +34,3 @@ function inject_config() { inject_config -exec nginx -g 'daemon off;' diff --git a/contrib/terraform/techdocs-s3-storage/terraform.tf b/contrib/terraform/techdocs-s3-storage/terraform.tf index ca267f0f12..f256663f43 100644 --- a/contrib/terraform/techdocs-s3-storage/terraform.tf +++ b/contrib/terraform/techdocs-s3-storage/terraform.tf @@ -2,15 +2,15 @@ # Variables #========================== -variable "backstage-bucket" { +variable "backstage_bucket" { default = "backstage_bucket_for_my_corp" } -variable "backstage-iam" { +variable "backstage_iam" { default = "backstage" } -variable "shared-managed-tag-value" { +variable "shared_managed_tag" { default = "terraform_for_my_corp" } @@ -19,7 +19,7 @@ variable "shared-managed-tag-value" { #========================== resource "aws_s3_bucket" "backstage" { - bucket = var.backstage-bucket + bucket = var.backstage_bucket acl = "private" provider = aws @@ -36,8 +36,8 @@ resource "aws_s3_bucket" "backstage" { } tags = { - Name = var.backstage-bucket - "Managed By Terraform" = var.shared-managed-tag-value + Name = var.backstage_bucket + "Managed By Terraform" = var.shared_managed_tag } } @@ -56,16 +56,16 @@ resource "aws_s3_bucket_public_access_block" "backstage" { #========================== resource "aws_iam_user" "backstage" { - name = var.backstage-iam + name = var.backstage_iam } resource "aws_iam_user_policy" "backstage" { - name = var.backstage-iam + name = var.backstage_iam user = aws_iam_user.backstage.name - policy = data.aws_iam_policy_document.backstage-policy.json + policy = data.aws_iam_policy_document.backstage_policy.json } -data "aws_iam_policy_document" "backstage-policy" { +data "aws_iam_policy_document" "backstage_policy" { statement { actions = [ "s3:PutObject", diff --git a/cypress/src/integration/integrations.ts b/cypress/src/integration/integrations.ts new file mode 100644 index 0000000000..5ee077502a --- /dev/null +++ b/cypress/src/integration/integrations.ts @@ -0,0 +1,76 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/// +import 'os'; + +describe('Integrations', () => { + describe('ReadTree', () => { + it('should work for github', () => { + cy.loginAsGuest(); + + cy.request('POST', '/api/catalog/locations', { + target: + 'https://github.com/backstage-verification/test-repo/blob/main/**/*', + type: 'url', + }); + + cy.visit('/catalog'); + cy.contains('All').click(); + cy.get('table').should('contain', 'github-repo'); + cy.get('table').should('contain', 'github-repo-nested'); + }); + + // it('should work for azure', () => { + // cy.loginAsGuest(); + + // cy.request('POST', '/api/catalog/locations', { + // target: + // 'https://dev.azure.com/backstage-verification/_git/test-repo?path=*', + // type: 'url', + // }); + // }); + + it('should work for gitlab', () => { + cy.loginAsGuest(); + + cy.request('POST', '/api/catalog/locations', { + target: + 'https://gitlab.com/backstage-verification/test-repo/-/tree/master/**/*', + type: 'url', + }); + + cy.visit('/catalog'); + cy.contains('All').click(); + cy.get('table').should('contain', 'gitlab-repo'); + cy.get('table').should('contain', 'gitlab-repo-nested'); + }); + + it('should work for bitbucket', () => { + cy.loginAsGuest(); + + cy.request('POST', '/api/catalog/locations', { + target: + 'https://bitbucket.org/backstage-verification/test-repo/src/master/**/*', + type: 'url', + }); + + cy.visit('/catalog'); + cy.contains('All').click(); + cy.get('table').should('contain', 'bitbucket-repo'); + cy.get('table').should('contain', 'bitbucket-repo-nested'); + }); + }); +}); diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index d3b14dd888..eb087a9c5f 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -44,7 +44,7 @@ metadata: name: artist-web description: The place to be, for great artists labels: - system: public-websites + example.com/custom: custom_label_value annotations: example.com/service-discovery: artistweb circleci.com/project-slug: github/example-org/artist-website @@ -58,6 +58,7 @@ spec: type: website lifecycle: production owner: artist-relations-team + system: public-websites ``` This is the same entity as returned in JSON from the software catalog API: @@ -76,7 +77,7 @@ This is the same entity as returned in JSON from the software catalog API: "etag": "ZjU2MWRkZWUtMmMxZS00YTZiLWFmMWMtOTE1NGNiZDdlYzNk", "generation": 1, "labels": { - "system": "public-websites" + "example.com/custom": "custom_label_value" }, "links": [{ "url": "https://admin.example-org.com", @@ -90,7 +91,8 @@ This is the same entity as returned in JSON from the software catalog API: "spec": { "lifecycle": "production", "owner": "artist-relations-team", - "type": "website" + "type": "website", + "system": "public-websites" } } ``` diff --git a/docs/plugins/proxying.md b/docs/plugins/proxying.md index 50c56af7a6..3e20e37cad 100644 --- a/docs/plugins/proxying.md +++ b/docs/plugins/proxying.md @@ -36,7 +36,7 @@ Example: ```yaml # in app-config.yaml proxy: - '/simple-example': http://simple.example.com:8080 + simple-example: http://simple.example.com:8080 '/larger-example/v1': target: http://larger.example.com:8080/svc.v1 headers: @@ -46,10 +46,11 @@ proxy: ``` Each key under the proxy configuration entry is a route to match, below the -prefix that the proxy plugin is mounted on. It must start with a slash. For -example, if the backend mounts the proxy plugin as `/proxy`, the above -configuration will lead to the proxy acting on backend requests to -`/api/proxy/simple-example/...` and `/api/proxy/larger-example/v1/...`. +prefix that the proxy plugin is mounted on. If it does not start with a slash, +one will be prefixed automatically. For example, if the backend mounts the proxy +plugin as `/proxy`, the above configuration will lead to the proxy acting on +backend requests to `/api/proxy/simple-example/...` and +`/api/proxy/larger-example/v1/...`. The value inside each route is either a simple URL string, or an object on the format accepted by diff --git a/microsite/data/plugins/circleci.yaml b/microsite/data/plugins/circleci.yaml index 0acad832fa..cffbedca37 100644 --- a/microsite/data/plugins/circleci.yaml +++ b/microsite/data/plugins/circleci.yaml @@ -5,7 +5,7 @@ authorUrl: https://github.com/spotify category: CI/CD description: Automate your development process with CI hosted in the cloud or on a private server. documentation: https://github.com/backstage/backstage/tree/master/plugins/circleci -iconUrl: https://www.saaves.com/storage/brochure/logo-circleci-icon1583764538.png +iconUrl: img/circleci.png npmPackageName: '@backstage/plugin-circleci' tags: - ci diff --git a/microsite/data/plugins/cost-insights.yaml b/microsite/data/plugins/cost-insights.yaml index daa4f55a02..ec2d162b9d 100644 --- a/microsite/data/plugins/cost-insights.yaml +++ b/microsite/data/plugins/cost-insights.yaml @@ -5,7 +5,7 @@ authorUrl: https://github.com/spotify category: Discovery description: Visualize, understand and optimize your team's cloud costs. documentation: https://github.com/backstage/backstage/tree/master/plugins/cost-insights -iconUrl: https://www.materialui.co/materialIcons/editor/monetization_on_white_192x192.png +iconUrl: img/cost-insights.png npmPackageName: '@backstage/plugin-cost-insights' tags: - web diff --git a/microsite/static/img/circleci.png b/microsite/static/img/circleci.png new file mode 100644 index 0000000000..85f262e4b3 Binary files /dev/null and b/microsite/static/img/circleci.png differ diff --git a/microsite/static/img/cost-insights.png b/microsite/static/img/cost-insights.png new file mode 100644 index 0000000000..7296dd3524 Binary files /dev/null and b/microsite/static/img/cost-insights.png differ diff --git a/packages/app/CHANGELOG.md b/packages/app/CHANGELOG.md index 7262b2ae29..e9d3e63ad0 100644 --- a/packages/app/CHANGELOG.md +++ b/packages/app/CHANGELOG.md @@ -1,5 +1,47 @@ # example-app +## 0.2.26 + +### Patch Changes + +- Updated dependencies [1ce80ff02] +- Updated dependencies [b98de52ae] +- Updated dependencies [4c42ecca2] +- Updated dependencies [c614ede9a] +- Updated dependencies [9afcac5af] +- Updated dependencies [07a7806c3] +- Updated dependencies [f6efa71ee] +- Updated dependencies [19a4dd710] +- Updated dependencies [dcd54c7cd] +- Updated dependencies [da546ce00] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6fbd7beca] +- Updated dependencies [15cbe6815] +- Updated dependencies [39bdaa004] +- Updated dependencies [cb8c848a3] +- Updated dependencies [21fddf452] +- Updated dependencies [17915e29b] +- Updated dependencies [6eaecbd81] +- Updated dependencies [23769512a] +- Updated dependencies [1a142ae8a] + - @backstage/plugin-api-docs@0.4.12 + - @backstage/plugin-cost-insights@0.8.5 + - @backstage/plugin-github-actions@0.4.4 + - @backstage/plugin-catalog-import@0.5.4 + - @backstage/plugin-circleci@0.2.13 + - @backstage/plugin-explore@0.3.4 + - @backstage/plugin-graphiql@0.2.10 + - @backstage/plugin-jenkins@0.4.2 + - @backstage/plugin-lighthouse@0.2.15 + - @backstage/plugin-pagerduty@0.3.3 + - @backstage/plugin-rollbar@0.3.4 + - @backstage/plugin-sentry@0.3.9 + - @backstage/core@0.7.7 + - @backstage/plugin-scaffolder@0.9.2 + - @backstage/plugin-catalog@0.5.6 + - @backstage/plugin-search@0.3.5 + - @backstage/plugin-techdocs@0.9.0 + ## 0.2.25 ### Patch Changes diff --git a/packages/app/package.json b/packages/app/package.json index ba9af03310..402d09ade4 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,39 +1,39 @@ { "name": "example-app", - "version": "0.2.25", + "version": "0.2.26", "private": true, "bundled": true, "dependencies": { "@backstage/catalog-model": "^0.7.7", "@backstage/cli": "^0.6.9", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/integration-react": "^0.1.1", - "@backstage/plugin-api-docs": "^0.4.11", + "@backstage/plugin-api-docs": "^0.4.12", "@backstage/plugin-badges": "^0.2.0", - "@backstage/plugin-catalog": "^0.5.5", - "@backstage/plugin-catalog-import": "^0.5.3", + "@backstage/plugin-catalog": "^0.5.6", + "@backstage/plugin-catalog-import": "^0.5.4", "@backstage/plugin-catalog-react": "^0.1.3", - "@backstage/plugin-circleci": "^0.2.12", + "@backstage/plugin-circleci": "^0.2.13", "@backstage/plugin-cloudbuild": "^0.2.13", "@backstage/plugin-code-coverage": "^0.1.2", - "@backstage/plugin-cost-insights": "^0.8.4", - "@backstage/plugin-explore": "^0.3.2", + "@backstage/plugin-cost-insights": "^0.8.5", + "@backstage/plugin-explore": "^0.3.4", "@backstage/plugin-gcp-projects": "^0.2.5", - "@backstage/plugin-github-actions": "^0.4.2", - "@backstage/plugin-graphiql": "^0.2.9", - "@backstage/plugin-jenkins": "^0.4.1", + "@backstage/plugin-github-actions": "^0.4.4", + "@backstage/plugin-graphiql": "^0.2.10", + "@backstage/plugin-jenkins": "^0.4.2", "@backstage/plugin-kafka": "^0.2.6", "@backstage/plugin-kubernetes": "^0.4.2", - "@backstage/plugin-lighthouse": "^0.2.14", + "@backstage/plugin-lighthouse": "^0.2.15", "@backstage/plugin-newrelic": "^0.2.6", "@backstage/plugin-org": "^0.3.12", - "@backstage/plugin-pagerduty": "0.3.2", - "@backstage/plugin-rollbar": "^0.3.3", - "@backstage/plugin-scaffolder": "^0.9.1", - "@backstage/plugin-search": "^0.3.4", - "@backstage/plugin-sentry": "^0.3.8", + "@backstage/plugin-pagerduty": "0.3.3", + "@backstage/plugin-rollbar": "^0.3.4", + "@backstage/plugin-scaffolder": "^0.9.2", + "@backstage/plugin-search": "^0.3.5", + "@backstage/plugin-sentry": "^0.3.9", "@backstage/plugin-tech-radar": "^0.3.9", - "@backstage/plugin-techdocs": "^0.8.0", + "@backstage/plugin-techdocs": "^0.9.0", "@backstage/plugin-todo": "^0.1.0", "@backstage/plugin-user-settings": "^0.2.8", "@backstage/theme": "^0.2.6", diff --git a/packages/backend/Dockerfile b/packages/backend/Dockerfile index 02aab45609..31231a3a4a 100644 --- a/packages/backend/Dockerfile +++ b/packages/backend/Dockerfile @@ -22,7 +22,7 @@ RUN tar xzf skeleton.tar.gz && rm skeleton.tar.gz RUN yarn install --frozen-lockfile --production --network-timeout 300000 && rm -rf "$(yarn cache dir)" # Then copy the rest of the backend bundle, along with any other files we might want. -ADD packages/backend/dist/bundle.tar.gz app-config.yaml ./ +COPY packages/backend/dist/bundle.tar.gz app-config.yaml ./ RUN tar xzf bundle.tar.gz && rm bundle.tar.gz CMD ["node", "packages/backend", "--config", "app-config.yaml"] diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index dcd48aaf40..1691f53485 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -32,7 +32,7 @@ export default async function createPlugin({ indexBuilder.addCollator({ type: 'software-catalog', defaultRefreshIntervalSeconds: 600, - collator: new DefaultCatalogCollator(discovery), + collator: new DefaultCatalogCollator({ discovery }), }); const { scheduler } = await indexBuilder.build(); diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json index 64550b35d1..941877aec5 100644 --- a/packages/catalog-model/package.json +++ b/packages/catalog-model/package.json @@ -33,7 +33,7 @@ "@types/json-schema": "^7.0.5", "@types/yup": "^0.29.8", "ajv": "^7.0.3", - "json-schema": "^0.2.5", + "json-schema": "^0.3.0", "lodash": "^4.17.15", "uuid": "^8.0.0", "yup": "^0.29.3" diff --git a/packages/cli/package.json b/packages/cli/package.json index cd61193efe..1110efefe6 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -82,7 +82,7 @@ "inquirer": "^7.0.4", "jest": "^26.0.1", "jest-css-modules": "^2.1.0", - "json-schema": "^0.2.5", + "json-schema": "^0.3.0", "lodash": "^4.17.19", "mini-css-extract-plugin": "^0.9.0", "ora": "^5.3.0", diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 0a0a76c2d6..55d86314af 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -35,7 +35,7 @@ "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", "fs-extra": "^9.0.0", - "json-schema": "^0.2.5", + "json-schema": "^0.3.0", "json-schema-merge-allof": "^0.7.0", "typescript-json-schema": "^0.49.0", "yaml": "^1.9.2", diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 1b556ac1c3..974dcd3b91 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,13 @@ # @backstage/core +## 0.7.7 + +### Patch Changes + +- 9afcac5af: Allow passing NavLinkProps to SidebarItem component to use in NavLink +- e0c9ed759: Add `if` prop to `EntityLayout.Route` to conditionally render tabs +- 6eaecbd81: Improve owner example value in `MissingAnnotationEmptyState`. + ## 0.7.6 ### Patch Changes diff --git a/packages/core/package.json b/packages/core/package.json index 344ea10398..59391e122a 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/core", "description": "Core API used by Backstage plugins and apps", - "version": "0.7.6", + "version": "0.7.7", "private": false, "publishConfig": { "access": "public", diff --git a/packages/core/src/components/TabbedLayout/index.ts b/packages/core/src/components/TabbedLayout/index.ts index 744b56959e..fe72b199ec 100644 --- a/packages/core/src/components/TabbedLayout/index.ts +++ b/packages/core/src/components/TabbedLayout/index.ts @@ -14,3 +14,4 @@ * limitations under the License. */ export { TabbedLayout } from './TabbedLayout'; +export { RoutedTabs } from './RoutedTabs'; diff --git a/packages/create-app/CHANGELOG.md b/packages/create-app/CHANGELOG.md index c821a3ab81..430e7f003a 100644 --- a/packages/create-app/CHANGELOG.md +++ b/packages/create-app/CHANGELOG.md @@ -1,5 +1,44 @@ # @backstage/create-app +## 0.3.20 + +### Patch Changes + +- 73f3f5d78: Updates the end to end test in the app to match the new catalog index page title. To apply this change to an existing app, update `packages/app/cypress/integration/app.js` to search for `"My Company Catalog"` instead of `"My Company Service Catalog"`. +- Updated dependencies [1ce80ff02] +- Updated dependencies [4c42ecca2] +- Updated dependencies [c614ede9a] +- Updated dependencies [9afcac5af] +- Updated dependencies [07a7806c3] +- Updated dependencies [f6efa71ee] +- Updated dependencies [19a4dd710] +- Updated dependencies [a99e0bc42] +- Updated dependencies [dcd54c7cd] +- Updated dependencies [da546ce00] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6fbd7beca] +- Updated dependencies [15cbe6815] +- Updated dependencies [39bdaa004] +- Updated dependencies [cb8c848a3] +- Updated dependencies [21fddf452] +- Updated dependencies [17915e29b] +- Updated dependencies [a1783f306] +- Updated dependencies [6eaecbd81] +- Updated dependencies [23769512a] +- Updated dependencies [1a142ae8a] + - @backstage/plugin-api-docs@0.4.12 + - @backstage/plugin-github-actions@0.4.4 + - @backstage/plugin-catalog-import@0.5.4 + - @backstage/plugin-explore@0.3.4 + - @backstage/plugin-lighthouse@0.2.15 + - @backstage/core@0.7.7 + - @backstage/plugin-scaffolder@0.9.2 + - @backstage/plugin-catalog@0.5.6 + - @backstage/plugin-catalog-backend@0.8.1 + - @backstage/plugin-search@0.3.5 + - @backstage/plugin-techdocs@0.9.0 + - @backstage/plugin-scaffolder-backend@0.10.1 + ## 0.3.19 ### Patch Changes diff --git a/packages/create-app/package.json b/packages/create-app/package.json index 5cf6cdd78e..071a0a5ebd 100644 --- a/packages/create-app/package.json +++ b/packages/create-app/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/create-app", "description": "Create app package for Backstage", - "version": "0.3.19", + "version": "0.3.20", "private": false, "publishConfig": { "access": "public" diff --git a/packages/techdocs-common/CHANGELOG.md b/packages/techdocs-common/CHANGELOG.md index 9dd6447a14..1788502e45 100644 --- a/packages/techdocs-common/CHANGELOG.md +++ b/packages/techdocs-common/CHANGELOG.md @@ -1,5 +1,11 @@ # @backstage/techdocs-common +## 0.5.1 + +### Patch Changes + +- f4af06ebe: Gracefully handle HTTP request failures in download method of AzureBlobStorage publisher. + ## 0.5.0 ### Minor Changes diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index c41dce72b1..e749b33919 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/techdocs-common", "description": "Common functionalities for TechDocs, to be shared between techdocs-backend plugin and techdocs-cli", - "version": "0.5.0", + "version": "0.5.1", "main": "src/index.ts", "types": "src/index.ts", "private": false, diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index 15da3b7cfc..f094457346 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger } from '@backstage/backend-common'; import { Entity, - ENTITY_DEFAULT_NAMESPACE, EntityName, + ENTITY_DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import mockFs from 'mock-fs'; @@ -154,7 +155,6 @@ describe('AwsS3Publish', () => { }); it('should fail to publish a directory', async () => { - expect.assertions(3); const wrongPathToGeneratedDirectory = path.join( rootDir, 'wrong', @@ -171,23 +171,22 @@ describe('AwsS3Publish', () => { }), ).rejects.toThrowError(); - await publisher - .publish({ - entity, - directory: wrongPathToGeneratedDirectory, - }) - .catch(error => { - expect(error.message).toEqual( - // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error - // Issue reported https://github.com/tschaub/mock-fs/issues/118 - expect.stringContaining( - `Unable to upload file(s) to AWS S3. Error: Failed to read template directory: ENOENT, no such file or directory`, - ), - ); - expect(error.message).toEqual( - expect.stringContaining(wrongPathToGeneratedDirectory), - ); - }); + const fails = publisher.publish({ + entity, + directory: wrongPathToGeneratedDirectory, + }); + + // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error + // Issue reported https://github.com/tschaub/mock-fs/issues/118 + await expect(fails).rejects.toMatchObject({ + message: expect.stringContaining( + 'Unable to upload file(s) to AWS S3. Error: Failed to read template directory: ENOENT, no such file or directory', + ), + }); + await expect(fails).rejects.toMatchObject({ + message: expect.stringContaining(wrongPathToGeneratedDirectory), + }); + mockFs.restore(); }); }); @@ -265,18 +264,12 @@ describe('AwsS3Publish', () => { const entity = createMockEntity(); const entityRootDir = getEntityRootDir(entity); - await publisher - .fetchTechDocsMetadata(entityNameMock) - .catch(error => - expect(error).toEqual( - new Error( - `TechDocs metadata fetch failed, The file ${path.join( - entityRootDir, - 'techdocs_metadata.json', - )} does not exist !`, - ), - ), - ); + const fails = publisher.fetchTechDocsMetadata(entityNameMock); + + const errorPath = path.join(entityRootDir, 'techdocs_metadata.json'); + await expect(fails).rejects.toMatchObject({ + message: `TechDocs metadata fetch failed, The file ${errorPath} does not exist !`, + }); }); }); }); diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index c431e924a8..1160ffe49d 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger } from '@backstage/backend-common'; import { Entity, - ENTITY_DEFAULT_NAMESPACE, EntityName, + ENTITY_DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import mockFs from 'mock-fs'; @@ -157,7 +158,6 @@ describe('publishing with valid credentials', () => { }); it('should fail to publish a directory', async () => { - expect.assertions(1); const wrongPathToGeneratedDirectory = path.join( rootDir, 'wrong', @@ -168,22 +168,20 @@ describe('publishing with valid credentials', () => { const entity = createMockEntity(); - await publisher - .publish({ - entity, - directory: wrongPathToGeneratedDirectory, - }) - .catch(error => { - // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error - // Issue reported https://github.com/tschaub/mock-fs/issues/118 - expect.stringContaining( - `Unable to upload file(s) to Azure Blob Storage. Error: Failed to read template directory: ENOENT, no such file or directory`, - ); + const fails = publisher.publish({ + entity, + directory: wrongPathToGeneratedDirectory, + }); + + await expect(fails).rejects.toMatchObject({ + message: expect.stringContaining( + `Unable to upload file(s) to Azure Blob Storage. Error: Failed to read template directory: ENOENT, no such file or directory`, + ), + }); + await expect(fails).rejects.toMatchObject({ + message: expect.stringContaining(wrongPathToGeneratedDirectory), + }); - expect(error.message).toEqual( - expect.stringContaining(wrongPathToGeneratedDirectory), - ); - }); mockFs.restore(); }); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index 7e580b16bc..19e4938763 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger } from '@backstage/backend-common'; import { Entity, @@ -146,8 +147,6 @@ describe('GoogleGCSPublish', () => { }); it('should fail to publish a directory', async () => { - expect.assertions(3); - const wrongPathToGeneratedDirectory = path.join( rootDir, 'wrong', @@ -165,23 +164,21 @@ describe('GoogleGCSPublish', () => { }), ).rejects.toThrowError(); - await publisher - .publish({ - entity, - directory: wrongPathToGeneratedDirectory, - }) - .catch(error => { - expect(error.message).toEqual( - // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error - // Issue reported https://github.com/tschaub/mock-fs/issues/118 - expect.stringContaining( - `Unable to upload file(s) to Google Cloud Storage. Error: Failed to read template directory: ENOENT, no such file or directory`, - ), - ); - expect(error.message).toEqual( - expect.stringContaining(wrongPathToGeneratedDirectory), - ); - }); + const fails = publisher.publish({ + entity, + directory: wrongPathToGeneratedDirectory, + }); + + // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error + // Issue reported https://github.com/tschaub/mock-fs/issues/118 + await expect(fails).rejects.toMatchObject({ + message: expect.stringContaining( + `Unable to upload file(s) to Google Cloud Storage. Error: Failed to read template directory: ENOENT, no such file or directory`, + ), + }); + await expect(fails).rejects.toMatchObject({ + message: expect.stringContaining(wrongPathToGeneratedDirectory), + }); mockFs.restore(); }); @@ -264,16 +261,14 @@ describe('GoogleGCSPublish', () => { const entity = createMockEntity(); const entityRootDir = getEntityRootDir(entity); - await publisher - .fetchTechDocsMetadata(entityNameMock) - .catch(errorMessage => - expect(errorMessage).toEqual( - `The file ${path.join( - entityRootDir, - 'techdocs_metadata.json', - )} does not exist !`, - ), - ); + const fails = publisher.fetchTechDocsMetadata(entityNameMock); + + await expect(fails).rejects.toMatchObject({ + message: `The file ${path.join( + entityRootDir, + 'techdocs_metadata.json', + )} does not exist !`, + }); }); }); }); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 28fbb42ece..8adafc361a 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -168,7 +168,7 @@ export class GoogleGCSPublish implements PublisherBase { .createReadStream() .on('error', err => { this.logger.error(err.message); - reject(err.message); + reject(err); }) .on('data', chunk => { fileStreamChunks.push(chunk); diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts index 6f0aacede0..e179aeb9a7 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger } from '@backstage/backend-common'; import { Entity, - ENTITY_DEFAULT_NAMESPACE, EntityName, + ENTITY_DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import mockFs from 'mock-fs'; @@ -157,7 +158,6 @@ describe('OpenStackSwiftPublish', () => { }); it('should fail to publish a directory', async () => { - expect.assertions(3); const wrongPathToGeneratedDirectory = path.join( rootDir, 'wrong', @@ -174,23 +174,22 @@ describe('OpenStackSwiftPublish', () => { }), ).rejects.toThrowError(); - await publisher - .publish({ - entity, - directory: wrongPathToGeneratedDirectory, - }) - .catch(error => { - expect(error.message).toEqual( - // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error - // Issue reported https://github.com/tschaub/mock-fs/issues/118 - expect.stringContaining( - `Unable to upload file(s) to OpenStack Swift. Error: Failed to read template directory: ENOENT, no such file or directory`, - ), - ); - expect(error.message).toEqual( - expect.stringContaining(wrongPathToGeneratedDirectory), - ); - }); + const fails = publisher.publish({ + entity, + directory: wrongPathToGeneratedDirectory, + }); + + // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error + // Issue reported https://github.com/tschaub/mock-fs/issues/118 + await expect(fails).rejects.toMatchObject({ + message: expect.stringContaining( + `Unable to upload file(s) to OpenStack Swift. Error: Failed to read template directory: ENOENT, no such file or directory`, + ), + }); + await expect(fails).rejects.toMatchObject({ + message: expect.stringContaining(wrongPathToGeneratedDirectory), + }); + mockFs.restore(); }); }); @@ -268,18 +267,14 @@ describe('OpenStackSwiftPublish', () => { const entity = createMockEntity(); const entityRootDir = getEntityRootDir(entity); - await publisher - .fetchTechDocsMetadata(entityNameMock) - .catch(error => - expect(error).toEqual( - new Error( - `TechDocs metadata fetch failed, The file ${path.join( - entityRootDir, - 'techdocs_metadata.json', - )} does not exist !`, - ), - ), - ); + const fails = publisher.fetchTechDocsMetadata(entityNameMock); + + await expect(fails).rejects.toMatchObject({ + message: `TechDocs metadata fetch failed, The file ${path.join( + entityRootDir, + 'techdocs_metadata.json', + )} does not exist !`, + }); }); }); }); diff --git a/plugins/api-docs/CHANGELOG.md b/plugins/api-docs/CHANGELOG.md index a2d5a28e41..f268ba1c4d 100644 --- a/plugins/api-docs/CHANGELOG.md +++ b/plugins/api-docs/CHANGELOG.md @@ -1,5 +1,19 @@ # @backstage/plugin-api-docs +## 0.4.12 + +### Patch Changes + +- 1ce80ff02: Resolve issues with AsyncAPI rendering by updating `@asyncapi/react-component` + to `0.23.0`. The theming of the component is adjusted to the latest styling + changes. +- c614ede9a: Updated README to have up-to-date install instructions. +- 07a7806c3: Added fields filtering in get API entities to avoid the requesting of unused data +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.4.11 ### Patch Changes diff --git a/plugins/api-docs/package.json b/plugins/api-docs/package.json index 51df09ba80..5c9f9ca7ee 100644 --- a/plugins/api-docs/package.json +++ b/plugins/api-docs/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-api-docs", - "version": "0.4.11", + "version": "0.4.12", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -29,9 +29,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@asyncapi/react-component": "^0.22.3", + "@asyncapi/react-component": "^0.23.0", "@backstage/catalog-model": "^0.7.7", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.4", "@backstage/theme": "^0.2.6", "@material-icons/font": "^1.0.2", diff --git a/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinitionWidget.tsx b/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinitionWidget.tsx index a908a91b14..300c9d0c8a 100644 --- a/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinitionWidget.tsx +++ b/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinitionWidget.tsx @@ -88,18 +88,10 @@ const useStyles = makeStyles(theme => ({ '& .asyncapi__enum': { color: theme.palette.secondary.main, }, - '& .asyncapi__toggle-arrow:before': { - content: '">"', - 'font-family': 'inherit', - }, - '& .asyncapi__anchor-icon:before': { - content: '"🔗"', - 'font-family': 'inherit', - }, '& .asyncapi__info, .asyncapi__channel, .asyncapi__channels > div, .asyncapi__schema, .asyncapi__channel-operations-list .asyncapi__messages-list-item .asyncapi__message, .asyncapi__message, .asyncapi__server, .asyncapi__servers > div, .asyncapi__messages > div, .asyncapi__schemas > div': { 'background-color': 'inherit', }, - '& .asyncapi__channel-parameters-header, .asyncapi__channel-operations-header, .asyncapi__channel-operation-oneOf-subscribe-header, .asyncapi__channel-operation-oneOf-publish-header': { + '& .asyncapi__channel-parameters-header, .asyncapi__channel-operations-header, .asyncapi__channel-operation-oneOf-subscribe-header, .asyncapi__channel-operation-oneOf-publish-header, .asyncapi__channel-operation-message-header, .asyncapi__message-header, .asyncapi__message-header-title, .asyncapi__message-header-title > h3, .asyncapi__bindings, .asyncapi__bindings-header, .asyncapi__bindings-header > h4': { 'background-color': 'inherit', color: theme.palette.text.primary, }, diff --git a/plugins/badges/package.json b/plugins/badges/package.json index 9f0c49af7e..8f7b5865a1 100644 --- a/plugins/badges/package.json +++ b/plugins/badges/package.json @@ -21,7 +21,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.6", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/errors": "^0.1.1", "@backstage/plugin-catalog-react": "^0.1.3", "@backstage/theme": "^0.2.6", diff --git a/plugins/bitrise/CHANGELOG.md b/plugins/bitrise/CHANGELOG.md index 66314933bb..ecb26efd50 100644 --- a/plugins/bitrise/CHANGELOG.md +++ b/plugins/bitrise/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-bitrise +## 0.1.2 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.1.1 ### Patch Changes diff --git a/plugins/bitrise/package.json b/plugins/bitrise/package.json index 83911cdfc4..950392f732 100644 --- a/plugins/bitrise/package.json +++ b/plugins/bitrise/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-bitrise", - "version": "0.1.1", + "version": "0.1.2", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,7 +21,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.2", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.2", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/catalog-backend/CHANGELOG.md b/plugins/catalog-backend/CHANGELOG.md index 28efc1c5d3..ed87e4dbc5 100644 --- a/plugins/catalog-backend/CHANGELOG.md +++ b/plugins/catalog-backend/CHANGELOG.md @@ -1,5 +1,13 @@ # @backstage/plugin-catalog-backend +## 0.8.1 + +### Patch Changes + +- a99e0bc42: Entity lifecycle and owner are now indexed by the `DefaultCatalogCollator`. A `locationTemplate` may now optionally be provided to its constructor to reflect a custom catalog entity path in the Backstage frontend. +- Updated dependencies [e1e757569] + - @backstage/plugin-search-backend-node@0.1.4 + ## 0.8.0 ### Minor Changes diff --git a/plugins/catalog-backend/migrationsv2/20200511113813_init.js b/plugins/catalog-backend/migrationsv2/20200511113813_init.js new file mode 100644 index 0000000000..7f3d75e35c --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200511113813_init.js @@ -0,0 +1,133 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + return ( + knex.schema + // + // locations + // + .createTable('locations', table => { + table.comment( + 'Registered locations that shall be contiuously scanned for catalog item updates', + ); + table + .uuid('id') + .primary() + .notNullable() + .comment('Auto-generated ID of the location'); + table.string('type').notNullable().comment('The type of location'); + table + .string('target') + .notNullable() + .comment('The actual target of the location'); + }) + // + // entities + // + .createTable('entities', table => { + table.comment('All entities currently stored in the catalog'); + table.uuid('id').primary().comment('Auto-generated ID of the entity'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .nullable() + .comment('The location that originated the entity'); + table + .string('etag') + .notNullable() + .comment( + 'An opaque string that changes for each update operation to any part of the entity, including metadata.', + ); + table + .string('generation') + .notNullable() + .unsigned() + .comment( + 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', + ); + table + .string('api_version') + .notNullable() + .comment('The apiVersion field of the entity'); + table + .string('kind') + .notNullable() + .comment('The kind field of the entity'); + table + .string('name') + .nullable() + .comment('The metadata.name field of the entity'); + table + .string('namespace') + .nullable() + .comment('The metadata.namespace field of the entity'); + table + .string('metadata') + .notNullable() + .comment('The entire metadata JSON blob of the entity'); + table + .string('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + }) + .alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); + }) + // + // entities_search + // + .createTable('entities_search', table => { + table.comment( + 'Flattened key-values from the entities, used for quick filtering', + ); + table + .uuid('entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .comment('The entity that matches this key/value'); + table + .string('key') + .notNullable() + .comment('A key that occurs in the entity'); + table + .string('value') + .nullable() + .comment('The corresponding value to match on'); + }) + ); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + return knex.schema + .dropTable('entities_search') + .alterTable('entities', table => { + table.dropUnique([], 'entities_unique_name'); + }) + .dropTable('entities') + .dropTable('locations'); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js b/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js new file mode 100644 index 0000000000..d8093fc9b4 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + return knex.schema.createTable('location_update_log', table => { + table.uuid('id').primary(); + table.enum('status', ['success', 'fail']).notNullable(); + table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table.string('message'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .onUpdate('CASCADE') + .onDelete('CASCADE'); + table.string('entity_name').nullable(); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + return knex.schema.dropTableIfExists('location_update_log'); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js b/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js new file mode 100644 index 0000000000..a0f0f33a65 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // Get list sorted by created_at timestamp in descending order + // Grouped by location_id + return knex.schema.raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(created_at) AS MAXDATE + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.created_at = t2.MAXDATE + ORDER BY created_at DESC; + `); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + return knex.schema.raw(`DROP VIEW location_update_log_latest;`); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200702153613_entities.js b/plugins/catalog-backend/migrationsv2/20200702153613_entities.js new file mode 100644 index 0000000000..0f1c204f9b --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200702153613_entities.js @@ -0,0 +1,236 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // SQLite does not support FK and PK + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_search', table => { + table.dropForeign(['entity_id']); + }); + await knex.schema.alterTable('entities', table => { + table.dropPrimary('entities_pkey'); + }); + } + await knex.schema.alterTable('entities', table => { + table.dropUnique([], 'entities_unique_name'); + }); + // Setup temporary tables + await knex.schema.renameTable('entities_search', 'tmp_entities_search'); + await knex.schema.renameTable('entities', 'tmp_entities'); + + // + // entities + // + await knex.schema + .createTable('entities', table => { + table.comment('All entities currently stored in the catalog'); + table.uuid('id').primary().comment('Auto-generated ID of the entity'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .nullable() + .comment('The location that originated the entity'); + table + .string('etag') + .notNullable() + .comment( + 'An opaque string that changes for each update operation to any part of the entity, including metadata.', + ); + table + .string('generation') + .notNullable() + .unsigned() + .comment( + 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', + ); + table + .string('api_version') + .notNullable() + .comment('The apiVersion field of the entity'); + table + .string('kind') + .notNullable() + .comment('The kind field of the entity'); + table + .string('name') + .nullable() + .comment('The metadata.name field of the entity'); + table + .string('namespace') + .nullable() + .comment('The metadata.namespace field of the entity'); + table + .text('metadata') + .notNullable() + .comment('The entire metadata JSON blob of the entity'); + table + .text('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + }) + .alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); + }); + + await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`); + + // + // entities_search + // + await knex.schema.createTable('entities_search', table => { + table.comment( + 'Flattened key-values from the entities, used for quick filtering', + ); + table + .uuid('entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .comment('The entity that matches this key/value'); + table + .string('key') + .notNullable() + .comment('A key that occurs in the entity'); + table + .string('value') + .nullable() + .comment('The corresponding value to match on'); + }); + await knex.schema.raw( + `INSERT INTO entities_search SELECT * FROM tmp_entities_search`, + ); + + // Clean up + await knex.schema.dropTable('tmp_entities'); + return knex.schema.dropTable('tmp_entities_search'); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + // SQLite does not support FK and PK + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_search', table => { + table.dropForeign(['entity_id']); + }); + await knex.schema.alterTable('entities', table => { + table.dropPrimary('entities_pkey'); + }); + } + await knex.schema.alterTable('entities', table => { + table.dropUnique([], 'entities_unique_name'); + }); + + // Setup temporary tables + await knex.schema.renameTable('entities_search', 'tmp_entities_search'); + await knex.schema.renameTable('entities', 'tmp_entities'); + + // + // entities + // + await knex.schema + .createTable('entities', table => { + table.comment('All entities currently stored in the catalog'); + table.uuid('id').primary().comment('Auto-generated ID of the entity'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .nullable() + .comment('The location that originated the entity'); + table + .string('etag') + .notNullable() + .comment( + 'An opaque string that changes for each update operation to any part of the entity, including metadata.', + ); + table + .string('generation') + .notNullable() + .unsigned() + .comment( + 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', + ); + table + .string('api_version') + .notNullable() + .comment('The apiVersion field of the entity'); + table + .string('kind') + .notNullable() + .comment('The kind field of the entity'); + table + .string('name') + .nullable() + .comment('The metadata.name field of the entity'); + table + .string('namespace') + .nullable() + .comment('The metadata.namespace field of the entity'); + table + .string('metadata') + .notNullable() + .comment('The entire metadata JSON blob of the entity'); + table + .string('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + }) + .alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); + }); + + await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`); + + // + // entities_search + // + await knex.schema.createTable('entities_search', table => { + table.comment( + 'Flattened key-values from the entities, used for quick filtering', + ); + table + .uuid('entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .comment('The entity that matches this key/value'); + table + .string('key') + .notNullable() + .comment('A key that occurs in the entity'); + table + .string('value') + .nullable() + .comment('The corresponding value to match on'); + }); + await knex.schema.raw( + `INSERT INTO entities_search SELECT * FROM tmp_entities_search`, + ); + + // Clean up + await knex.schema.dropTable('tmp_entities'); + return knex.schema.dropTable('tmp_entities_search'); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js b/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js new file mode 100644 index 0000000000..87b41a80fc --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = function up(knex) { + return knex.schema.raw(`DROP VIEW location_update_log_latest;`).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(created_at) AS MAXDATE + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.created_at = t2.MAXDATE + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; +`); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = function down(knex) { + knex.schema.raw(`DROP VIEW location_update_log_latest;`); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js b/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js new file mode 100644 index 0000000000..de2b194cff --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = function up(knex) { + return knex.schema + .raw('DROP VIEW location_update_log_latest;') + .dropTable('location_update_log') + .createTable('location_update_log', table => { + table.bigIncrements('id').primary(); // instead of uuid, so we can MAX it + table.enum('status', ['success', 'fail']).notNullable(); + table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table.string('message'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .onUpdate('CASCADE') + .onDelete('CASCADE'); + table.string('entity_name').nullable(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(id) AS MAXID + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.id = t2.MAXID + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = function down(knex) { + return knex.schema + .raw('DROP VIEW location_update_log_latest;') + .dropTable('location_update_log') + .createTable('location_update_log', table => { + table.uuid('id').primary(); + table.enum('status', ['success', 'fail']).notNullable(); + table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table.string('message'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .onUpdate('CASCADE') + .onDelete('CASCADE'); + table.string('entity_name').nullable(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(created_at) AS MAXDATE + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.created_at = t2.MAXDATE + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js b/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js new file mode 100644 index 0000000000..45226e53b4 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // Sqlite does not support alter column. + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_search', table => { + table.text('value').nullable().alter(); + }); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + // Sqlite does not support alter column. + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_search', table => { + table.string('value').nullable().alter(); + }); + } +}; diff --git a/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js b/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js new file mode 100644 index 0000000000..a90813fe85 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // Adds a single 'bootstrap' location that can be used to trigger work in processors. + // This is primarily here to fulfill foreign key constraints. + await knex('locations').insert({ + id: require('uuid').v4(), + type: 'bootstrap', + target: 'bootstrap', + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex('locations') + .where({ + type: 'bootstrap', + target: 'bootstrap', + }) + .del(); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js b/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js new file mode 100644 index 0000000000..ea5ba9e58d --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex('entities') + .where({ namespace: null }) + .update({ namespace: 'default' }); + await knex('entities_search').update({ + key: knex.raw('LOWER(key)'), + value: knex.raw('LOWER(value)'), + }); +}; + +exports.down = async function down() {}; diff --git a/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js b/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js new file mode 100644 index 0000000000..aae1861658 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('entities', table => { + table.text('full_name').nullable(); + }); + + await knex('entities').update({ + full_name: knex.raw( + "LOWER(kind) || ':' || LOWER(COALESCE(namespace, 'default')) || '/' || LOWER(name)", + ), + }); + + // SQLite does not support alter column + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities', table => { + table.text('full_name').notNullable().alter(); + }); + } + + await knex.schema.alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.unique(['full_name'], 'entities_unique_full_name'); + table.dropUnique([], 'entities_unique_name'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.dropUnique([], 'entities_unique_full_name'); + table.unique(['kind', 'namespace', 'name'], 'entities_unique_name'); + }); + + await knex.schema.alterTable('entities_search', table => { + table.dropColumn('full_name'); + }); +}; diff --git a/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js b/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js new file mode 100644 index 0000000000..a8964efbf6 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('entities', table => { + table + .text('data') + .nullable() + .comment('The entire JSON data blob of the entity'); + }); + + await knex('entities').update({ + // apiVersion and kind should not contain any JSON unsafe chars, and both + // metadata and spec are already valid serialized JSON + data: knex.raw( + `'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`, + ), + }); + + await knex.schema.alterTable('entities', table => { + table.dropColumn('metadata'); + table.dropColumn('spec'); + }); + + // SQLite does not support ALTER COLUMN. + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities', table => { + table.text('data').notNullable().alter(); + }); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('entities', table => { + table + .text('metadata') + .notNullable() + .comment('The entire metadata JSON blob of the entity'); + table + .text('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + table.dropColumn('data'); + }); +}; diff --git a/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js b/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js new file mode 100644 index 0000000000..f40df5f73e --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('entities', table => { + table.dropColumn('api_version'); + table.dropColumn('kind'); + table.dropColumn('name'); + table.dropColumn('namespace'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('entities', table => { + table + .string('api_version') + .notNullable() + .comment('The apiVersion field of the entity'); + table.string('kind').notNullable().comment('The kind field of the entity'); + table + .string('name') + .nullable() + .comment('The metadata.name field of the entity'); + table + .string('namespace') + .nullable() + .comment('The metadata.namespace field of the entity'); + }); +}; diff --git a/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js b/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js new file mode 100644 index 0000000000..77bf0529eb --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('entities_search', table => { + table.index(['key'], 'entities_search_key'); + table.index(['value'], 'entities_search_value'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('entities_search', table => { + table.dropIndex('', 'entities_search_key'); + table.dropIndex('', 'entities_search_value'); + }); +}; diff --git a/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js b/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js new file mode 100644 index 0000000000..85e729f814 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('entities_relations', table => { + table.comment('All relations between entities in the catalog'); + table + .uuid('originating_entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .notNullable() + .comment('The entity that provided the relation'); + table + .string('source_full_name') + .notNullable() + .comment('The full name of the source entity of the relation'); + table + .string('type') + .notNullable() + .comment('The type of the relation between the entities'); + table + .string('target_full_name') + .notNullable() + .comment('The full name of the target entity of the relation'); + + table.primary(['source_full_name', 'type', 'target_full_name']); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.dropTable('entities_relations'); +}; diff --git a/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js b/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js new file mode 100644 index 0000000000..9e8198b5eb --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + if (knex.client.config.client === 'sqlite3') { + // sqlite doesn't support dropPrimary so we recreate it properly instead + await knex.schema.dropTable('entities_relations'); + await knex.schema.createTable('entities_relations', table => { + table.comment('All relations between entities in the catalog'); + table + .uuid('originating_entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .notNullable() + .comment('The entity that provided the relation'); + table + .string('source_full_name') + .notNullable() + .comment('The full name of the source entity of the relation'); + table + .string('type') + .notNullable() + .comment('The type of the relation between the entities'); + table + .string('target_full_name') + .notNullable() + .comment('The full name of the target entity of the relation'); + table.index('source_full_name', 'source_full_name_idx'); + }); + } else { + await knex.schema.alterTable('entities_relations', table => { + table.dropPrimary(); + table.index('source_full_name', 'source_full_name_idx'); + }); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client === 'sqlite3') { + await knex.schema.dropTable('entities_relations'); + await knex.schema.createTable('entities_relations', table => { + table.comment('All relations between entities in the catalog'); + table + .uuid('originating_entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .notNullable() + .comment('The entity that provided the relation'); + table + .string('source_full_name') + .notNullable() + .comment('The full name of the source entity of the relation'); + table + .string('type') + .notNullable() + .comment('The type of the relation between the entities'); + table + .string('target_full_name') + .notNullable() + .comment('The full name of the target entity of the relation'); + + table.primary(['source_full_name', 'type', 'target_full_name']); + }); + } else { + await knex.schema.alterTable('entities_relations', table => { + table.dropIndex([], 'source_full_name_idx'); + table.primary(['source_full_name', 'type', 'target_full_name']); + }); + } +}; diff --git a/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js b/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js new file mode 100644 index 0000000000..abb26cd5fc --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_relations', table => { + table.index('originating_entity_id', 'originating_entity_id_idx'); + }); + await knex.schema.alterTable('entities_search', table => { + table.index('entity_id', 'entity_id_idx'); + }); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_relations', table => { + table.dropIndex([], 'originating_entity_id_idx'); + }); + await knex.schema.alterTable('entities_relations', table => { + table.dropIndex([], 'entity_id_idx'); + }); + } +}; diff --git a/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js b/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js new file mode 100644 index 0000000000..d924b0414a --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + if (knex.client.config.client !== 'sqlite3') { + // We actually just want to widen columns, but can't do that while a + // view is dependent on them - so we just reconstruct it exactly as it was + await knex.schema + .raw('DROP VIEW location_update_log_latest;') + .alterTable('location_update_log', table => { + table.text('message').alter(); + table.text('entity_name').nullable().alter(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(id) AS MAXID + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.id = t2.MAXID + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema + .raw('DROP VIEW location_update_log_latest;') + .alterTable('location_update_log', table => { + table.string('message').alter(); + table.string('entity_name').nullable().alter(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(id) AS MAXID + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.id = t2.MAXID + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); + } +}; diff --git a/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js b/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js new file mode 100644 index 0000000000..ccfb1faffb --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities', table => { + table.index('location_id', 'entity_location_id_idx'); + }); + await knex.schema.alterTable('location_update_log', table => { + table.index('location_id', 'update_log_location_id_idx'); + }); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities', table => { + table.dropIndex([], 'entity_location_id_idx'); + }); + await knex.schema.alterTable('location_update_log', table => { + table.dropIndex([], 'update_log_location_id_idx'); + }); + } +}; diff --git a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js index 6abab6270b..4c03405910 100644 --- a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js @@ -33,7 +33,6 @@ exports.up = async function up(knex) { ); table .text('entity_ref') - .unique() .notNullable() .comment('A reference to the entity that the refresh state is tied to'); table @@ -59,13 +58,15 @@ exports.up = async function up(knex) { .notNullable() .comment('JSON array containing all errors related to entity'); table - .dateTime('next_update_at') + .dateTime('next_update_at') // TODO: timezone or change to epoch-millis or similar .notNullable() .comment('Timestamp of when entity should be updated'); table - .dateTime('last_discovery_at') + .dateTime('last_discovery_at') // TODO: timezone or change to epoch-millis or similar .notNullable() .comment('The last timestamp of which this entity was discovered'); + table.unique(['entity_ref'], 'refresh_state_entity_ref_uniq'); + table.index('entity_id', 'refresh_state_entity_id_idx'); table.index('entity_ref', 'refresh_state_entity_ref_idx'); table.index('next_update_at', 'refresh_state_next_update_at_idx'); }); @@ -82,10 +83,18 @@ exports.up = async function up(knex) { .inTable('refresh_state') .onDelete('CASCADE') .comment( - 'Entity ID which correspond to the ID in the refresh_state table', + 'Entity ID which corresponds to the ID in the refresh_state table', ); - table.text('etag').notNullable().comment('Etag to be used for caching'); - table.text('finalized_entity').notNullable().comment('The final entity'); + table + .text('hash') + .notNullable() + .comment( + 'Stable hash of the entity data, to be used for caching and avoiding redundant work', + ); + table + .text('final_entity') + .notNullable() + .comment('The JSON encoded final entity'); table.index('entity_id', 'final_entities_entity_id_idx'); }); @@ -94,38 +103,38 @@ exports.up = async function up(knex) { 'Holds edges between refresh state rows. Every time when an entity is processed and emits another entity, an edge will be stored to represent that fact. This is used to detect orphans and ultimately deletions.', ); table - .text('source_special_key') + .increments('id') + .comment('Primary key to distinguish unique lines from each other'); + table + .text('source_key') .nullable() .comment( 'When the reference source is not an entity, this is an opaque identifier for that source.', ); table - .text('source_entity_id') + .text('source_entity_ref') .nullable() - .references('entity_id') + .references('entity_ref') .inTable('refresh_state') .onDelete('CASCADE') .comment( - 'When the reference source is an entity, this is the ID of the source entity.', + 'When the reference source is an entity, this is the EntityRef of the source entity.', ); table - .text('target_entity_id') + .text('target_entity_ref') .notNullable() - .references('entity_id') + .references('entity_ref') .inTable('refresh_state') .onDelete('CASCADE') - .comment('The ID of the target entity.'); + .comment('The EntityRef of the target entity.'); + table.index('source_key', 'refresh_state_references_source_key_idx'); table.index( - 'source_special_key', - 'refresh_state_references_source_special_key_idx', + 'source_entity_ref', + 'refresh_state_references_source_entity_ref_idx', ); table.index( - 'source_entity_id', - 'refresh_state_references_source_entity_id_idx', - ); - table.index( - 'target_entity_id', - 'refresh_state_references_target_entity_id_idx', + 'target_entity_ref', + 'refresh_state_references_target_entity_ref_idx', ); }); @@ -153,6 +162,29 @@ exports.up = async function up(knex) { table.index('source_entity_ref', 'relations_source_entity_ref_idx'); table.index('originating_entity_id', 'relations_source_entity_id_idx'); }); + + await knex.schema.createTable('search', table => { + table.comment( + 'Flattened key-values from the entities, used for quick filtering', + ); + table + .text('entity_id') + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE') + .comment('The entity that matches this key/value'); + table + .string('key') + .notNullable() + .comment('A key that occurs in the entity'); + table + .string('value') + .nullable() + .comment('The corresponding value to match on'); + table.index(['entity_id'], 'search_entity_id_idx'); + table.index(['key'], 'search_key_idx'); + table.index(['value'], 'search_value_idx'); + }); }; /** @@ -165,6 +197,8 @@ exports.down = async function down(knex) { table.dropIndex([], 'refresh_state_references_target_entity_id_idx'); }); await knex.schema.alterTable('refresh_state', table => { + table.dropUnique([], 'refresh_state_entity_ref_uniq'); + table.dropIndex([], 'refresh_state_entity_id_idx'); table.dropIndex([], 'refresh_state_entity_ref_idx'); table.dropIndex([], 'refresh_state_next_update_at_idx'); }); @@ -175,6 +209,13 @@ exports.down = async function down(knex) { table.index('source_entity_ref', 'relations_source_entity_ref_idx'); table.index('originating_entity_id', 'relations_source_entity_id_idx'); }); + await knex.schema.alterTable('search', table => { + table.dropIndex([], 'search_entity_id_idx'); + table.dropIndex([], 'search_key_idx'); + table.dropIndex([], 'search_value_idx'); + }); + + await knex.schema.dropTable('search'); await knex.schema.dropTable('final_entities'); await knex.schema.dropTable('relations'); await knex.schema.dropTable('references'); diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 9afb537637..7649067a44 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-backend", - "version": "0.8.0", + "version": "0.8.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -33,10 +33,9 @@ "@backstage/backend-common": "^0.6.3", "@backstage/catalog-model": "^0.7.7", "@backstage/config": "^0.1.4", - "@backstage/core": "^0.7.6", "@backstage/errors": "^0.1.1", "@backstage/integration": "^0.5.1", - "@backstage/plugin-search-backend-node": "^0.1.3", + "@backstage/plugin-search-backend-node": "^0.1.4", "@backstage/search-common": "^0.1.1", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", @@ -61,8 +60,7 @@ "winston": "^3.2.1", "yaml": "^1.9.2", "yn": "^4.0.0", - "yup": "^0.29.3", - "zen-observable": "^0.8.15" + "yup": "^0.29.3" }, "devDependencies": { "@backstage/cli": "^0.6.9", @@ -80,7 +78,6 @@ "files": [ "dist", "migrations/**/*.{js,d.ts}", - "migrationsv2/**/*.{js,d.ts}", "config.d.ts" ], "configSchema": "config.d.ts" diff --git a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.test.ts b/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.test.ts new file mode 100644 index 0000000000..226e5c06a8 --- /dev/null +++ b/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolvePackagePath } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import path from 'path'; +import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider'; +import { EntityProviderConnection } from './types'; + +describe('ConfigLocationEntityProvider', () => { + it('should apply mutation with the correct paths in the config', async () => { + const mockConfig = new ConfigReader({ + catalog: { + locations: [ + { type: 'file', target: './lols.yaml' }, + { type: 'url', target: 'https://github.com/backstage/backstage' }, + ], + }, + }); + + const mockConnection = ({ + applyMutation: jest.fn(), + } as unknown) as EntityProviderConnection; + const locationProvider = new ConfigLocationEntityProvider(mockConfig); + + await locationProvider.connect(mockConnection); + + expect(mockConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expect.arrayContaining([ + expect.objectContaining({ + spec: { + target: path.join( + resolvePackagePath('@backstage/plugin-catalog-backend'), + './lols.yaml', + ), + type: 'file', + }, + }), + ]), + }); + expect(mockConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expect.arrayContaining([ + expect.objectContaining({ + spec: { + target: 'https://github.com/backstage/backstage', + type: 'url', + }, + }), + ]), + }); + }); +}); diff --git a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts b/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts new file mode 100644 index 0000000000..bccbd0ba41 --- /dev/null +++ b/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts @@ -0,0 +1,51 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import path from 'path'; +import { EntityProvider, EntityProviderConnection } from './types'; +import { locationSpecToLocationEntity } from './util'; + +export class ConfigLocationEntityProvider implements EntityProvider { + private connection: EntityProviderConnection | undefined; + + constructor(private readonly config: Config) {} + + getProviderName(): string { + return 'ConfigLocationProvider'; + } + + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + + const locationConfigs = + this.config.getOptionalConfigArray('catalog.locations') ?? []; + + const entities = locationConfigs.map(location => { + const type = location.getString('type'); + const target = location.getString('target'); + return locationSpecToLocationEntity({ + type, + target: type === 'file' ? path.resolve(target) : target, + }); + }); + + await this.connection.applyMutation({ + type: 'full', + entities, + }); + } +} diff --git a/plugins/catalog-backend/src/next/Context/BackgroundContext.ts b/plugins/catalog-backend/src/next/Context/BackgroundContext.ts new file mode 100644 index 0000000000..72b9a3b1ed --- /dev/null +++ b/plugins/catalog-backend/src/next/Context/BackgroundContext.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Context, ContextKey } from './types'; + +/** + * A base Context implementation that does not hold any value. + */ +export class BackgroundContext implements Context { + getContextValue(key: ContextKey): T { + return key.defaultValue; + } +} diff --git a/plugins/catalog-backend/src/next/Context/ContextWithValue.ts b/plugins/catalog-backend/src/next/Context/ContextWithValue.ts new file mode 100644 index 0000000000..e8f94fd922 --- /dev/null +++ b/plugins/catalog-backend/src/next/Context/ContextWithValue.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Context, ContextKey } from './types'; + +/** + * A Context implementation that holds a single value, optionally extending an existing context. + */ +export class ContextWithValue implements Context { + static create(parent: Context, key: ContextKey, value: unknown) { + return new ContextWithValue(parent, key, value); + } + + private constructor( + private readonly parent: Context, + private readonly key: ContextKey, + private readonly value: unknown, + ) {} + + getContextValue(key: ContextKey): T { + if (this.key === key) { + return this.value as T; + } + return this.parent.getContextValue(key); + } +} diff --git a/plugins/catalog-backend/src/next/Context/TransactionValue.ts b/plugins/catalog-backend/src/next/Context/TransactionValue.ts new file mode 100644 index 0000000000..6959d13a18 --- /dev/null +++ b/plugins/catalog-backend/src/next/Context/TransactionValue.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Context, ContextKey } from './types'; +import { Knex } from 'knex'; +import { ContextWithValue } from './ContextWithValue'; + +const transactionContextKey = new ContextKey( + undefined, +); + +/** + * TransactionValue handles the wrapping of a knex transaction in a Context. + */ +export class TransactionValue { + static in(parent: Context, tx: Knex.Transaction) { + return ContextWithValue.create(parent, transactionContextKey, tx); + } + + static from(context: Context): Knex.Transaction { + const transaction = context.getContextValue(transactionContextKey); + if (!transaction) { + throw new Error(`No transaction available in context`); + } + return transaction; + } +} diff --git a/plugins/catalog-backend/src/next/Context/index.ts b/plugins/catalog-backend/src/next/Context/index.ts new file mode 100644 index 0000000000..61dd4a2958 --- /dev/null +++ b/plugins/catalog-backend/src/next/Context/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { BackgroundContext } from './BackgroundContext'; +export { ContextWithValue } from './ContextWithValue'; +export { TransactionValue } from './TransactionValue'; +export { ContextKey } from './types'; +export type { Context } from './types'; diff --git a/plugins/catalog-backend/src/next/Context/types.ts b/plugins/catalog-backend/src/next/Context/types.ts new file mode 100644 index 0000000000..0973b83515 --- /dev/null +++ b/plugins/catalog-backend/src/next/Context/types.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class ContextKey { + constructor(readonly defaultValue: T) {} +} + +export interface Context { + getContextValue(key: ContextKey): T; +} diff --git a/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts b/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts deleted file mode 100644 index eaf5a78556..0000000000 --- a/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Observable } from '@backstage/core'; -import { ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; -import { EntityProvider, LocationStore, EntityMessage } from './types'; -import ObservableImpl from 'zen-observable'; -import { - locationSpecToLocationEntity, - locationSpecToMetadataName, -} from './util'; - -export class DatabaseLocationProvider implements EntityProvider { - private subscribers = new Set< - ZenObservable.SubscriptionObserver - >(); - - constructor(private readonly store: LocationStore) { - store.location$().subscribe({ - next: locations => { - if ('all' in locations) { - this.notify({ - all: locations.all.map(l => locationSpecToLocationEntity(l)), - }); - } else { - this.notify({ - added: locations.added.map(l => locationSpecToLocationEntity(l)), - removed: locations.removed.map(l => ({ - kind: 'Location', - namespace: ENTITY_DEFAULT_NAMESPACE, - name: locationSpecToMetadataName(l), - })), - }); - } - }, - }); - } - - private notify(message: EntityMessage) { - for (const subscriber of this.subscribers) { - subscriber.next(message); - } - } - - entityChange$(): Observable { - return new ObservableImpl(subscriber => { - this.store.listLocations().then(locations => { - subscriber.next({ - all: locations.map(l => locationSpecToLocationEntity(l)), - }); - this.subscribers.add(subscriber); - }); - return () => { - this.subscribers.delete(subscriber); - }; - }); - } -} diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index 934ab3a1cc..e724fe79d8 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -14,21 +14,47 @@ * limitations under the License. */ -import { Subscription } from '@backstage/core'; +import { stringifyEntityRef } from '@backstage/catalog-model'; +import { Logger } from 'winston'; +import { Stitcher } from './Stitcher'; import { CatalogProcessingEngine, - EntityProvider, - EntityMessage, - ProcessingStateManager, CatalogProcessingOrchestrator, + EntityProvider, + EntityProviderConnection, + EntityProviderMutation, + ProcessingStateManager, } from './types'; -import { Logger } from 'winston'; -import { stringifyEntityRef } from '@backstage/catalog-model'; -import { Stitcher } from './Stitcher'; +class Connection implements EntityProviderConnection { + constructor( + private readonly config: { + stateManager: ProcessingStateManager; + id: string; + }, + ) {} + + async applyMutation(mutation: EntityProviderMutation): Promise { + if (mutation.type === 'full') { + await this.config.stateManager.replaceProcessingItems({ + sourceKey: this.config.id, + type: 'full', + items: mutation.entities, + }); + + return; + } + + await this.config.stateManager.replaceProcessingItems({ + sourceKey: this.config.id, + type: 'delta', + added: mutation.added, + removed: mutation.removed, + }); + } +} export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { - private subscriptions: Subscription[] = []; private running: boolean = false; constructor( @@ -41,11 +67,12 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { async start() { for (const provider of this.entityProviders) { - const id = 'databaseProvider'; - const subscription = provider - .entityChange$() - .subscribe({ next: m => this.onNext(id, m) }); - this.subscriptions.push(subscription); + await provider.connect( + new Connection({ + stateManager: this.stateManager, + id: provider.getProviderName(), + }), + ); } this.running = true; @@ -54,12 +81,12 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { const { id, entity, - state: intialState, + state: initialState, } = await this.stateManager.getNextProcessingItem(); const result = await this.orchestrator.process({ entity, - state: intialState, + state: initialState, }); for (const error of result.errors) { @@ -77,7 +104,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { state: result.state, errors: result.errors, relations: result.relations, - deferredEntities: result.deferredEntites, + deferredEntities: result.deferredEntities, }); const setOfThingsToStitch = new Set([ @@ -92,30 +119,5 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { async stop() { this.running = false; - - for (const subscription of this.subscriptions) { - subscription.unsubscribe(); - } - } - - private async onNext(id: string, message: EntityMessage) { - if ('all' in message) { - // TODO unhandled rejection - await this.stateManager.addProcessingItems({ - id, - type: 'provider', - entities: message.all, - }); - } - - if ('added' in message) { - await this.stateManager.addProcessingItems({ - id, - type: 'provider', - entities: message.added, - }); - - // TODO deletions of message.removed - } } } diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingOrchestrator.ts index 1880b5d17e..8fbb19e536 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingOrchestrator.ts @@ -179,7 +179,7 @@ export class DefaultCatalogProcessingOrchestrator ); } - // Backwards compatible processing of location entites + // Backwards compatible processing of location entities if (isLocationEntity(entity)) { const { type = location.type } = entity.spec; const targets = new Array(); @@ -268,7 +268,7 @@ function createEmitter(logger: Logger, parentEntity: Entity) { const errors = new Array(); const relations = new Array(); - const deferredEntites = new Array(); + const deferredEntities = new Array(); const emit = (i: CatalogProcessorResult) => { if (done) { @@ -282,7 +282,7 @@ function createEmitter(logger: Logger, parentEntity: Entity) { if (i.type === 'entity') { const originLocation = getEntityOriginLocationRef(parentEntity); - deferredEntites.push({ + deferredEntities.push({ ...i.entity, metadata: { ...i.entity.metadata, @@ -294,7 +294,7 @@ function createEmitter(logger: Logger, parentEntity: Entity) { }, }); } else if (i.type === 'location') { - deferredEntites.push( + deferredEntities.push( locationSpecToLocationEntity(i.location, parentEntity), ); } else if (i.type === 'relation') { @@ -311,7 +311,7 @@ function createEmitter(logger: Logger, parentEntity: Entity) { return { errors, relations, - deferredEntites, + deferredEntities, }; }, }; diff --git a/plugins/catalog-backend/src/next/DefaultService.ts b/plugins/catalog-backend/src/next/DefaultLocationService.ts similarity index 100% rename from plugins/catalog-backend/src/next/DefaultService.ts rename to plugins/catalog-backend/src/next/DefaultLocationService.ts diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts new file mode 100644 index 0000000000..f9bb97477c --- /dev/null +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts @@ -0,0 +1,147 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { v4 as uuid } from 'uuid'; +import { DatabaseManager } from './database/DatabaseManager'; +import { DefaultLocationStore } from './DefaultLocationStore'; + +/* eslint-disable */ +xdescribe('DefaultLocationStore', () => { + const createLocationStore = async () => { + const db = await DatabaseManager.createTestDatabase(); + const connection = { applyMutation: jest.fn() }; + const store = new DefaultLocationStore(db); + await store.connect(connection); + return { store, connection }; + }; + + it('should do a full sync with the locations on connect', async () => { + const { connection } = await createLocationStore(); + + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [], + }); + }); + + describe('listLocations', () => { + it('lists empty locations when there is no locations', async () => { + const { store } = await createLocationStore(); + + expect(await store.listLocations()).toEqual([]); + }); + + it('lists locations that are added to the db', async () => { + const { store } = await createLocationStore(); + + await store.createLocation({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }); + + const listLocations = await store.listLocations(); + + expect(listLocations).toHaveLength(1); + expect(listLocations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }), + ]), + ); + }); + }); + + describe('createLocation', () => { + it('throws when the location already exists', async () => { + const { store } = await createLocationStore(); + const spec = { + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }; + await store.createLocation(spec); + + await expect(() => store.createLocation(spec)).rejects.toThrow( + new RegExp(`Location ${spec.type}:${spec.target} already exists`), + ); + }); + + it('calls apply mutation when adding a new location', async () => { + const { store, connection } = await createLocationStore(); + + await store.createLocation({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }); + + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [], + added: expect.arrayContaining([ + expect.objectContaining({ + spec: { + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }, + }), + ]), + }); + }); + }); + + describe('deleteLocation', () => { + it('throws if the location does not exist', async () => { + const { store } = await createLocationStore(); + + const id = uuid(); + + await expect(() => store.deleteLocation(id)).rejects.toThrow( + new RegExp(`Found no location with ID ${id}`), + ); + }); + + it('calls apply mutation when adding a new location', async () => { + const { store, connection } = await createLocationStore(); + + const location = await store.createLocation({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }); + + await store.deleteLocation(location.id); + + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: expect.arrayContaining([ + expect.objectContaining({ + spec: { + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }, + }), + ]), + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.ts index b21fe9da02..f6267df6a5 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.ts @@ -16,27 +16,26 @@ import { LocationSpec, Location } from '@backstage/catalog-model'; import { Database } from '../database'; -import { LocationStore } from './types'; -import { v4 as uuidv4 } from 'uuid'; +import { + LocationStore, + EntityProvider, + EntityProviderConnection, +} from './types'; +import { v4 as uuid } from 'uuid'; +import { locationSpecToLocationEntity } from './util'; import { ConflictError } from '@backstage/errors'; -import { Observable } from '@backstage/core'; -import ObservableImpl from 'zen-observable'; -export type LocationMessage = - | { all: Location[] } - | { added: Location[]; removed: Location[] }; - -export class DefaultLocationStore implements LocationStore { - private subscribers = new Set< - ZenObservable.SubscriptionObserver - >(); +export class DefaultLocationStore implements LocationStore, EntityProvider { + private _connection: EntityProviderConnection | undefined; constructor(private readonly db: Database) {} - createLocation(spec: LocationSpec): Promise { - return this.db.transaction(async tx => { - // TODO: id should really be type and target combined and not a uuid. + getProviderName(): string { + return 'DefaultLocationStore'; + } + async createLocation(spec: LocationSpec): Promise { + return this.db.transaction(async tx => { // Attempt to find a previous location matching the spec const previousLocations = await this.listLocations(); const previousLocation = previousLocations.some( @@ -49,13 +48,18 @@ export class DefaultLocationStore implements LocationStore { ); } + // TODO: id should really be type and target combined and not a uuid. const location = await this.db.addLocation(tx, { - id: uuidv4(), + id: uuid(), type: spec.type, target: spec.target, }); - this.notifyAddition(location); + await this.connection.applyMutation({ + type: 'delta', + added: [locationSpecToLocationEntity(location)], + removed: [], + }); return location; }); @@ -63,11 +67,17 @@ export class DefaultLocationStore implements LocationStore { async listLocations(): Promise { const dbLocations = await this.db.locations(); - return dbLocations.map(item => ({ - id: item.id, - target: item.target, - type: item.type, - })); + return ( + dbLocations + // TODO(blam): We should create a mutation to remove this location for everyone + // eventually when it's all done and dusted + .filter(({ type }) => type !== 'bootstrap') + .map(item => ({ + id: item.id, + target: item.target, + type: item.type, + })) + ); } getLocation(id: string): Promise { @@ -75,40 +85,38 @@ export class DefaultLocationStore implements LocationStore { } deleteLocation(id: string): Promise { + if (!this.connection) { + throw new Error('location store is not initialized'); + } + return this.db.transaction(async tx => { const location = await this.db.location(id); - if (!location) { - throw new ConflictError(`No location found with id: ${id}`); - } await this.db.removeLocation(tx, id); - this.notifyDeletion(location); + await this.connection.applyMutation({ + type: 'delta', + added: [], + removed: [locationSpecToLocationEntity(location)], + }); }); } - private notifyAddition(location: Location) { - for (const subscriber of this.subscribers) { - subscriber.next({ - added: [location], - removed: [], - }); + private get connection(): EntityProviderConnection { + if (!this._connection) { + throw new Error('location store is not initialized'); } + + return this._connection; } - private notifyDeletion(location: Location) { - for (const subscriber of this.subscribers) { - subscriber.next({ - added: [], - removed: [location], - }); - } - } - - location$(): Observable { - return new ObservableImpl(subscriber => { - this.subscribers.add(subscriber); - return () => { - this.subscribers.delete(subscriber); - }; + async connect(connection: EntityProviderConnection): Promise { + this._connection = connection; + const locations = await this.listLocations(); + const entities = locations.map(location => { + return locationSpecToLocationEntity(location); + }); + await this.connection.applyMutation({ + type: 'full', + entities, }); } } diff --git a/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts b/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts index 91090bc139..9c1d20ae9c 100644 --- a/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts +++ b/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts @@ -14,17 +14,25 @@ * limitations under the License. */ -import { ProcessingDatabase, RefreshStateItem } from './database/types'; +import { ProcessingDatabase } from './database/types'; import { - AddProcessingItemRequest, - ProccessingItem, + ProcessingItem, ProcessingItemResult, ProcessingStateManager, + ReplaceProcessingItemsRequest, } from './types'; export class DefaultProcessingStateManager implements ProcessingStateManager { constructor(private readonly db: ProcessingDatabase) {} + replaceProcessingItems( + request: ReplaceProcessingItemsRequest, + ): Promise { + return this.db.transaction(async tx => { + await this.db.replaceUnprocessedEntities(tx, request); + }); + } + async setProcessingItemResult(result: ProcessingItemResult) { return this.db.transaction(async tx => { await this.db.updateProcessedEntity(tx, { @@ -38,37 +46,24 @@ export class DefaultProcessingStateManager implements ProcessingStateManager { }); } - async addProcessingItems(request: AddProcessingItemRequest) { - return this.db.transaction(async tx => { - await this.db.addUnprocessedEntities(tx, request); - }); - } - - async getNextProcessingItem(): Promise { - const entities = await new Promise(resolve => - this.popFromQueue(resolve), - ); - const { id, state, unprocessedEntity } = entities[0]; - return { - id, - entity: unprocessedEntity, - state, - }; - } - - async popFromQueue(resolve: (rows: RefreshStateItem[]) => void) { - const entities = await this.db.transaction(async tx => { - return this.db.getProcessableEntities(tx, { - processBatchSize: 1, + async getNextProcessingItem(): Promise { + for (;;) { + const { items } = await this.db.transaction(async tx => { + return this.db.getProcessableEntities(tx, { + processBatchSize: 1, + }); }); - }); - // No entities require refresh, wait and try again. - if (!entities.items.length) { - setTimeout(() => this.popFromQueue(resolve), 1000); - return; + if (items.length) { + const { id, state, unprocessedEntity } = items[0]; + return { + id, + entity: unprocessedEntity, + state, + }; + } + + await new Promise(resolve => setTimeout(resolve, 1000)); } - - resolve(entities.items); } } diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index c3e0a6bd9c..10500fdc15 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -19,7 +19,6 @@ import { resolvePackagePath, UrlReader, } from '@backstage/backend-common'; -import fs from 'fs-extra'; import { DefaultNamespaceEntityPolicy, EntityPolicies, @@ -39,6 +38,7 @@ import { EntitiesCatalog, LocationsCatalog, } from '../catalog'; +import { CommonDatabase } from '../database/CommonDatabase'; import { AnnotateLocationEntityProcessor, BitbucketDiscoveryProcessor, @@ -53,7 +53,6 @@ import { MicrosoftGraphOrgReaderProcessor, PlaceholderProcessor, PlaceholderResolver, - StaticLocationProcessor, UrlReaderProcessor, } from '../ingestion'; import { RepoLocationAnalyzer } from '../ingestion/LocationAnalyzer'; @@ -64,16 +63,15 @@ import { } from '../ingestion/processors/PlaceholderProcessor'; import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; import { LocationAnalyzer } from '../ingestion/types'; +import { CatalogProcessingEngine } from '../next/types'; +import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider'; +import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; import { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; -import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; -import { DatabaseLocationProvider } from '../next/DatabaseLocationProvider'; import { DefaultLocationStore } from './DefaultLocationStore'; import { DefaultProcessingStateManager } from './DefaultProcessingStateManager'; -import { CatalogProcessingEngine } from '../next/types'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; import { Stitcher } from './Stitcher'; -import { CommonDatabase } from '../database/CommonDatabase'; export type CatalogEnvironment = { logger: Logger; @@ -244,19 +242,13 @@ export class NextCatalogBuilder { const parser = this.parser || defaultEntityDataParser; const dbClient = await database.getClient(); - const allMigrations = resolvePackagePath( - '@backstage/plugin-catalog-backend', - 'migrations', - ); - - const migrationsDir = resolvePackagePath( - '@backstage/plugin-catalog-backend', - 'migrationsv2', - ); - await fs.copy(allMigrations, migrationsDir); await dbClient.migrate.latest({ - directory: migrationsDir, + directory: resolvePackagePath( + '@backstage/plugin-catalog-backend', + 'migrationsv2', + ), }); + const db = new CommonDatabase(dbClient, logger); const processingDatabase = new DefaultProcessingDatabase(dbClient, logger); @@ -272,11 +264,11 @@ export class NextCatalogBuilder { const entitiesCatalog = new NextEntitiesCatalog(dbClient); const locationStore = new DefaultLocationStore(db); - const dbLocationProvider = new DatabaseLocationProvider(locationStore); const stitcher = new Stitcher(dbClient, logger); + const configLocationProvider = new ConfigLocationEntityProvider(config); const processingEngine = new DefaultCatalogProcessingEngine( logger, - [dbLocationProvider], // entityproviders + [locationStore, configLocationProvider], stateManager, orchestrator, stitcher, @@ -324,7 +316,6 @@ export class NextCatalogBuilder { // These are always there no matter what const processors: CatalogProcessor[] = [ - StaticLocationProcessor.fromConfig(config), new PlaceholderProcessor({ resolvers: placeholderResolvers, reader }), new BuiltinKindsEntityProcessor(), ]; @@ -340,7 +331,6 @@ export class NextCatalogBuilder { MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), - // new LocationEntityProcessor({ integrations }), new AnnotateLocationEntityProcessor({ integrations }), ); } diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts index 5bddd47f41..e87bb01e1f 100644 --- a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts @@ -37,7 +37,7 @@ export class NextEntitiesCatalog implements EntitiesCatalog { 'final_entities', ).select(); - const entities = dbResponse.map(e => JSON.parse(e.finalized_entity)); + const entities = dbResponse.map(e => JSON.parse(e.final_entity)); return { entities, diff --git a/plugins/catalog-backend/src/next/Stitcher.test.ts b/plugins/catalog-backend/src/next/Stitcher.test.ts new file mode 100644 index 0000000000..f38af384fa --- /dev/null +++ b/plugins/catalog-backend/src/next/Stitcher.test.ts @@ -0,0 +1,206 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { Knex } from 'knex'; +import { DatabaseManager } from './database/DatabaseManager'; +import { + DbRefreshStateReferencesRow, + DbRefreshStateRow, + DbRelationsRow, +} from './database/DefaultProcessingDatabase'; +import { DbSearchRow } from './search'; +import { DbFinalEntitiesRow, Stitcher } from './Stitcher'; + +describe('Stitcher', () => { + let db: Knex; + const logger = getVoidLogger(); + + beforeEach(async () => { + db = await DatabaseManager.createTestDatabaseConnection(); + await DatabaseManager.createDatabase(db); + }); + + it('runs the happy path', async () => { + const stitcher = new Stitcher(db, logger); + + await db.transaction(async tx => { + await tx('refresh_state').insert([ + { + entity_id: 'my-id', + entity_ref: 'k:ns/n', + unprocessed_entity: JSON.stringify({}), + processed_entity: JSON.stringify({ + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + }, + spec: { + k: 'v', + }, + }), + errors: '[]', + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), + }, + ]); + await tx('refresh_state_references').insert([ + { source_key: 'a', target_entity_ref: 'k:ns/n' }, + ]); + await tx('relations').insert([ + { + originating_entity_id: 'my-id', + source_entity_ref: 'k:ns/n', + type: 'looksAt', + target_entity_ref: 'k:ns/other', + }, + ]); + }); + + await stitcher.stitch(new Set(['k:ns/n'])); + + let firstHash: string; + await db.transaction(async tx => { + const entities = await tx('final_entities'); + + expect(entities.length).toBe(1); + const entity = JSON.parse(entities[0].final_entity); + expect(entity).toEqual({ + relations: [ + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'other', + }, + }, + ], + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + etag: expect.any(String), + generation: 1, + uid: 'my-id', + }, + spec: { + k: 'v', + }, + }); + + expect(entity.metadata.etag).toEqual(entities[0].hash); + firstHash = entities[0].hash; + + const search = await tx('search'); + expect(search).toEqual( + expect.arrayContaining([ + { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, + { entity_id: 'my-id', key: 'apiversion', value: 'a' }, + { entity_id: 'my-id', key: 'kind', value: 'k' }, + { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, + { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, + { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, + { entity_id: 'my-id', key: 'spec.k', value: 'v' }, + ]), + ); + }); + + // Re-stitch without any changes + await stitcher.stitch(new Set(['k:ns/n'])); + + await db.transaction(async tx => { + const entities = await tx('final_entities'); + expect(entities.length).toBe(1); + const entity = JSON.parse(entities[0].final_entity); + expect(entities[0].hash).toEqual(firstHash); + expect(entity.metadata.etag).toEqual(firstHash); + }); + + // Now add one more relation and re-stitch + await db.transaction(async tx => { + await tx('relations').insert([ + { + originating_entity_id: 'my-id', + source_entity_ref: 'k:ns/n', + type: 'looksAt', + target_entity_ref: 'k:ns/third', + }, + ]); + }); + + await stitcher.stitch(new Set(['k:ns/n'])); + + await db.transaction(async tx => { + const entities = await tx('final_entities'); + + expect(entities.length).toBe(1); + const entity = JSON.parse(entities[0].final_entity); + expect(entity).toEqual({ + relations: expect.arrayContaining([ + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'other', + }, + }, + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'third', + }, + }, + ]), + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + etag: expect.any(String), + generation: 1, + uid: 'my-id', + }, + spec: { + k: 'v', + }, + }); + + expect(entities[0].hash).not.toEqual(firstHash); + expect(entities[0].hash).toEqual(entity.metadata.etag); + + const search = await tx('search'); + expect(search).toEqual( + expect.arrayContaining([ + { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, + { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/third' }, + { entity_id: 'my-id', key: 'apiversion', value: 'a' }, + { entity_id: 'my-id', key: 'kind', value: 'k' }, + { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, + { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, + { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, + { entity_id: 'my-id', key: 'spec.k', value: 'v' }, + ]), + ); + }); + }); +}); diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index 3e5e3ac24b..bdc028d923 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -14,26 +14,28 @@ * limitations under the License. */ +import { Entity, parseEntityRef } from '@backstage/catalog-model'; +import { ConflictError } from '@backstage/errors'; +import { createHash } from 'crypto'; +import stableStringify from 'fast-json-stable-stringify'; import { Knex } from 'knex'; import { Logger } from 'winston'; import { Transaction } from '../database'; -import { ConflictError } from '@backstage/errors'; -import { - DbRefreshStateReferences, - DbRefreshStateRow, - DbRelationsRow, -} from './database/DefaultProcessingDatabase'; -import { Entity, parseEntityRef } from '@backstage/catalog-model'; -import { createHash } from 'crypto'; -import stableStringify from 'fast-json-stable-stringify'; +import { buildEntitySearch, DbSearchRow } from './search'; + +// The number of items that are sent per batch to the database layer, when +// doing .batchInsert calls to knex. This needs to be low enough to not cause +// errors in the underlying engine due to exceeding query limits, but large +// enough to get the speed benefits. +const BATCH_SIZE = 50; export type DbFinalEntitiesRow = { entity_id: string; - etag: string; - finalized_entity: string; + hash: string; + final_entity: string; }; -function generateEntityEtag(entity: Entity) { +function generateStableHash(entity: Entity) { return createHash('sha1') .update(stableStringify({ ...entity })) .digest('hex'); @@ -49,64 +51,129 @@ export class Stitcher { for (const entityRef of entityRefs) { await this.transaction(async txOpaque => { const tx = txOpaque as Knex.Transaction; - const [result] = await tx('refresh_state') - .select('entity_id', 'processed_entity') - .where({ entity_ref: entityRef }); - if (!result) { + // Selecting from refresh_state and final_entities should yield exactly + // one row (except in abnormal cases where the stitch was invoked for + // something that didn't exist at all, in which case it's zero rows). + // The join with the temporary incoming_references still gives one row. + // The only result set "expanding" join is the one with relations, so + // the output should be at least one row (if zero or one relations were + // found), or at most the same number of rows as relations. + const result: Array<{ + entityId: string; + processedEntity?: string; + errors: string; + incomingReferenceCount: string | number; + previousHash?: string; + relationType?: string; + relationTarget?: string; + }> = await tx + .with('incoming_references', function incomingReferences(builder) { + return builder + .from('refresh_state_references') + .where({ target_entity_ref: entityRef }) + .count({ count: '*' }); + }) + .select({ + entityId: 'refresh_state.entity_id', + processedEntity: 'refresh_state.processed_entity', + errors: 'refresh_state.errors', + incomingReferenceCount: 'incoming_references.count', + previousHash: 'final_entities.hash', + relationType: 'relations.type', + relationTarget: 'relations.target_entity_ref', + }) + .from('refresh_state') + .leftJoin('incoming_references', {}) + .leftOuterJoin('final_entities', { + 'final_entities.entity_id': 'refresh_state.entity_id', + }) + .leftOuterJoin('relations', { + 'relations.source_entity_ref': 'refresh_state.entity_ref', + }) + .where({ 'refresh_state.entity_ref': entityRef }) + .orderBy('relationType', 'asc') + .orderBy('relationTarget', 'asc'); + + // If there were no rows returned, it would mean that there was no + // matching row even in the refresh_state. This can happen for example + // if we emit a relation to something that hasn't been ingested yet. + // It's safe to ignore this stitch attempt in that case. + if (!result.length) { this.logger.debug( `Unable to stitch ${entityRef}, item does not exist in refresh state table`, ); return; - } else if (!result.processed_entity) { + } + + const { + entityId, + processedEntity, + // errors, + incomingReferenceCount, + previousHash, + } = result[0]; + + // If there was no processed entity in place, the target hasn't been + // through the processing steps yet. It's safe to ignore this stitch + // attempt in that case, since another stitch will be triggered when + // that processing has finished. + if (!processedEntity) { this.logger.debug( `Unable to stitch ${entityRef}, the entity has not yet been processed`, ); return; } - const entity: Entity = JSON.parse(result.processed_entity); + // Grab the processed entity and stitch all of the relevant data into + // it + const entity = JSON.parse(processedEntity) as Entity; + const isOrphan = Number(incomingReferenceCount) === 0; - const entityId = entity?.metadata?.uid; - if (!entityId) { - this.logger.error(`missing ID in entity ${JSON.stringify(entity)}`); - return; - } - - const [reference_count_result] = await tx( - 'refresh_state_references', - ) - .where({ target_entity_id: entity.metadata.uid }) - .count({ reference_count: 'target_entity_id' }); - - if (Number(reference_count_result.reference_count) === 0) { - this.logger.debug(`${entityRef} is orphan`); + if (isOrphan) { + this.logger.debug(`${entityRef} is an orphan`); entity.metadata.annotations = { ...entity.metadata.annotations, ['backstage.io/orphan']: 'true', }; } - const relationResults = await tx('relations') - .where({ source_entity_ref: entityRef }) - .select(); + // TODO: entityRef is lower case and should be uppercase in the final + // result + entity.relations = result + .filter(row => row.relationType /* exclude null row, if relevant */) + .map(row => ({ + type: row.relationType!, + target: parseEntityRef(row.relationTarget!), + })); - // TODO: entityRef is lower case and should be uppercase in the final result. - entity.relations = relationResults.map(relation => ({ - type: relation.type, - target: parseEntityRef(relation.target_entity_ref), - })); + // If the output entity was actually not changed, just abort + const hash = generateStableHash(entity); + if (hash === previousHash) { + this.logger.debug(`Skipped stitching of ${entityRef}, no changes`); + return; + } + + entity.metadata.uid = entityId; entity.metadata.generation = 1; - const etag = generateEntityEtag(entity); - entity.metadata.etag = etag; + if (!entity.metadata.etag) { + // If the original data source did not have its own etag handling, + // use the hash as a good-quality etag + entity.metadata.etag = hash; + } + await tx('final_entities') .insert({ - finalized_entity: JSON.stringify(entity), entity_id: entityId, - etag, + final_entity: JSON.stringify(entity), + hash, }) .onConflict('entity_id') - .merge(['finalized_entity', 'etag']); + .merge(['final_entity', 'hash']); + + const searchEntries = buildEntitySearch(entityId, entity); + await tx('search').where({ entity_id: entityId }).delete(); + await tx.batchInsert('search', searchEntries, BATCH_SIZE); }); } } diff --git a/plugins/catalog-backend/src/next/database/DatabaseManager.ts b/plugins/catalog-backend/src/next/database/DatabaseManager.ts new file mode 100644 index 0000000000..22ae4e782f --- /dev/null +++ b/plugins/catalog-backend/src/next/database/DatabaseManager.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; +import knexFactory, { Knex } from 'knex'; +import { v4 as uuid } from 'uuid'; +import { Logger } from 'winston'; +import { CommonDatabase } from '../../database/CommonDatabase'; +import { Database } from '../../database/types'; + +export type CreateDatabaseOptions = { + logger: Logger; +}; + +const defaultOptions: CreateDatabaseOptions = { + logger: getVoidLogger(), +}; + +export class DatabaseManager { + public static async createDatabase( + knex: Knex, + options: Partial = {}, + ): Promise { + const migrationsDir = resolvePackagePath( + '@backstage/plugin-catalog-backend', + 'migrationsv2', + ); + + await knex.migrate.latest({ + directory: migrationsDir, + }); + const { logger } = { ...defaultOptions, ...options }; + return new CommonDatabase(knex, logger); + } + + public static async createInMemoryDatabase(): Promise { + const knex = await this.createInMemoryDatabaseConnection(); + return await this.createDatabase(knex); + } + + public static async createInMemoryDatabaseConnection(): Promise { + const knex = knexFactory({ + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }); + + knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { + resource.run('PRAGMA foreign_keys = ON', () => {}); + }); + + return knex; + } + + public static async createTestDatabase(): Promise { + const knex = await this.createTestDatabaseConnection(); + return await this.createDatabase(knex); + } + + public static async createTestDatabaseConnection(): Promise { + const config: Knex.Config = { + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }; + + let knex = knexFactory(config); + if (typeof config.connection !== 'string') { + const tempDbName = `d${uuid().replace(/-/g, '')}`; + await knex.raw(`CREATE DATABASE ${tempDbName};`); + knex = knexFactory({ + ...config, + connection: { + ...config.connection, + database: tempDbName, + }, + }); + } + + knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { + resource.run('PRAGMA foreign_keys = ON', () => {}); + }); + + return knex; + } +} diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts new file mode 100644 index 0000000000..d76bd74260 --- /dev/null +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -0,0 +1,517 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; +import { DatabaseManager } from './DatabaseManager'; +import { Knex } from 'knex'; +import { + DbRefreshStateReferencesRow, + DbRefreshStateRow, + DbRelationsRow, + DefaultProcessingDatabase, +} from './DefaultProcessingDatabase'; + +import { Entity, EntityRelationSpec } from '@backstage/catalog-model'; +import * as uuid from 'uuid'; +import { getVoidLogger } from '@backstage/backend-common'; + +describe('Default Processing Database', () => { + let db: Knex; + let processingDatabase: DefaultProcessingDatabase; + const logger = getVoidLogger(); + + beforeEach(async () => { + db = await DatabaseManager.createTestDatabaseConnection(); + await DatabaseManager.createDatabase(db); + + processingDatabase = new DefaultProcessingDatabase(db, logger); + }); + + describe('replaceUnprocessedEntities', () => { + const insertRefRow = async (ref: DbRefreshStateReferencesRow) => { + return db('refresh_state_references').insert( + ref, + ); + }; + + const insertRefreshStateRow = async (ref: DbRefreshStateRow) => { + await db('refresh_state').insert(ref); + }; + + const createLocations = async (entityRefs: string[]) => { + for (const ref of entityRefs) { + await insertRefreshStateRow({ + entity_id: uuid.v4(), + entity_ref: ref, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '', + next_update_at: 'now()', + last_discovery_at: 'now()', + }); + } + }; + + it('replaces all existing state correctly for simple dependency chains', async () => { + /* + config -> location:default/root -> location:default/root-1 -> location:default/root-2 + database -> location:default/second -> location:default/root-2 + */ + await createLocations([ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-2', + 'location:default/second', + ]); + + await insertRefRow({ + source_key: 'config', + target_entity_ref: 'location:default/root', + }); + + await insertRefRow({ + source_key: 'database', + target_entity_ref: 'location:default/second', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root-1', + target_entity_ref: 'location:default/root-2', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/second', + target_entity_ref: 'location:default/root-2', + }); + + await processingDatabase.transaction(async tx => { + await processingDatabase.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config', + items: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }); + }); + + const currentRefreshState = await db( + 'refresh_state', + ).select(); + + const currentRefRowState = await db( + 'refresh_state_references', + ).select(); + + for (const ref of ['location:default/root', 'location:default/root-1']) { + expect(currentRefreshState.some(t => t.entity_ref === ref)).toBeFalsy(); + } + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.target_entity_ref === 'location:default/root-1' && + t.source_key === 'config', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.target_entity_ref === 'location:default/new-root' && + t.source_key === 'config', + ), + ).toBeTruthy(); + }); + + it('should work for more complex chains', async () => { + /* + config -> location:default/root -> location:default/root-1 -> location:default/root-2 + config -> location:default/root -> location:default/root-1a -> location:default/root-2 + */ + await createLocations([ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-2', + 'location:default/root-1a', + ]); + + await insertRefRow({ + source_key: 'config', + target_entity_ref: 'location:default/root', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1a', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root-1', + target_entity_ref: 'location:default/root-2', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root-1a', + target_entity_ref: 'location:default/root-2', + }); + + await processingDatabase.transaction(async tx => { + await processingDatabase.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config', + items: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }); + }); + + const currentRefreshState = await db( + 'refresh_state', + ).select(); + + const currentRefRowState = await db( + 'refresh_state_references', + ).select(); + + const deletedRefs = [ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-1a', + 'location:default/root-2', + ]; + + for (const ref of deletedRefs) { + expect(currentRefreshState.some(t => t.entity_ref === ref)).toBeFalsy(); + } + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'config' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'config' && + t.target_entity_ref === 'location:default/root', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1a', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1a' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); + }); + + it('should add new locations using the delta options', async () => { + await processingDatabase.transaction(async tx => { + await processingDatabase.replaceUnprocessedEntities(tx, { + type: 'delta', + sourceKey: 'lols', + removed: [], + added: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }); + }); + + const currentRefreshState = await db( + 'refresh_state', + ).select(); + + const currentRefRowState = await db( + 'refresh_state_references', + ).select(); + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'lols' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + }); + + it('should not remove locations that are referenced elsewhere', async () => { + /* + config-1 -> location:default/root + config-2 -> location:default/root + */ + await createLocations(['location:default/root']); + + await insertRefRow({ + source_key: 'config-1', + target_entity_ref: 'location:default/root', + }); + await insertRefRow({ + source_key: 'config-2', + target_entity_ref: 'location:default/root', + }); + + await processingDatabase.transaction(async tx => { + await processingDatabase.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config-1', + items: [], + }); + }); + + const currentRefreshState = await db( + 'refresh_state', + ).select(); + + const currentRefRowState = await db( + 'refresh_state_references', + ).select(); + + expect(currentRefRowState).toEqual([ + expect.objectContaining({ + source_key: 'config-2', + target_entity_ref: 'location:default/root', + }), + ]); + + expect(currentRefreshState).toEqual([ + expect.objectContaining({ + entity_ref: 'location:default/root', + }), + ]); + }); + + it('should remove old locations using the delta options', async () => { + await createLocations(['location:default/new-root']); + + await insertRefRow({ + source_key: 'lols', + target_entity_ref: 'location:default/new-root', + }); + + await processingDatabase.transaction(async tx => { + await processingDatabase.replaceUnprocessedEntities(tx, { + type: 'delta', + sourceKey: 'lols', + added: [], + removed: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }); + }); + + const currentRefreshState = await db( + 'refresh_state', + ).select(); + + const currentRefRowState = await db( + 'refresh_state_references', + ).select(); + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'lols' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeFalsy(); + }); + }); + + describe('updateProcessedEntity', () => { + it('should throw if the entity does not exist', async () => { + await processingDatabase.transaction(async tx => { + await expect( + processingDatabase.updateProcessedEntity(tx, { + id: '9', + processedEntity: { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + deferredEntities: [], + relations: [], + }), + ).rejects.toThrow('Processing state not found for 9'); + }); + }); + + it('should update a processed entity', async () => { + await db('refresh_state').insert({ + entity_id: '321', + entity_ref: 'location:default/new-root', + unprocessed_entity: '', + errors: '', + next_update_at: 'now()', + last_discovery_at: 'now()', + }); + + const deferredEntity = { + apiVersion: '1.0.0', + metadata: { + name: 'deferred', + }, + kind: 'Location', + } as Entity; + + const relation: EntityRelationSpec = { + source: { + kind: 'Component', + namespace: 'Default', + name: 'foo', + }, + target: { + kind: 'Component', + namespace: 'Default', + name: 'foo', + }, + type: 'url', + }; + + await processingDatabase.transaction(async tx => { + await processingDatabase.updateProcessedEntity(tx, { + id: '321', + processedEntity: { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + deferredEntities: [deferredEntity], + relations: [relation], + }); + }); + + const deferredResult = await db('refresh_state') + .where({ entity_ref: 'location:default/deferred' }) + .select(); + expect(deferredResult.length).toBe(1); + + const [relations] = await db('relations') + .where({ originating_entity_id: '321' }) + .select(); + expect(relations).toEqual({ + originating_entity_id: '321', + source_entity_ref: 'component:default/foo', + type: 'url', + target_entity_ref: 'component:default/foo', + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index cf2d4e121c..dc5e6036f5 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -15,6 +15,7 @@ */ import { ConflictError, NotFoundError } from '@backstage/errors'; +import { stringifyEntityRef, Entity } from '@backstage/catalog-model'; import { Knex } from 'knex'; import { Transaction } from '../../database'; import lodash from 'lodash'; @@ -24,20 +25,23 @@ import { AddUnprocessedEntitiesOptions, UpdateProcessedEntityOptions, GetProcessableEntitiesResult, + ReplaceUnprocessedEntitiesOptions, + RefreshStateItem, } from './types'; import type { Logger } from 'winston'; -import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; + import { v4 as uuid } from 'uuid'; +import { JsonObject } from '@backstage/config'; export type DbRefreshStateRow = { entity_id: string; entity_ref: string; unprocessed_entity: string; - processed_entity: string; - cache: string; + processed_entity?: string; + cache?: string; next_update_at: string; last_discovery_at: string; // remove? - errors: string; + errors?: string; }; export type DbRelationsRow = { @@ -47,10 +51,10 @@ export type DbRelationsRow = { type: string; }; -export type DbRefreshStateReferences = { - source_special_key?: string; - source_entity_id?: string; - target_entity_id: string; +export type DbRefreshStateReferencesRow = { + source_key?: string; + source_entity_ref?: string; + target_entity_ref: string; }; // The number of items that are sent per batch to the database layer, when @@ -86,7 +90,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { errors, }) .where('entity_id', id); - if (refreshResult === 0) { throw new NotFoundError(`Processing state not found for ${id}`); } @@ -94,8 +97,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { // Schedule all deferred entities for future processing. await this.addUnprocessedEntities(tx, { entities: deferredEntities, - id, - type: 'entity', + entityRef: stringifyEntityRef(processedEntity), }); // Update fragments @@ -128,49 +130,230 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ); } + private async createDelta( + tx: Knex.Transaction, + options: ReplaceUnprocessedEntitiesOptions, + ): Promise<{ toAdd: Entity[]; toRemove: string[] }> { + if (options.type === 'delta') { + return { + toAdd: options.added, + toRemove: options.removed.map(e => stringifyEntityRef(e)), + }; + } + + const oldRefs = await tx( + 'refresh_state_references', + ) + .where({ source_key: options.sourceKey }) + .select('target_entity_ref') + .then(rows => rows.map(r => r.target_entity_ref)); + + const items = options.items.map(entity => ({ + entity, + ref: stringifyEntityRef(entity), + })); + + const oldRefsSet = new Set(oldRefs); + const newRefsSet = new Set(items.map(item => item.ref)); + const toAdd = items.filter(item => !oldRefsSet.has(item.ref)); + const toRemove = oldRefs.filter(ref => !newRefsSet.has(ref)); + + return { toAdd: toAdd.map(({ entity }) => entity), toRemove }; + } + + async replaceUnprocessedEntities( + txOpaque: Transaction, + options: ReplaceUnprocessedEntitiesOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + + const { toAdd, toRemove } = await this.createDelta(tx, options); + + if (toRemove.length) { + // TODO(freben): Batch split, to not hit variable limits? + /* + WITH RECURSIVE + -- All the nodes that can be reached downwards from our root + descendants(root_id, entity_ref) AS ( + SELECT id, target_entity_ref + FROM refresh_state_references + WHERE source_key = "R1" AND target_entity_ref = "A" + UNION + SELECT descendants.root_id, target_entity_ref + FROM descendants + JOIN refresh_state_references ON source_entity_ref = descendants.entity_ref + ), + -- All the nodes that can be reached upwards from the descendants + ancestors(root_id, via_entity_ref, to_entity_ref) AS ( + SELECT NULL, entity_ref, entity_ref + FROM descendants + UNION + SELECT + CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END, + source_entity_ref, + ancestors.to_entity_ref + FROM ancestors + JOIN refresh_state_references ON target_entity_ref = ancestors.via_entity_ref + ) + -- Start out with all of the descendants + SELECT descendants.entity_ref + FROM descendants + -- Expand with all ancestors that point to those, but aren't the current root + LEFT OUTER JOIN ancestors + ON ancestors.to_entity_ref = descendants.entity_ref + AND ancestors.root_id IS NOT NULL + AND ancestors.root_id != descendants.root_id + -- Exclude all lines that had such a foreign ancestor + WHERE ancestors.root_id IS NULL; + */ + const removedCount = await tx('refresh_state') + .whereIn('entity_ref', function orphanedEntityRefs(orphans) { + return ( + orphans + // All the nodes that can be reached downwards from our root + .withRecursive('descendants', function descendants(outer) { + return outer + .select({ root_id: 'id', entity_ref: 'target_entity_ref' }) + .from('refresh_state_references') + .where('source_key', options.sourceKey) + .whereIn('target_entity_ref', toRemove) + .union(function recursive(inner) { + return inner + .select({ + root_id: 'descendants.root_id', + entity_ref: + 'refresh_state_references.target_entity_ref', + }) + .from('descendants') + .join('refresh_state_references', { + 'descendants.entity_ref': + 'refresh_state_references.source_entity_ref', + }); + }); + }) + // All the nodes that can be reached upwards from the descendants + .withRecursive('ancestors', function ancestors(outer) { + return outer + .select({ + root_id: tx.raw('NULL', []), + via_entity_ref: 'entity_ref', + to_entity_ref: 'entity_ref', + }) + .from('descendants') + .union(function recursive(inner) { + return inner + .select({ + root_id: tx.raw( + 'CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END', + [], + ), + via_entity_ref: 'source_entity_ref', + to_entity_ref: 'ancestors.to_entity_ref', + }) + .from('ancestors') + .join('refresh_state_references', { + target_entity_ref: 'ancestors.via_entity_ref', + }); + }); + }) + // Start out with all of the descendants + .select('descendants.entity_ref') + .from('descendants') + // Expand with all ancestors that point to those, but aren't the current root + .leftOuterJoin('ancestors', function keepaliveRoots() { + this.on( + 'ancestors.to_entity_ref', + '=', + 'descendants.entity_ref', + ); + this.andOnNotNull('ancestors.root_id'); + this.andOn('ancestors.root_id', '!=', 'descendants.root_id'); + }) + .whereNull('ancestors.root_id') + ); + }) + .delete(); + + await tx('refresh_state_references') + .where('source_key', '=', options.sourceKey) + .whereIn('target_entity_ref', toRemove) + .delete(); + + this.logger.debug( + `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`, + ); + } + + if (toAdd.length) { + const state: Knex.DbRecord[] = toAdd.map(entity => ({ + entity_id: uuid(), + entity_ref: stringifyEntityRef(entity), + unprocessed_entity: JSON.stringify(entity), + errors: '', + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), + })); + + const stateReferences: DbRefreshStateReferencesRow[] = toAdd.map( + entity => ({ + source_key: options.sourceKey, + target_entity_ref: stringifyEntityRef(entity), + }), + ); + // TODO(freben): Concurrency? If we did these one by one, a .onConflict().merge would have made sense + await tx.batchInsert('refresh_state', state, BATCH_SIZE); + await tx.batchInsert( + 'refresh_state_references', + stateReferences, + BATCH_SIZE, + ); + } + } + async addUnprocessedEntities( txOpaque: Transaction, options: AddUnprocessedEntitiesOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - const entityIds = new Array(); - for (const entity of options.entities) { - const entityRef = stringifyEntityRef(entity); - await tx('refresh_state') - .insert({ + const stateRows = options.entities.map( + entity => + ({ entity_id: uuid(), - entity_ref: entityRef, + entity_ref: stringifyEntityRef(entity), unprocessed_entity: JSON.stringify(entity), errors: '', next_update_at: tx.fn.now(), last_discovery_at: tx.fn.now(), - }) + } as Knex.DbRecord), + ); + const stateReferenceRows = stateRows.map( + stateRow => + ({ + source_entity_ref: options.entityRef, + target_entity_ref: stateRow.entity_ref, + } as Knex.DbRecord), + ); + + // Upsert all of the unprocessed entities into the refresh_state table, by + // their entity ref. + // TODO(freben): Can this be batched somehow? + for (const row of stateRows) { + await tx('refresh_state') + .insert(row) .onConflict('entity_ref') .merge(['unprocessed_entity', 'last_discovery_at']); - - const [{ entity_id: entityId }] = await tx( - 'refresh_state', - ).where({ entity_ref: entityRef }); - entityIds.push(entityId); } - const key = - options.type === 'provider' - ? { source_special_key: options.id } - : { source_entity_id: options.id }; - // copied from update refs - await tx('refresh_state_references') - .where(key) + // Replace all references for the originating entity before creating new ones + await tx('refresh_state_references') + .where({ source_entity_ref: options.entityRef }) .delete(); - - const referenceRows: DbRefreshStateReferences[] = entityIds.map( - entityId => ({ - ...key, - target_entity_id: entityId, - }), + await tx.batchInsert( + 'refresh_state_references', + stateReferenceRows, + BATCH_SIZE, ); - await tx.batchInsert('refresh_state_references', referenceRows, BATCH_SIZE); } async getProcessableEntities( @@ -198,16 +381,23 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { }); return { - items: items.map(i => ({ - id: i.entity_id, - entityRef: i.entity_ref, - unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity, - processedEntity: JSON.parse(i.processed_entity) as Entity, - nextUpdateAt: i.next_update_at, - lastDiscoveryAt: i.last_discovery_at, - state: JSON.parse(i.cache), - errors: i.errors, - })), + items: items.map( + i => + ({ + id: i.entity_id, + entityRef: i.entity_ref, + unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity, + processedEntity: i.processed_entity + ? (JSON.parse(i.processed_entity) as Entity) + : undefined, + nextUpdateAt: i.next_update_at, + lastDiscoveryAt: i.last_discovery_at, + state: i.cache + ? JSON.parse(i.cache) + : new Map(), + errors: i.errors, + } as RefreshStateItem), + ), }; } diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index 22e9e90c8e..d7a1dcd20b 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -19,8 +19,7 @@ import { JsonObject } from '@backstage/config'; import { Transaction } from '../../database/types'; export type AddUnprocessedEntitiesOptions = { - type: 'entity' | 'provider'; - id: string; + entityRef: string; entities: Entity[]; }; @@ -39,17 +38,30 @@ export type RefreshStateItem = { id: string; entityRef: string; unprocessedEntity: Entity; - processedEntity: Entity; + processedEntity?: Entity; nextUpdateAt: string; lastDiscoveryAt: string; // remove? state: Map; - errors: string; + errors?: string; }; export type GetProcessableEntitiesResult = { items: RefreshStateItem[]; }; +export type ReplaceUnprocessedEntitiesOptions = + | { + sourceKey: string; + items: Entity[]; + type: 'full'; + } + | { + sourceKey: string; + added: Entity[]; + removed: Entity[]; + type: 'delta'; + }; + export interface ProcessingDatabase { transaction(fn: (tx: Transaction) => Promise): Promise; @@ -58,6 +70,10 @@ export interface ProcessingDatabase { options: AddUnprocessedEntitiesOptions, ): Promise; + replaceUnprocessedEntities( + txOpaque: Transaction, + options: ReplaceUnprocessedEntitiesOptions, + ): Promise; getProcessableEntities( txOpaque: Transaction, request: { processBatchSize: number }, diff --git a/plugins/catalog-backend/src/next/search.test.ts b/plugins/catalog-backend/src/next/search.test.ts new file mode 100644 index 0000000000..2e0b07b4a9 --- /dev/null +++ b/plugins/catalog-backend/src/next/search.test.ts @@ -0,0 +1,161 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; +import { buildEntitySearch, mapToRows, traverse } from './search'; + +describe('search', () => { + describe('traverse', () => { + it('expands lists of strings to several rows', () => { + const input = { a: ['b', 'c', 'd'] }; + const output = traverse(input); + expect(output).toEqual([ + { key: 'a', value: 'b' }, + { key: 'a.b', value: true }, + { key: 'a', value: 'c' }, + { key: 'a.c', value: true }, + { key: 'a', value: 'd' }, + { key: 'a.d', value: true }, + ]); + }); + + it('expands objects', () => { + const input = { a: { b: { c: 'd' }, e: 'f' } }; + const output = traverse(input); + expect(output).toEqual([ + { key: 'a.b.c', value: 'd' }, + { key: 'a.e', value: 'f' }, + ]); + }); + + it('expands list of objects', () => { + const input = { root: { list: [{ a: 1 }, { a: 2 }] } }; + const output = traverse(input); + expect(output).toEqual([ + { key: 'root.list.a', value: 1 }, + { key: 'root.list.a', value: 2 }, + ]); + }); + + it('skips over special keys', () => { + const input = { + status: { x: 1 }, + attachments: [{ y: 2 }], + relations: [{ z: 3 }], + a: 'a', + metadata: { + b: 'b', + name: 'name', + namespace: 'namespace', + uid: 'uid', + etag: 'etag', + generation: 'generation', + c: 'c', + }, + d: 'd', + }; + const output = traverse(input); + expect(output).toEqual([ + { key: 'a', value: 'a' }, + { key: 'metadata.b', value: 'b' }, + { key: 'metadata.c', value: 'c' }, + { key: 'd', value: 'd' }, + ]); + }); + }); + + describe('mapToRows', () => { + it('converts base types to strings or null', () => { + const input = [ + { key: 'a', value: true }, + { key: 'b', value: false }, + { key: 'c', value: 7 }, + { key: 'd', value: 'string' }, + { key: 'e', value: null }, + { key: 'f', value: undefined }, + ]; + const output = mapToRows(input, 'eid'); + expect(output).toEqual([ + { entity_id: 'eid', key: 'a', value: 'true' }, + { entity_id: 'eid', key: 'b', value: 'false' }, + { entity_id: 'eid', key: 'c', value: '7' }, + { entity_id: 'eid', key: 'd', value: 'string' }, + { entity_id: 'eid', key: 'e', value: null }, + { entity_id: 'eid', key: 'f', value: null }, + ]); + }); + + it('emits lowercase version of keys and values', () => { + const input = [{ key: 'fOo', value: 'BaR' }]; + const output = mapToRows(input, 'eid'); + expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: 'bar' }]); + }); + + it('skips very large values', () => { + const input = [{ key: 'foo', value: 'a'.repeat(10000) }]; + const output = mapToRows(input, 'eid'); + expect(output).toEqual([]); + }); + }); + + describe('buildEntitySearch', () => { + it('adds special keys even if missing', () => { + const input: Entity = { + apiVersion: 'a', + kind: 'b', + metadata: { name: 'n' }, + }; + expect(buildEntitySearch('eid', input)).toEqual([ + { entity_id: 'eid', key: 'apiversion', value: 'a' }, + { entity_id: 'eid', key: 'kind', value: 'b' }, + { entity_id: 'eid', key: 'metadata.name', value: 'n' }, + { entity_id: 'eid', key: 'metadata.namespace', value: null }, + { entity_id: 'eid', key: 'metadata.uid', value: null }, + { + entity_id: 'eid', + key: 'metadata.namespace', + value: ENTITY_DEFAULT_NAMESPACE, + }, + ]); + }); + + it('adds relations', () => { + const input: Entity = { + relations: [ + { type: 't1', target: { kind: 'k', namespace: 'ns', name: 'a' } }, + { type: 't2', target: { kind: 'k', namespace: 'ns', name: 'b' } }, + ], + apiVersion: 'a', + kind: 'b', + metadata: { name: 'n' }, + }; + expect(buildEntitySearch('eid', input)).toEqual([ + { entity_id: 'eid', key: 'apiversion', value: 'a' }, + { entity_id: 'eid', key: 'kind', value: 'b' }, + { entity_id: 'eid', key: 'metadata.name', value: 'n' }, + { entity_id: 'eid', key: 'metadata.namespace', value: null }, + { entity_id: 'eid', key: 'metadata.uid', value: null }, + { + entity_id: 'eid', + key: 'metadata.namespace', + value: ENTITY_DEFAULT_NAMESPACE, + }, + { entity_id: 'eid', key: 'relations.t1', value: 'k:ns/a' }, + { entity_id: 'eid', key: 'relations.t2', value: 'k:ns/b' }, + ]); + }); + }); +}); diff --git a/plugins/catalog-backend/src/next/search.ts b/plugins/catalog-backend/src/next/search.ts new file mode 100644 index 0000000000..383b29fd70 --- /dev/null +++ b/plugins/catalog-backend/src/next/search.ts @@ -0,0 +1,192 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + ENTITY_DEFAULT_NAMESPACE, + stringifyEntityRef, +} from '@backstage/catalog-model'; + +export type DbSearchRow = { + entity_id: string; + key: string; + value: string | null; +}; + +// These are excluded in the generic loop, either because they do not make sense +// to index, or because they are special-case always inserted whether they are +// null or not +const SPECIAL_KEYS = [ + 'attachments', + 'relations', + 'status', + 'metadata.name', + 'metadata.namespace', + 'metadata.uid', + 'metadata.etag', + 'metadata.generation', +]; + +// The maximum length allowed for search values. These columns are indexed, and +// database engines do not like to index on massive values. For example, +// postgres will balk after 8191 byte line sizes. +const MAX_VALUE_LENGTH = 200; + +type Kv = { + key: string; + value: unknown; +}; + +// Helper for traversing through a nested structure and outputting a list of +// path->value entries of the leaves. +// +// For example, this yaml structure +// +// a: 1 +// b: +// c: null +// e: [f, g] +// h: +// - i: 1 +// j: k +// - i: 2 +// j: l +// +// will result in +// +// "a", 1 +// "b.c", null +// "b.e": "f" +// "b.e.f": true +// "b.e": "g" +// "b.e.g": true +// "h.i": 1 +// "h.j": "k" +// "h.i": 2 +// "h.j": "l" +export function traverse(root: unknown): Kv[] { + const output: Kv[] = []; + + function visit(path: string, current: unknown) { + if (SPECIAL_KEYS.includes(path)) { + return; + } + + // empty or scalar + if ( + current === undefined || + current === null || + ['string', 'number', 'boolean'].includes(typeof current) + ) { + output.push({ key: path, value: current }); + return; + } + + // unknown + if (typeof current !== 'object') { + return; + } + + // array + if (Array.isArray(current)) { + for (const item of current) { + // NOTE(freben): The reason that these are output in two different ways, + // is to support use cases where you want to express that MORE than one + // tag is present in a list. Since the EntityFilters structure is a + // record, you can't have several entries of the same key. Therefore + // you will have to match on + // + // { "a.b": ["true"], "a.c": ["true"] } + // + // rather than + // + // { "a": ["b", "c"] } + // + // because the latter means EITHER b or c has to be present. + visit(path, item); + if (typeof item === 'string') { + output.push({ key: `${path}.${item}`, value: true }); + } + } + return; + } + + // object + for (const [key, value] of Object.entries(current!)) { + visit(path ? `${path}.${key}` : key, value); + } + } + + visit('', root); + + return output; +} + +// Translates a number of raw data rows to search table rows +export function mapToRows(input: Kv[], entityId: string): DbSearchRow[] { + const result: DbSearchRow[] = []; + + for (const { key: rawKey, value: rawValue } of input) { + const key = rawKey.toLocaleLowerCase('en-US'); + if (rawValue === undefined || rawValue === null) { + result.push({ entity_id: entityId, key, value: null }); + } else { + const value = String(rawValue).toLocaleLowerCase('en-US'); + if (value.length <= MAX_VALUE_LENGTH) { + result.push({ entity_id: entityId, key, value }); + } + } + } + + return result; +} + +/** + * Generates all of the search rows that are relevant for this entity. + * + * @param entityId The uid of the entity + * @param entity The entity + * @returns A list of entity search rows + */ +export function buildEntitySearch( + entityId: string, + entity: Entity, +): DbSearchRow[] { + // Visit the base structure recursively + const raw = traverse(entity); + + // Start with some special keys that are always present because you want to + // be able to easily search for null specifically + raw.push({ key: 'metadata.name', value: entity.metadata.name }); + raw.push({ key: 'metadata.namespace', value: entity.metadata.namespace }); + raw.push({ key: 'metadata.uid', value: entity.metadata.uid }); + + // Namespace not specified has the default value "default", so we want to + // match on that as well + if (!entity.metadata.namespace) { + raw.push({ key: 'metadata.namespace', value: ENTITY_DEFAULT_NAMESPACE }); + } + + // Visit relations + for (const relation of entity.relations ?? []) { + raw.push({ + key: `relations.${relation.type}`, + value: stringifyEntityRef(relation.target), + }); + } + + return mapToRows(raw, entityId); +} diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index 0aeb290224..202576fddd 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -13,27 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Entity, - EntityName, LocationSpec, Location, EntityRelationSpec, } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/config'; -import { Observable } from '@backstage/core'; // << nooo - -export interface LocationEntity { - apiVersion: 'backstage.io/v1alpha1'; - kind: 'Location'; - metadata: { - name: string; // type:target - namespace: 'default'; - }; - spec: { - location: { type: string; target: string }; - }; -} export interface LocationService { createLocation( @@ -45,20 +32,11 @@ export interface LocationService { deleteLocation(id: string): Promise; } -export type EntityMessage = - | { all: Entity[] } - | { added: Entity[]; removed: EntityName[] }; - export interface LocationStore { - // extends EntityProvider createLocation(spec: LocationSpec): Promise; listLocations(): Promise; getLocation(id: string): Promise; deleteLocation(id: string): Promise; - - location$(): Observable< - { all: Location[] } | { added: Location[]; removed: Location[] } - >; } export interface CatalogProcessingEngine { @@ -66,8 +44,17 @@ export interface CatalogProcessingEngine { stop(): Promise; } +export type EntityProviderMutation = + | { type: 'full'; entities: Entity[] } + | { type: 'delta'; added: Entity[]; removed: Entity[] }; + +export interface EntityProviderConnection { + applyMutation(mutation: EntityProviderMutation): Promise; +} + export interface EntityProvider { - entityChange$(): Observable; + getProviderName(): string; + connect(connection: EntityProviderConnection): Promise; } export type EntityProcessingRequest = { @@ -81,7 +68,7 @@ export type EntityProcessingResult = ok: true; state: Map; completedEntity: Entity; - deferredEntites: Entity[]; + deferredEntities: Entity[]; relations: EntityRelationSpec[]; errors: Error[]; } @@ -103,20 +90,27 @@ export type ProcessingItemResult = { deferredEntities: Entity[]; }; -export type AddProcessingItemRequest = { - type: 'entity' | 'provider'; - id: string; - entities: Entity[]; -}; - -export type ProccessingItem = { +export type ProcessingItem = { id: string; entity: Entity; state: Map; }; +export type ReplaceProcessingItemsRequest = + | { + sourceKey: string; + items: Entity[]; + type: 'full'; + } + | { + sourceKey: string; + added: Entity[]; + removed: Entity[]; + type: 'delta'; + }; + export interface ProcessingStateManager { setProcessingItemResult(result: ProcessingItemResult): Promise; - getNextProcessingItem(): Promise; - addProcessingItems(request: AddProcessingItemRequest): Promise; + getNextProcessingItem(): Promise; + replaceProcessingItems(request: ReplaceProcessingItemsRequest): Promise; } diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.test.ts new file mode 100644 index 0000000000..0764912f0f --- /dev/null +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.test.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { DefaultCatalogCollator } from './DefaultCatalogCollator'; + +const expectedEntities: Entity[] = [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'test-entity', + description: 'The expected description', + }, + spec: { + type: 'some-type', + lifecycle: 'experimental', + owner: 'someone', + }, + }, +]; + +jest.mock('cross-fetch', () => ({ + __esModule: true, + default: async () => { + return { + json: async () => { + return expectedEntities; + }, + }; + }, +})); + +describe('DefaultCatalogCollator', () => { + let mockDiscoveryApi: jest.Mocked; + let collator: DefaultCatalogCollator; + + beforeEach(() => { + mockDiscoveryApi = { + getBaseUrl: jest.fn().mockResolvedValueOnce('http://localhost:7000'), + getExternalBaseUrl: jest.fn(), + }; + collator = new DefaultCatalogCollator({ discovery: mockDiscoveryApi }); + }); + + it('fetches from the configured catalog service', async () => { + const documents = await collator.execute(); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog'); + expect(documents).toHaveLength(expectedEntities.length); + }); + + it('maps a returned entity to an expected CatalogEntityDocument', async () => { + const documents = await collator.execute(); + expect(documents[0]).toMatchObject({ + title: expectedEntities[0].metadata.name, + location: '/catalog/default/component/test-entity', + text: expectedEntities[0].metadata.description, + namespace: 'default', + componentType: expectedEntities[0]!.spec!.type, + lifecycle: expectedEntities[0]!.spec!.lifecycle, + owner: expectedEntities[0]!.spec!.owner, + }); + }); + + it('maps a returned entity with a custom locationTemplate', async () => { + // Provide an alternate location template. + collator = new DefaultCatalogCollator({ + discovery: mockDiscoveryApi, + locationTemplate: '/software/:name', + }); + + const documents = await collator.execute(); + expect(documents[0]).toMatchObject({ + location: '/software/test-entity', + }); + }); +}); diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts index 72bf43f8f9..068b9e36ae 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts @@ -23,13 +23,35 @@ export interface CatalogEntityDocument extends IndexableDocument { componentType: string; namespace: string; kind: string; + lifecycle: string; + owner: string; } export class DefaultCatalogCollator implements DocumentCollator { protected discovery: PluginEndpointDiscovery; + protected locationTemplate: string; - constructor(discovery: PluginEndpointDiscovery) { + constructor({ + discovery, + locationTemplate, + }: { + discovery: PluginEndpointDiscovery; + locationTemplate?: string; + }) { this.discovery = discovery; + this.locationTemplate = + locationTemplate || '/catalog/:namespace/:kind/:name'; + } + + protected applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + return formatted.toLowerCase(); } async execute() { @@ -37,17 +59,20 @@ export class DefaultCatalogCollator implements DocumentCollator { const res = await fetch(`${baseUrl}/entities`); const entities: Entity[] = await res.json(); return entities.map( - (entity): CatalogEntityDocument => { + (entity: Entity): CatalogEntityDocument => { return { title: entity.metadata.name, - // TODO: Use a config-based template approach for entity location. - location: `/catalog/${ - entity.metadata.namespace || 'default' - }/component/${entity.metadata.name}`, + location: this.applyArgsToFormat(this.locationTemplate, { + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + name: entity.metadata.name, + }), text: entity.metadata.description || '', componentType: entity.spec?.type?.toString() || 'other', namespace: entity.metadata.namespace || 'default', kind: entity.kind, + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: (entity.spec?.owner as string) || '', }; }, ); diff --git a/plugins/catalog-import/CHANGELOG.md b/plugins/catalog-import/CHANGELOG.md index 8c189a5f9b..4b0d264a52 100644 --- a/plugins/catalog-import/CHANGELOG.md +++ b/plugins/catalog-import/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-catalog-import +## 0.5.4 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.5.3 ### Patch Changes diff --git a/plugins/catalog-import/package.json b/plugins/catalog-import/package.json index 9ba32d830c..ca540fe8be 100644 --- a/plugins/catalog-import/package.json +++ b/plugins/catalog-import/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-import", - "version": "0.5.3", + "version": "0.5.4", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -32,7 +32,7 @@ "dependencies": { "@backstage/catalog-model": "^0.7.6", "@backstage/catalog-client": "^0.3.9", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/integration": "^0.5.0", "@backstage/integration-react": "^0.1.1", "@backstage/plugin-catalog-react": "^0.1.4", diff --git a/plugins/catalog/CHANGELOG.md b/plugins/catalog/CHANGELOG.md index 35e000f130..d6254a347a 100644 --- a/plugins/catalog/CHANGELOG.md +++ b/plugins/catalog/CHANGELOG.md @@ -1,5 +1,18 @@ # @backstage/plugin-catalog +## 0.5.6 + +### Patch Changes + +- 19a4dd710: Removed unused `swr` dependency. +- da546ce00: Support `gridItem` variant for `EntityLinksCard`. +- e0c9ed759: Add `if` prop to `EntityLayout.Route` to conditionally render tabs +- 1a142ae8a: Switch out the time-based personal greeting for a plain title on the catalog index page, and remove the clocks for different timezones. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.5.5 ### Patch Changes diff --git a/plugins/catalog/package.json b/plugins/catalog/package.json index 965a0b327b..4f54735611 100644 --- a/plugins/catalog/package.json +++ b/plugins/catalog/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog", - "version": "0.5.5", + "version": "0.5.6", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -32,7 +32,7 @@ "dependencies": { "@backstage/catalog-client": "^0.3.10", "@backstage/catalog-model": "^0.7.7", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/errors": "^0.1.1", "@backstage/integration": "^0.5.1", "@backstage/integration-react": "^0.1.1", diff --git a/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx b/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx index 7356ccc27b..c5b51b8334 100644 --- a/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx +++ b/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx @@ -101,4 +101,39 @@ describe('EntityLayout', () => { expect(rendered.getByText('tabbed-test-title-2')).toBeInTheDocument(); expect(rendered.queryByText('tabbed-test-content-2')).toBeInTheDocument(); }); + + it('should conditionally render tabs', async () => { + const shouldRenderTab = (e: Entity) => e.metadata.name === 'my-entity'; + const shouldNotRenderTab = (e: Entity) => e.metadata.name === 'some-entity'; + + const rendered = await renderInTestApp( + + + + +
tabbed-test-content
+ + +
tabbed-test-content-2
+ + +
tabbed-test-content-3
+ + + + , + ); + + expect(rendered.queryByText('tabbed-test-title')).toBeInTheDocument(); + expect(rendered.queryByText('tabbed-test-title-2')).not.toBeInTheDocument(); + expect(rendered.queryByText('tabbed-test-title-3')).toBeInTheDocument(); + }); }); diff --git a/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx b/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx index 6a9fa38e73..dfd27631f8 100644 --- a/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx +++ b/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx @@ -20,13 +20,14 @@ import { RELATION_OWNED_BY, } from '@backstage/catalog-model'; import { + attachComponentData, Content, Header, HeaderLabel, IconComponent, Page, Progress, - TabbedLayout, + RoutedTabs, } from '@backstage/core'; import { EntityContext, @@ -34,14 +35,69 @@ import { getEntityRelations, useEntityCompoundName, } from '@backstage/plugin-catalog-react'; -import { Box } from '@material-ui/core'; +import { Box, TabProps } from '@material-ui/core'; import { Alert } from '@material-ui/lab'; -import { default as React, useContext, useState } from 'react'; +import { + Children, + default as React, + Fragment, + isValidElement, + useContext, + useState, +} from 'react'; import { useNavigate } from 'react-router'; import { EntityContextMenu } from '../EntityContextMenu/EntityContextMenu'; import { FavouriteEntity } from '../FavouriteEntity/FavouriteEntity'; import { UnregisterEntityDialog } from '../UnregisterEntityDialog/UnregisterEntityDialog'; +type SubRoute = { + path: string; + title: string; + children: JSX.Element; + if?: (entity: Entity) => boolean; + tabProps?: TabProps; +}; + +const Route: (props: SubRoute) => null = () => null; + +// This causes all mount points that are discovered within this route to use the path of the route itself +attachComponentData(Route, 'core.gatherMountPoints', true); + +function createSubRoutesFromChildren( + childrenProps: React.ReactNode, + entity: Entity | undefined, +): SubRoute[] { + // Directly comparing child.type with Route will not work with in + // combination with react-hot-loader in storybook + // https://github.com/gaearon/react-hot-loader/issues/304 + const routeType = ( + +
+ + ).type; + + return Children.toArray(childrenProps).flatMap(child => { + if (!isValidElement(child)) { + return []; + } + + if (child.type === Fragment) { + return createSubRoutesFromChildren(child.props.children, entity); + } + + if (child.type !== routeType) { + throw new Error('Child of EntityLayout must be an EntityLayout.Route'); + } + + const { path, title, children, if: condition, tabProps } = child.props; + if (condition && entity && !condition(entity)) { + return []; + } + + return [{ path, title, children, tabProps }]; + }); +} + const EntityLayoutTitle = ({ entity, title, @@ -139,6 +195,7 @@ export const EntityLayout = ({ const { kind, namespace, name } = useEntityCompoundName(); const { entity, loading, error } = useContext(EntityContext); + const routes = createSubRoutesFromChildren(children, entity); const { headerTitle, headerType } = headerProps( kind, namespace, @@ -175,7 +232,7 @@ export const EntityLayout = ({ {loading && } - {entity && {children}} + {entity && } {error && ( @@ -192,4 +249,4 @@ export const EntityLayout = ({ ); }; -EntityLayout.Route = TabbedLayout.Route; +EntityLayout.Route = Route; diff --git a/plugins/catalog/src/components/EntityLinksCard/EntityLinksCard.tsx b/plugins/catalog/src/components/EntityLinksCard/EntityLinksCard.tsx index f7e558e285..59de0a4587 100644 --- a/plugins/catalog/src/components/EntityLinksCard/EntityLinksCard.tsx +++ b/plugins/catalog/src/components/EntityLinksCard/EntityLinksCard.tsx @@ -27,9 +27,10 @@ type Props = { /** @deprecated The entity is now grabbed from context instead */ entity?: Entity; cols?: ColumnBreakpoints | number; + variant?: 'gridItem'; }; -export const EntityLinksCard = ({ cols = undefined }: Props) => { +export const EntityLinksCard = ({ cols = undefined, variant }: Props) => { const { entity } = useEntity(); const app = useApp(); @@ -39,7 +40,7 @@ export const EntityLinksCard = ({ cols = undefined }: Props) => { const links = entity?.metadata?.links; return ( - + {!links || links.length === 0 ? ( ) : ( diff --git a/plugins/circleci/CHANGELOG.md b/plugins/circleci/CHANGELOG.md index ff01f44258..8c6d6e094d 100644 --- a/plugins/circleci/CHANGELOG.md +++ b/plugins/circleci/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-circleci +## 0.2.13 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.2.12 ### Patch Changes diff --git a/plugins/circleci/package.json b/plugins/circleci/package.json index 5c05c0ee6f..20c757eda0 100644 --- a/plugins/circleci/package.json +++ b/plugins/circleci/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-circleci", - "version": "0.2.12", + "version": "0.2.13", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -32,7 +32,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.3", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.2", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/cloudbuild/package.json b/plugins/cloudbuild/package.json index 6ad0b8bfa9..cb2a08fce1 100644 --- a/plugins/cloudbuild/package.json +++ b/plugins/cloudbuild/package.json @@ -32,7 +32,7 @@ "dependencies": { "@backstage/catalog-model": "^0.7.3", "@backstage/plugin-catalog-react": "^0.1.2", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/code-coverage-backend/CHANGELOG.md b/plugins/code-coverage-backend/CHANGELOG.md index cfc08db5c4..d7a0f136f5 100644 --- a/plugins/code-coverage-backend/CHANGELOG.md +++ b/plugins/code-coverage-backend/CHANGELOG.md @@ -1,5 +1,11 @@ # @backstage/plugin-code-coverage-backend +## 0.1.3 + +### Patch Changes + +- d47c2628b: Include migrations + ## 0.1.2 ### Patch Changes diff --git a/plugins/code-coverage-backend/package.json b/plugins/code-coverage-backend/package.json index 3ae76e3fe0..c2bebbf293 100644 --- a/plugins/code-coverage-backend/package.json +++ b/plugins/code-coverage-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-code-coverage-backend", - "version": "0.1.2", + "version": "0.1.3", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", diff --git a/plugins/code-coverage/package.json b/plugins/code-coverage/package.json index 1adc94fff7..f5c36811cc 100644 --- a/plugins/code-coverage/package.json +++ b/plugins/code-coverage/package.json @@ -22,7 +22,7 @@ "dependencies": { "@backstage/catalog-model": "^0.7.7", "@backstage/config": "^0.1.4", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/errors": "^0.1.1", "@backstage/plugin-catalog-react": "^0.1.4", "@backstage/theme": "^0.2.6", diff --git a/plugins/config-schema/package.json b/plugins/config-schema/package.json index 55f785a7a4..a017529400 100644 --- a/plugins/config-schema/package.json +++ b/plugins/config-schema/package.json @@ -21,7 +21,7 @@ }, "dependencies": { "@backstage/config": "^0.1.4", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/errors": "^0.1.1", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/cost-insights/CHANGELOG.md b/plugins/cost-insights/CHANGELOG.md index 47fa9633bf..c86a71f8c9 100644 --- a/plugins/cost-insights/CHANGELOG.md +++ b/plugins/cost-insights/CHANGELOG.md @@ -1,5 +1,16 @@ # @backstage/plugin-cost-insights +## 0.8.5 + +### Patch Changes + +- b98de52ae: Support a `name` prop for Projects for display purposes +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.8.4 ### Patch Changes diff --git a/plugins/cost-insights/package.json b/plugins/cost-insights/package.json index c68fdde509..39e3fe1791 100644 --- a/plugins/cost-insights/package.json +++ b/plugins/cost-insights/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-cost-insights", - "version": "0.8.4", + "version": "0.8.5", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -31,7 +31,7 @@ }, "dependencies": { "@backstage/config": "^0.1.3", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/cost-insights/src/components/ProjectSelect/ProjectSelect.test.tsx b/plugins/cost-insights/src/components/ProjectSelect/ProjectSelect.test.tsx index 1bd57a9eed..1820b9e35a 100644 --- a/plugins/cost-insights/src/components/ProjectSelect/ProjectSelect.test.tsx +++ b/plugins/cost-insights/src/components/ProjectSelect/ProjectSelect.test.tsx @@ -23,7 +23,7 @@ import { renderInTestApp } from '@backstage/test-utils'; const mockProjects = [ { id: 'project1' }, - { id: 'project2' }, + { id: 'project2', name: 'Project 2' }, { id: 'project3' }, ]; @@ -56,7 +56,9 @@ describe('', () => { await waitFor(() => rendered.getByTestId('option-all')); mockProjects.forEach(project => - expect(rendered.getByText(project.id)).toBeInTheDocument(), + expect( + rendered.getByText(project.name ?? project.id), + ).toBeInTheDocument(), ); }); }); diff --git a/plugins/cost-insights/src/components/ProjectSelect/ProjectSelect.tsx b/plugins/cost-insights/src/components/ProjectSelect/ProjectSelect.tsx index bc34eb9c8d..25fdc0c171 100644 --- a/plugins/cost-insights/src/components/ProjectSelect/ProjectSelect.tsx +++ b/plugins/cost-insights/src/components/ProjectSelect/ProjectSelect.tsx @@ -33,7 +33,9 @@ export const ProjectSelect = ({ const projectOptions = projects .filter(p => p.id) - .sort((a, b) => (a.id as string).localeCompare(b.id as string)); + .sort((a, b) => + ((a.name ?? a.id) as string).localeCompare((b.name ?? b.id) as string), + ); const handleOnChange = (e: React.ChangeEvent<{ value: unknown }>) => { onSelect(e.target.value as string); @@ -41,9 +43,10 @@ export const ProjectSelect = ({ const renderValue = (value: unknown) => { const proj = value as string; + const projectObj = projects.find(p => p.id === proj); return ( - {proj === 'all' ? 'All Projects' : proj} + {proj === 'all' ? 'All Projects' : projectObj?.name ?? proj} ); }; @@ -52,7 +55,7 @@ export const ProjectSelect = ({ diff --git a/plugins/cost-insights/src/types/Project.ts b/plugins/cost-insights/src/types/Project.ts index 24259f6910..6ac61f1fb7 100644 --- a/plugins/cost-insights/src/types/Project.ts +++ b/plugins/cost-insights/src/types/Project.ts @@ -16,4 +16,5 @@ export interface Project { id: string; + name?: string; } diff --git a/plugins/explore/CHANGELOG.md b/plugins/explore/CHANGELOG.md index bebf6a8c44..78275577f5 100644 --- a/plugins/explore/CHANGELOG.md +++ b/plugins/explore/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-explore +## 0.3.4 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.3.3 ### Patch Changes diff --git a/plugins/explore/package.json b/plugins/explore/package.json index 7c2eb5c90d..be61bc1fd9 100644 --- a/plugins/explore/package.json +++ b/plugins/explore/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-explore", - "version": "0.3.3", + "version": "0.3.4", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -31,7 +31,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.5", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.4", "@backstage/plugin-explore-react": "^0.0.4", "@backstage/theme": "^0.2.6", diff --git a/plugins/fossa/CHANGELOG.md b/plugins/fossa/CHANGELOG.md index fe916804d2..bd7af91dca 100644 --- a/plugins/fossa/CHANGELOG.md +++ b/plugins/fossa/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-fossa +## 0.2.6 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.2.5 ### Patch Changes diff --git a/plugins/fossa/package.json b/plugins/fossa/package.json index 23f9cfde74..ba7e1a381e 100644 --- a/plugins/fossa/package.json +++ b/plugins/fossa/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-fossa", - "version": "0.2.5", + "version": "0.2.6", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -32,7 +32,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.3", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/errors": "^0.1.1", "@backstage/plugin-catalog-react": "^0.1.1", "@backstage/theme": "^0.2.6", diff --git a/plugins/gcp-projects/package.json b/plugins/gcp-projects/package.json index 509576cf2b..0930c791e4 100644 --- a/plugins/gcp-projects/package.json +++ b/plugins/gcp-projects/package.json @@ -30,7 +30,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/github-actions/CHANGELOG.md b/plugins/github-actions/CHANGELOG.md index a2c6e6bf1f..6896a7f877 100644 --- a/plugins/github-actions/CHANGELOG.md +++ b/plugins/github-actions/CHANGELOG.md @@ -1,5 +1,16 @@ # @backstage/plugin-github-actions +## 0.4.4 + +### Patch Changes + +- 4c42ecca2: Wrap EmptyState in Card +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.4.3 ### Patch Changes diff --git a/plugins/github-actions/package.json b/plugins/github-actions/package.json index 6a95803d6f..2c26061cf3 100644 --- a/plugins/github-actions/package.json +++ b/plugins/github-actions/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-github-actions", - "version": "0.4.3", + "version": "0.4.4", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -34,7 +34,7 @@ "dependencies": { "@backstage/catalog-model": "^0.7.5", "@backstage/plugin-catalog-react": "^0.1.4", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/integration": "^0.5.1", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx b/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx index 1126c2a2a0..f0366e5590 100644 --- a/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx +++ b/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx @@ -16,21 +16,21 @@ import { Entity } from '@backstage/catalog-model'; import { configApiRef, - EmptyState, errorApiRef, InfoCard, InfoCardVariants, + Link, Table, useApi, } from '@backstage/core'; import { readGitHubIntegrationConfigs } from '@backstage/integration'; import { useEntity } from '@backstage/plugin-catalog-react'; -import { Button, Link } from '@material-ui/core'; import React, { useEffect } from 'react'; import { generatePath, Link as RouterLink } from 'react-router-dom'; import { GITHUB_ACTIONS_ANNOTATION } from '../useProjectName'; import { useWorkflowRuns } from '../useWorkflowRuns'; import { WorkflowRunStatus } from '../WorkflowRunStatus'; +import { Typography } from '@material-ui/core'; const firstLine = (message: string): string => message.split('\n')[0]; @@ -74,54 +74,53 @@ export const RecentWorkflowRunsCard = ({ const githubHost = hostname || 'github.com'; - return !runs.length ? ( - - Create new Workflow - - } - /> - ) : ( + return ( - ( - - {firstLine(data.message)} - - ), - }, - { title: 'Branch', field: 'source.branchName' }, - { title: 'Status', field: 'status', render: WorkflowRunStatus }, - ]} - data={runs} - /> + {!runs.length ? ( +
+ + This component has GitHub Actions enabled, but no workflows were + found. + + + + Create a new workflow + + +
+ ) : ( +
( + + {firstLine(data.message)} + + ), + }, + { title: 'Branch', field: 'source.branchName' }, + { title: 'Status', field: 'status', render: WorkflowRunStatus }, + ]} + data={runs} + /> + )} ); }; diff --git a/plugins/github-deployments/CHANGELOG.md b/plugins/github-deployments/CHANGELOG.md index e7f849ef2e..23c8943349 100644 --- a/plugins/github-deployments/CHANGELOG.md +++ b/plugins/github-deployments/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-github-deployments +## 0.1.4 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.1.3 ### Patch Changes diff --git a/plugins/github-deployments/package.json b/plugins/github-deployments/package.json index 12a36c160e..049e02de4c 100644 --- a/plugins/github-deployments/package.json +++ b/plugins/github-deployments/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-github-deployments", - "version": "0.1.3", + "version": "0.1.4", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,7 +21,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.6", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.3", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/gitops-profiles/package.json b/plugins/gitops-profiles/package.json index ae614fe504..d1897b018c 100644 --- a/plugins/gitops-profiles/package.json +++ b/plugins/gitops-profiles/package.json @@ -31,7 +31,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/graphiql/CHANGELOG.md b/plugins/graphiql/CHANGELOG.md index 67e39d6600..681005b13f 100644 --- a/plugins/graphiql/CHANGELOG.md +++ b/plugins/graphiql/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-graphiql +## 0.2.10 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.2.9 ### Patch Changes diff --git a/plugins/graphiql/package.json b/plugins/graphiql/package.json index 4aa26587ad..9ce1603b57 100644 --- a/plugins/graphiql/package.json +++ b/plugins/graphiql/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/plugin-graphiql", "description": "Backstage plugin for browsing GraphQL APIs", - "version": "0.2.9", + "version": "0.2.10", "private": false, "publishConfig": { "access": "public", @@ -31,7 +31,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/jenkins/CHANGELOG.md b/plugins/jenkins/CHANGELOG.md index ab57d71f61..ac734e9a95 100644 --- a/plugins/jenkins/CHANGELOG.md +++ b/plugins/jenkins/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-jenkins +## 0.4.2 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.4.1 ### Patch Changes diff --git a/plugins/jenkins/package.json b/plugins/jenkins/package.json index edee4bc2ee..d1cabd2143 100644 --- a/plugins/jenkins/package.json +++ b/plugins/jenkins/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-jenkins", - "version": "0.4.1", + "version": "0.4.2", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -32,7 +32,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.3", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.3", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/kafka/package.json b/plugins/kafka/package.json index a7295469d6..0963d3178f 100644 --- a/plugins/kafka/package.json +++ b/plugins/kafka/package.json @@ -21,7 +21,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.4", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.1", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/kubernetes/package.json b/plugins/kubernetes/package.json index 03013e1480..8d21c79a24 100644 --- a/plugins/kubernetes/package.json +++ b/plugins/kubernetes/package.json @@ -33,7 +33,7 @@ "dependencies": { "@backstage/catalog-model": "^0.7.4", "@backstage/config": "^0.1.4", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.3", "@backstage/plugin-kubernetes-backend": "^0.3.2", "@backstage/theme": "^0.2.6", diff --git a/plugins/lighthouse/CHANGELOG.md b/plugins/lighthouse/CHANGELOG.md index 26eb041442..cb2cb5b9d1 100644 --- a/plugins/lighthouse/CHANGELOG.md +++ b/plugins/lighthouse/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-lighthouse +## 0.2.15 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.2.14 ### Patch Changes diff --git a/plugins/lighthouse/package.json b/plugins/lighthouse/package.json index 45980df00c..2fce507756 100644 --- a/plugins/lighthouse/package.json +++ b/plugins/lighthouse/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-lighthouse", - "version": "0.2.14", + "version": "0.2.15", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -33,7 +33,7 @@ "dependencies": { "@backstage/catalog-model": "^0.7.3", "@backstage/config": "^0.1.4", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.2", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/newrelic/package.json b/plugins/newrelic/package.json index 0951d972c2..2dbb2a7eb8 100644 --- a/plugins/newrelic/package.json +++ b/plugins/newrelic/package.json @@ -31,7 +31,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/org/package.json b/plugins/org/package.json index 5bed02e9c8..9ad4704630 100644 --- a/plugins/org/package.json +++ b/plugins/org/package.json @@ -21,7 +21,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.6", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/core-api": "^0.2.16", "@backstage/plugin-catalog-react": "^0.1.4", "@backstage/theme": "^0.2.6", diff --git a/plugins/pagerduty/CHANGELOG.md b/plugins/pagerduty/CHANGELOG.md index 0cd9d0cfb8..b7fc7fec40 100644 --- a/plugins/pagerduty/CHANGELOG.md +++ b/plugins/pagerduty/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-pagerduty +## 0.3.3 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.3.2 ### Patch Changes diff --git a/plugins/pagerduty/package.json b/plugins/pagerduty/package.json index c0e3afec34..821b89a5b1 100644 --- a/plugins/pagerduty/package.json +++ b/plugins/pagerduty/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-pagerduty", - "version": "0.3.2", + "version": "0.3.3", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -31,7 +31,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.3", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.1", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/proxy-backend/src/service/router.test.ts b/plugins/proxy-backend/src/service/router.test.ts index 6c47043602..571de96ac4 100644 --- a/plugins/proxy-backend/src/service/router.test.ts +++ b/plugins/proxy-backend/src/service/router.test.ts @@ -59,8 +59,8 @@ describe('buildMiddleware', () => { mockCreateProxyMiddleware.mockClear(); }); - it('accepts strings', async () => { - buildMiddleware('/api/', logger, 'test', 'http://mocked'); + it('accepts strings prefixed by /', async () => { + buildMiddleware('/proxy', logger, '/test', 'http://mocked'); expect(createProxyMiddleware).toHaveBeenCalledTimes(1); @@ -74,13 +74,53 @@ describe('buildMiddleware', () => { expect(filter('', { method: 'PATCH', headers: {} })).toBe(true); expect(filter('', { method: 'DELETE', headers: {} })).toBe(true); - expect(fullConfig.pathRewrite).toEqual({ '^/api/test/': '/' }); + expect(fullConfig.pathRewrite).toEqual({ '^/proxy/test/': '/' }); + expect(fullConfig.changeOrigin).toBe(true); + expect(fullConfig.logProvider!(logger)).toBe(logger); + }); + + it('accepts routes not prefixed with / when path is not suffixed with /', async () => { + buildMiddleware('/proxy', logger, 'test', 'http://mocked'); + + expect(createProxyMiddleware).toHaveBeenCalledTimes(1); + + const [filter, fullConfig] = mockCreateProxyMiddleware.mock.calls[0] as [ + (pathname: string, req: Partial) => boolean, + ProxyMiddlewareConfig, + ]; + expect(filter('', { method: 'GET', headers: {} })).toBe(true); + expect(filter('', { method: 'POST', headers: {} })).toBe(true); + expect(filter('', { method: 'PUT', headers: {} })).toBe(true); + expect(filter('', { method: 'PATCH', headers: {} })).toBe(true); + expect(filter('', { method: 'DELETE', headers: {} })).toBe(true); + + expect(fullConfig.pathRewrite).toEqual({ '^/proxy/test/': '/' }); + expect(fullConfig.changeOrigin).toBe(true); + expect(fullConfig.logProvider!(logger)).toBe(logger); + }); + + it('accepts routes prefixed with / when path is suffixed with /', async () => { + buildMiddleware('/proxy/', logger, '/test', 'http://mocked'); + + expect(createProxyMiddleware).toHaveBeenCalledTimes(1); + + const [filter, fullConfig] = mockCreateProxyMiddleware.mock.calls[0] as [ + (pathname: string, req: Partial) => boolean, + ProxyMiddlewareConfig, + ]; + expect(filter('', { method: 'GET', headers: {} })).toBe(true); + expect(filter('', { method: 'POST', headers: {} })).toBe(true); + expect(filter('', { method: 'PUT', headers: {} })).toBe(true); + expect(filter('', { method: 'PATCH', headers: {} })).toBe(true); + expect(filter('', { method: 'DELETE', headers: {} })).toBe(true); + + expect(fullConfig.pathRewrite).toEqual({ '^/proxy/test/': '/' }); expect(fullConfig.changeOrigin).toBe(true); expect(fullConfig.logProvider!(logger)).toBe(logger); }); it('limits allowedMethods', async () => { - buildMiddleware('/api/', logger, 'test', { + buildMiddleware('/proxy', logger, '/test', { target: 'http://mocked', allowedMethods: ['GET', 'DELETE'], }); @@ -97,13 +137,13 @@ describe('buildMiddleware', () => { expect(filter('', { method: 'PATCH', headers: {} })).toBe(false); expect(filter('', { method: 'DELETE', headers: {} })).toBe(true); - expect(fullConfig.pathRewrite).toEqual({ '^/api/test/': '/' }); + expect(fullConfig.pathRewrite).toEqual({ '^/proxy/test/': '/' }); expect(fullConfig.changeOrigin).toBe(true); expect(fullConfig.logProvider!(logger)).toBe(logger); }); it('permits default headers', async () => { - buildMiddleware('/api/', logger, 'test', { + buildMiddleware('/proxy', logger, '/test', { target: 'http://mocked', }); @@ -143,7 +183,7 @@ describe('buildMiddleware', () => { }); it('permits default and configured headers', async () => { - buildMiddleware('/api/', logger, 'test', { + buildMiddleware('/proxy', logger, '/test', { target: 'http://mocked', headers: { Authorization: 'my-token', @@ -176,7 +216,7 @@ describe('buildMiddleware', () => { }); it('permits configured headers', async () => { - buildMiddleware('/api/', logger, 'test', { + buildMiddleware('/proxy', logger, '/test', { target: 'http://mocked', allowedHeaders: ['authorization', 'cookie'], }); @@ -208,7 +248,7 @@ describe('buildMiddleware', () => { }); it('responds default headers', async () => { - buildMiddleware('/api/', logger, 'test', { + buildMiddleware('/proxy', logger, '/test', { target: 'http://mocked', }); @@ -251,7 +291,7 @@ describe('buildMiddleware', () => { }); it('responds configured headers', async () => { - buildMiddleware('/api/', logger, 'test', { + buildMiddleware('/proxy', logger, '/test', { target: 'http://mocked', allowedHeaders: ['set-cookie'], }); @@ -282,10 +322,10 @@ describe('buildMiddleware', () => { it('rejects malformed target URLs', async () => { expect(() => - buildMiddleware('/api/', logger, 'test', 'backstage.io'), + buildMiddleware('/proxy', logger, '/test', 'backstage.io'), ).toThrowError(/Proxy target is not a valid URL/); expect(() => - buildMiddleware('/api/', logger, 'test', { target: 'backstage.io' }), + buildMiddleware('/proxy', logger, '/test', { target: 'backstage.io' }), ).toThrowError(/Proxy target is not a valid URL/); }); }); diff --git a/plugins/proxy-backend/src/service/router.ts b/plugins/proxy-backend/src/service/router.ts index 5c306de966..71ed82c23b 100644 --- a/plugins/proxy-backend/src/service/router.ts +++ b/plugins/proxy-backend/src/service/router.ts @@ -77,10 +77,24 @@ export function buildMiddleware( `Proxy target is not a valid URL: ${fullConfig.target ?? ''}`, ); } + // Default is to do a path rewrite that strips out the proxy's path prefix // and the rest of the route. if (fullConfig.pathRewrite === undefined) { - const routeWithSlash = route.endsWith('/') ? route : `${route}/`; + let routeWithSlash = route.endsWith('/') ? route : `${route}/`; + + if (!pathPrefix.endsWith('/') && !routeWithSlash.startsWith('/')) { + // Need to insert a / between pathPrefix and routeWithSlash + routeWithSlash = `/${routeWithSlash}`; + } else if (pathPrefix.endsWith('/') && routeWithSlash.startsWith('/')) { + // Never expect this to happen at this point in time as + // pathPrefix is set using `getExternalBaseUrl` which "Returns the + // external HTTP base backend URL for a given plugin, + // **without a trailing slash.**". But in case this changes in future, we + // need to drop a / on either pathPrefix or routeWithSlash + routeWithSlash = routeWithSlash.substring(1); + } + fullConfig.pathRewrite = { [`^${pathPrefix}${routeWithSlash}`]: '/', }; diff --git a/plugins/register-component/CHANGELOG.md b/plugins/register-component/CHANGELOG.md index 424887263c..3634972e9a 100644 --- a/plugins/register-component/CHANGELOG.md +++ b/plugins/register-component/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-register-component +## 0.2.14 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.2.13 ### Patch Changes diff --git a/plugins/register-component/package.json b/plugins/register-component/package.json index 2870ef79f1..38d1e3202e 100644 --- a/plugins/register-component/package.json +++ b/plugins/register-component/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-register-component", - "version": "0.2.13", + "version": "0.2.14", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -31,7 +31,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.5", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.4", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/rollbar/CHANGELOG.md b/plugins/rollbar/CHANGELOG.md index 2e9bc946a2..1d8c2c399e 100644 --- a/plugins/rollbar/CHANGELOG.md +++ b/plugins/rollbar/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-rollbar +## 0.3.4 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.3.3 ### Patch Changes diff --git a/plugins/rollbar/package.json b/plugins/rollbar/package.json index 6d654866af..64ea010b54 100644 --- a/plugins/rollbar/package.json +++ b/plugins/rollbar/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-rollbar", - "version": "0.3.3", + "version": "0.3.4", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -32,7 +32,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.3", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.1", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/scaffolder-backend/CHANGELOG.md b/plugins/scaffolder-backend/CHANGELOG.md index cb1855e041..f591568d44 100644 --- a/plugins/scaffolder-backend/CHANGELOG.md +++ b/plugins/scaffolder-backend/CHANGELOG.md @@ -1,5 +1,11 @@ # @backstage/plugin-scaffolder-backend +## 0.10.1 + +### Patch Changes + +- a1783f306: Added the `nebula-preview` preview to `Octokit` for repository visibility. + ## 0.10.0 ### Minor Changes diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 729ef58d99..355bed4de0 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-scaffolder-backend", - "version": "0.10.0", + "version": "0.10.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", diff --git a/plugins/scaffolder/CHANGELOG.md b/plugins/scaffolder/CHANGELOG.md index fee76f8775..1d0bbcede9 100644 --- a/plugins/scaffolder/CHANGELOG.md +++ b/plugins/scaffolder/CHANGELOG.md @@ -1,5 +1,17 @@ # @backstage/plugin-scaffolder +## 0.9.2 + +### Patch Changes + +- f6efa71ee: Enable starred templates on Scaffolder frontend +- 19a4dd710: Removed unused `swr` dependency. +- 23769512a: Support `anyOf`, `oneOf` and `allOf` schemas in the scaffolder template. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.9.1 ### Patch Changes diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 2b4feecb7b..32229ccbc6 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-scaffolder", - "version": "0.9.1", + "version": "0.9.2", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -33,7 +33,7 @@ "@backstage/catalog-client": "^0.3.10", "@backstage/catalog-model": "^0.7.7", "@backstage/config": "^0.1.4", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/integration": "^0.5.1", "@backstage/integration-react": "^0.1.1", "@backstage/plugin-catalog-react": "^0.1.4", @@ -44,7 +44,7 @@ "@rjsf/core": "^2.4.0", "@rjsf/material-ui": "^2.4.0", "classnames": "^2.2.6", - "json-schema": "^0.2.5", + "json-schema": "^0.3.0", "git-url-parse": "^11.4.4", "humanize-duration": "^3.25.1", "immer": "^9.0.1", diff --git a/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx b/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx new file mode 100644 index 0000000000..ae6057beff --- /dev/null +++ b/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, { ComponentProps, useMemo } from 'react'; +import { useStarredEntities } from '@backstage/plugin-catalog-react'; +import { IconButton, makeStyles, Tooltip, withStyles } from '@material-ui/core'; +import StarBorder from '@material-ui/icons/StarBorder'; +import Star from '@material-ui/icons/Star'; +import { Entity } from '@backstage/catalog-model'; + +type Props = ComponentProps & { entity: Entity }; + +const YellowStar = withStyles({ + root: { + color: '#f3ba37', + }, +})(Star); + +const WhiteBorderStar = withStyles({ + root: { + color: '#ffffff', + }, +})(StarBorder); + +const useStyles = makeStyles(theme => ({ + starButton: { + position: 'absolute', + top: theme.spacing(0.5), + right: theme.spacing(0.5), + padding: '0.25rem', + }, +})); + +export const favouriteTemplateTooltip = (isStarred: boolean) => + isStarred ? 'Remove from favorites' : 'Add to favorites'; + +export const favouriteTemplateIcon = (isStarred: boolean) => + isStarred ? : ; + +/** + * IconButton for showing if a current entity is starred and adding/removing it from the favourite entities + * @param props MaterialUI IconButton props extended by required `entity` prop + */ +export const FavouriteTemplate = (props: Props) => { + const classes = useStyles(); + const { toggleStarredEntity, isStarredEntity } = useStarredEntities(); + const isStarred = useMemo(() => isStarredEntity(props.entity), [ + isStarredEntity, + props.entity, + ]); + return ( + toggleStarredEntity(props.entity)} + > + + {favouriteTemplateIcon(isStarred)} + + + ); +}; diff --git a/plugins/scaffolder/src/components/MultistepJsonForm/schema.test.ts b/plugins/scaffolder/src/components/MultistepJsonForm/schema.test.ts index b83725c2bb..64dd705952 100644 --- a/plugins/scaffolder/src/components/MultistepJsonForm/schema.test.ts +++ b/plugins/scaffolder/src/components/MultistepJsonForm/schema.test.ts @@ -70,4 +70,220 @@ describe('transformSchemaToProps', () => { uiSchema: expectedUiSchema, }); }); + + it('transforms schema with anyOf fields', () => { + const inputSchema = { + type: 'object', + anyOf: [ + { + properties: { + field3: { + type: 'string', + default: 'Value 1', + 'ui:readonly': true, + }, + }, + }, + { + properties: { + field3: { + type: 'string', + default: 'Value 2', + 'ui:readonly': true, + }, + }, + }, + ], + oneOf: [ + { + properties: { + field4: { + type: 'string', + default: 'Value 1', + 'ui:readonly': true, + }, + }, + }, + ], + allOf: [ + { + properties: { + field5: { + type: 'string', + default: 'Value 1', + 'ui:readonly': true, + }, + }, + }, + ], + properties: { + field1: { + type: 'object', + anyOf: [ + { + properties: { + field3: { + type: 'string', + default: 'Value 1', + 'ui:readonly': true, + }, + }, + }, + { + properties: { + field3: { + type: 'string', + default: 'Value 2', + 'ui:readonly': true, + }, + }, + }, + ], + oneOf: [ + { + properties: { + field4: { + type: 'string', + default: 'Value 1', + 'ui:readonly': true, + }, + }, + }, + ], + allOf: [ + { + properties: { + field5: { + type: 'string', + default: 'Value 1', + 'ui:readonly': true, + }, + }, + }, + ], + }, + field2: { + type: 'string', + 'ui:derp': 'xerp', + }, + }, + }; + const expectedSchema = { + type: 'object', + anyOf: [ + { + properties: { + field3: { + type: 'string', + default: 'Value 1', + }, + }, + }, + { + properties: { + field3: { + type: 'string', + default: 'Value 2', + }, + }, + }, + ], + oneOf: [ + { + properties: { + field4: { + type: 'string', + default: 'Value 1', + }, + }, + }, + ], + allOf: [ + { + properties: { + field5: { + type: 'string', + default: 'Value 1', + }, + }, + }, + ], + properties: { + field1: { + type: 'object', + anyOf: [ + { + properties: { + field3: { + type: 'string', + default: 'Value 1', + }, + }, + }, + { + properties: { + field3: { + type: 'string', + default: 'Value 2', + }, + }, + }, + ], + oneOf: [ + { + properties: { + field4: { + type: 'string', + default: 'Value 1', + }, + }, + }, + ], + allOf: [ + { + properties: { + field5: { + type: 'string', + default: 'Value 1', + }, + }, + }, + ], + }, + field2: { + type: 'string', + }, + }, + }; + const expectedUiSchema = { + field3: { + 'ui:readonly': true, + }, + field4: { + 'ui:readonly': true, + }, + field5: { + 'ui:readonly': true, + }, + field1: { + field3: { + 'ui:readonly': true, + }, + field4: { + 'ui:readonly': true, + }, + field5: { + 'ui:readonly': true, + }, + }, + field2: { + 'ui:derp': 'xerp', + }, + }; + + expect(transformSchemaToProps(inputSchema)).toEqual({ + schema: expectedSchema, + uiSchema: expectedUiSchema, + }); + }); }); diff --git a/plugins/scaffolder/src/components/MultistepJsonForm/schema.ts b/plugins/scaffolder/src/components/MultistepJsonForm/schema.ts index e591589bd8..b4aa3e309d 100644 --- a/plugins/scaffolder/src/components/MultistepJsonForm/schema.ts +++ b/plugins/scaffolder/src/components/MultistepJsonForm/schema.ts @@ -26,7 +26,7 @@ function extractUiSchema(schema: JsonObject, uiSchema: JsonObject) { return; } - const { properties } = schema; + const { properties, anyOf, oneOf, allOf } = schema; for (const propName in schema) { if (!schema.hasOwnProperty(propName)) { @@ -39,22 +39,47 @@ function extractUiSchema(schema: JsonObject, uiSchema: JsonObject) { } } - if (!isObject(properties)) { - return; + if (isObject(properties)) { + for (const propName in properties) { + if (!properties.hasOwnProperty(propName)) { + continue; + } + + const schemaNode = properties[propName]; + if (!isObject(schemaNode)) { + continue; + } + const innerUiSchema = {}; + uiSchema[propName] = innerUiSchema; + extractUiSchema(schemaNode, innerUiSchema); + } } - for (const propName in properties) { - if (!properties.hasOwnProperty(propName)) { - continue; + if (Array.isArray(anyOf)) { + for (const schemaNode of anyOf) { + if (!isObject(schemaNode)) { + continue; + } + extractUiSchema(schemaNode, uiSchema); } + } - const schemaNode = properties[propName]; - if (!isObject(schemaNode)) { - continue; + if (Array.isArray(oneOf)) { + for (const schemaNode of oneOf) { + if (!isObject(schemaNode)) { + continue; + } + extractUiSchema(schemaNode, uiSchema); + } + } + + if (Array.isArray(allOf)) { + for (const schemaNode of allOf) { + if (!isObject(schemaNode)) { + continue; + } + extractUiSchema(schemaNode, uiSchema); } - const innerUiSchema = {}; - uiSchema[propName] = innerUiSchema; - extractUiSchema(schemaNode, innerUiSchema); } } diff --git a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx index 68bf588d01..4a482402f8 100644 --- a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx +++ b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx @@ -39,7 +39,7 @@ import { ResultsFilter } from '../ResultsFilter/ResultsFilter'; import { ScaffolderFilter } from '../ScaffolderFilter'; import { ButtonGroup } from '../ScaffolderFilter/ScaffolderFilter'; import SearchToolbar from '../SearchToolbar/SearchToolbar'; -import { TemplateCard, TemplateCardProps } from '../TemplateCard'; +import { TemplateCard } from '../TemplateCard'; import { registerComponentRouteRef } from '../../routes'; const useStyles = makeStyles(theme => ({ @@ -51,19 +51,6 @@ const useStyles = makeStyles(theme => ({ }, })); -const getTemplateCardProps = ( - template: TemplateEntityV1alpha1, -): TemplateCardProps & { key: string } => { - return { - key: template.metadata.uid!, - name: template.metadata.name, - title: `${(template.metadata.title || template.metadata.name) ?? ''}`, - type: template.spec.type ?? '', - description: template.metadata.description ?? '-', - tags: (template.metadata?.tags as string[]) ?? [], - }; -}; - export const ScaffolderPageContents = () => { const styles = useStyles(); const { @@ -188,7 +175,7 @@ export const ScaffolderPageContents = () => { {matchingEntities && matchingEntities?.length > 0 && matchingEntities.map(template => ( - + ))} diff --git a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx index 2ba83fdaae..8c64d981ad 100644 --- a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx +++ b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx @@ -28,8 +28,13 @@ import { import React from 'react'; import { generatePath } from 'react-router'; import { rootRouteRef } from '../../routes'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { FavouriteTemplate } from '../FavouriteTemplate/FavouriteTemplate'; const useStyles = makeStyles({ + cardHeader: { + position: 'relative', + }, title: { backgroundImage: ({ backgroundImage }: any) => backgroundImage, }, @@ -43,6 +48,10 @@ const useStyles = makeStyles({ }); export type TemplateCardProps = { + template: TemplateEntityV1alpha1; +}; + +type TemplateProps = { description: string; tags: string[]; title: string; @@ -50,42 +59,55 @@ export type TemplateCardProps = { name: string; }; -export const TemplateCard = ({ - description, - tags, - title, - type, - name, -}: TemplateCardProps) => { +const getTemplateCardProps = ( + template: TemplateEntityV1alpha1, +): TemplateProps & { key: string } => { + return { + key: template.metadata.uid!, + name: template.metadata.name, + title: `${(template.metadata.title || template.metadata.name) ?? ''}`, + type: template.spec.type ?? '', + description: template.metadata.description ?? '-', + tags: (template.metadata?.tags as string[]) ?? [], + }; +}; + +export const TemplateCard = ({ template }: TemplateCardProps) => { const backstageTheme = useTheme(); const rootLink = useRouteRef(rootRouteRef); + const templateProps = getTemplateCardProps(template); - const themeId = pageTheme[type] ? type : 'other'; + const themeId = pageTheme[templateProps.type] ? templateProps.type : 'other'; const theme = backstageTheme.getPageTheme({ themeId }); const classes = useStyles({ backgroundImage: theme.backgroundImage }); const href = generatePath(`${rootLink()}/templates/:templateName`, { - templateName: name, + templateName: templateProps.name, }); return ( - + + - {tags?.map(tag => ( + {templateProps.tags?.map(tag => ( ))} - {description} + {templateProps.description} - diff --git a/plugins/search-backend-node/CHANGELOG.md b/plugins/search-backend-node/CHANGELOG.md index e7a9d9aa0a..da49bd0726 100644 --- a/plugins/search-backend-node/CHANGELOG.md +++ b/plugins/search-backend-node/CHANGELOG.md @@ -1,5 +1,11 @@ # @backstage/plugin-search-backend-node +## 0.1.4 + +### Patch Changes + +- e1e757569: Introduced Scheduler which is responsible for adding new tasks to a schedule together with it's interval timer as well as starting and stopping the scheduler processes. + ## 0.1.3 ### Patch Changes diff --git a/plugins/search-backend-node/package.json b/plugins/search-backend-node/package.json index b5d13af01c..e39fbb5579 100644 --- a/plugins/search-backend-node/package.json +++ b/plugins/search-backend-node/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-search-backend-node", - "version": "0.1.3", + "version": "0.1.4", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", diff --git a/plugins/search/CHANGELOG.md b/plugins/search/CHANGELOG.md index 35f3ee97c1..b4bcd1f855 100644 --- a/plugins/search/CHANGELOG.md +++ b/plugins/search/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-search +## 0.3.5 + +### Patch Changes + +- dcd54c7cd: Use `RouteRef` to generate path to search page. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.3.4 ### Patch Changes diff --git a/plugins/search/package.json b/plugins/search/package.json index c7ab201efe..2b9db3444e 100644 --- a/plugins/search/package.json +++ b/plugins/search/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-search", - "version": "0.3.4", + "version": "0.3.5", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -29,7 +29,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/catalog-model": "^0.7.3", "@backstage/plugin-catalog-react": "^0.1.2", "@backstage/search-common": "^0.1.1", diff --git a/plugins/sentry/CHANGELOG.md b/plugins/sentry/CHANGELOG.md index dc45bcae60..261972384a 100644 --- a/plugins/sentry/CHANGELOG.md +++ b/plugins/sentry/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-sentry +## 0.3.9 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.3.8 ### Patch Changes diff --git a/plugins/sentry/package.json b/plugins/sentry/package.json index 9908af2e24..e34d8bcef9 100644 --- a/plugins/sentry/package.json +++ b/plugins/sentry/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-sentry", - "version": "0.3.8", + "version": "0.3.9", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -32,7 +32,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.3", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.1", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/sonarqube/CHANGELOG.md b/plugins/sonarqube/CHANGELOG.md index 0ed86737dd..48b9292791 100644 --- a/plugins/sonarqube/CHANGELOG.md +++ b/plugins/sonarqube/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-sonarqube +## 0.1.17 + +### Patch Changes + +- c614ede9a: Updated README to have up-to-date install instructions. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.1.16 ### Patch Changes diff --git a/plugins/sonarqube/package.json b/plugins/sonarqube/package.json index 97243bb930..c31b99bac8 100644 --- a/plugins/sonarqube/package.json +++ b/plugins/sonarqube/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-sonarqube", - "version": "0.1.16", + "version": "0.1.17", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -34,7 +34,7 @@ "dependencies": { "@backstage/catalog-model": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.3", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/splunk-on-call/package.json b/plugins/splunk-on-call/package.json index ecaf21f8eb..9e93004bf5 100644 --- a/plugins/splunk-on-call/package.json +++ b/plugins/splunk-on-call/package.json @@ -31,7 +31,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.4", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.1", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", diff --git a/plugins/tech-radar/README.md b/plugins/tech-radar/README.md index 21f90ec432..7f97f88ed4 100644 --- a/plugins/tech-radar/README.md +++ b/plugins/tech-radar/README.md @@ -18,7 +18,7 @@ It serves and scales well for teams and companies of all sizes that want to have The Tech Radar can be used in two ways: -- **Simple (Recommended)** - This gives you an out-of-the-box Tech Radar experience. It lives on the `/tech-radar` URL of your Backstage installation, and you can set a variety of configuration directly in your `apis.ts`. +- **Simple (Recommended)** - This gives you an out-of-the-box Tech Radar experience. It lives on the `/tech-radar` URL of your Backstage installation. - **Advanced** - This gives you the React UI component directly. It enables you to insert the Radar on your own layout or page for a more customized feel. ### Install @@ -26,6 +26,7 @@ The Tech Radar can be used in two ways: For either simple or advanced installations, you'll need to add the dependency using Yarn: ```sh +cd packages/app yarn add @backstage/plugin-tech-radar ``` @@ -34,17 +35,16 @@ yarn add @backstage/plugin-tech-radar Modify your app routes to include the Router component exported from the tech radar, for example: ```tsx -import { Router as TechRadarRouter } from '@backstage/plugin-tech-radar'; +// in packages/app/src/App.tsx +import { TechRadarPage } from '@backstage/plugin-tech-radar'; -// Inside App component - - {/* other routes ... */} - } - /> - {/* other routes ... */} -; +const routes = ( + + {/* ... */} + } + /> ``` If you'd like to configure it more, see the `TechRadarPageProps` and `TechRadarComponentProps` types for options: diff --git a/plugins/tech-radar/package.json b/plugins/tech-radar/package.json index 5a1c2e2095..37ad7d8211 100644 --- a/plugins/tech-radar/package.json +++ b/plugins/tech-radar/package.json @@ -30,7 +30,7 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/techdocs/CHANGELOG.md b/plugins/techdocs/CHANGELOG.md index 4f89cd9844..847121c0cf 100644 --- a/plugins/techdocs/CHANGELOG.md +++ b/plugins/techdocs/CHANGELOG.md @@ -1,5 +1,31 @@ # @backstage/plugin-techdocs +## 0.9.0 + +### Minor Changes + +- 21fddf452: Make `techdocsStorageApiRef` and `techdocsApiRef` use interfaces instead of the + actual implementation classes. + + This renames the classes `TechDocsApi` to `TechDocsClient` and `TechDocsStorageApi` + to `TechDocsStorageClient` and renames the interfaces `TechDocs` to `TechDocsApi` + and `TechDocsStorage` to `TechDocsStorageApi` to comply the pattern elsewhere in + the project. This also fixes the types returned by some methods on those + interfaces. + +### Patch Changes + +- 6fbd7beca: Use `EntityRefLink` in header and use relations to reference the owner of the + document. +- 15cbe6815: Fix TechDocs landing page table wrong copied link +- 39bdaa004: Add customization and exportable components for TechDocs landing page +- cb8c848a3: Disable color transitions on links to avoid issues in dark mode. +- 17915e29b: Rework state management to avoid rendering multiple while navigating between pages. +- Updated dependencies [9afcac5af] +- Updated dependencies [e0c9ed759] +- Updated dependencies [6eaecbd81] + - @backstage/core@0.7.7 + ## 0.8.0 ### Minor Changes diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index 7bde151663..632feac3c1 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-techdocs", - "version": "0.8.0", + "version": "0.9.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -33,7 +33,7 @@ "dependencies": { "@backstage/config": "^0.1.4", "@backstage/catalog-model": "^0.7.7", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/plugin-catalog-react": "^0.1.4", "@backstage/theme": "^0.2.6", "@backstage/errors": "^0.1.1", diff --git a/plugins/todo/package.json b/plugins/todo/package.json index 7339e568c3..92390857ef 100644 --- a/plugins/todo/package.json +++ b/plugins/todo/package.json @@ -27,7 +27,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.3", - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/errors": "^0.1.1", "@backstage/plugin-catalog-react": "^0.1.1", "@backstage/theme": "^0.2.6", diff --git a/plugins/user-settings/package.json b/plugins/user-settings/package.json index d410dc103e..08ae87d3ec 100644 --- a/plugins/user-settings/package.json +++ b/plugins/user-settings/package.json @@ -30,7 +30,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/welcome/package.json b/plugins/welcome/package.json index b276b86835..189b6eeb59 100644 --- a/plugins/welcome/package.json +++ b/plugins/welcome/package.json @@ -30,7 +30,7 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/core": "^0.7.6", + "@backstage/core": "^0.7.7", "@backstage/theme": "^0.2.6", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/yarn.lock b/yarn.lock index b21e2a9bf5..e5bbc026c6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -69,22 +69,22 @@ dependencies: tslib "~2.0.1" -"@asyncapi/avro-schema-parser@^0.2.0": - version "0.2.0" - resolved "https://registry.npmjs.org/@asyncapi/avro-schema-parser/-/avro-schema-parser-0.2.0.tgz#c9da2bb2858aca5a3b9e9a0e600084afd3765551" - integrity sha512-/oPDPudF82RGFXz5uhF77PSDZkAR+yHpdxUEbJ5jwk/X3RB74VRk8dqSgTqhUO+pLh+/Hut3x3VFacA8C9L2QA== +"@asyncapi/avro-schema-parser@^0.2.1": + version "0.2.1" + resolved "https://registry.npmjs.org/@asyncapi/avro-schema-parser/-/avro-schema-parser-0.2.1.tgz#0b7d9953e12084e6f556db261ae08bd8f6690faa" + integrity sha512-RZJaHsdYM4dChYSrb/TWrVCn/r2qcus+9/8iLL8+SMINHb0ECgH8tFZFJpr3Tq+LV2SBFaRQ+9kuecjQ8BNDSA== -"@asyncapi/openapi-schema-parser@^2.0.0": - version "2.0.0" - resolved "https://registry.npmjs.org/@asyncapi/openapi-schema-parser/-/openapi-schema-parser-2.0.0.tgz#80e2f38e92b6635dde19aae07b92e3caa0effc58" - integrity sha512-XfDp3EIs6ptar3jARQZzi3ObmS44l6Qozc5GJmZJUQ6mHLTTqUGJ0nxcrXAW88vosjilgJVaQ63oGolA6smSHQ== +"@asyncapi/openapi-schema-parser@^2.0.1": + version "2.0.1" + resolved "https://registry.npmjs.org/@asyncapi/openapi-schema-parser/-/openapi-schema-parser-2.0.1.tgz#4d6e82cced907b14e0ad6f98261ff2562d968d96" + integrity sha512-algbtdM1gcAOa8+V8kp7WeBhdaNac82jmZUXx8YjyNfRVo02N2juDrjeBAGJd+FNva9Mb4MM7qfkJoAFpTL5VQ== dependencies: "@openapi-contrib/openapi-schema-to-json-schema" "^3.0.0" -"@asyncapi/parser@^1.4.4": - version "1.4.4" - resolved "https://registry.npmjs.org/@asyncapi/parser/-/parser-1.4.4.tgz#66f2642e3f9ae4166cdea2480b665250b1edbd59" - integrity sha512-HEYEDM0BzfCxXNAv/pIS5yZWe11xB8fQo9G/SmsbpJavOpcF0sVZVIELva/NxHVz/ZUKPMKCa4Gqz7DF/lMqpw== +"@asyncapi/parser@^1.5.0": + version "1.5.0" + resolved "https://registry.npmjs.org/@asyncapi/parser/-/parser-1.5.0.tgz#d70616a1e6081c7dd86957befd5dccc46b9a77df" + integrity sha512-HzrehCcT6R+iqtktNmrUM9wRUEMBqnCeXIrGJT0txBMS1QduNRmXGjvuDpxqwyaATPH/xu1gSp2l6pFP/hyVbA== dependencies: "@apidevtools/json-schema-ref-parser" "^9.0.6" "@asyncapi/specs" "^2.7.7" @@ -96,14 +96,14 @@ node-fetch "^2.6.0" tiny-merge-patch "^0.1.2" -"@asyncapi/react-component@^0.22.3": - version "0.22.3" - resolved "https://registry.npmjs.org/@asyncapi/react-component/-/react-component-0.22.3.tgz#6ea7fb1044308e6d46f8455218920389becf2b22" - integrity sha512-f47sboqEQ0jNp0z2A+WGzBYYMS3ASmTwAXG/q6SwLuHBW15bSjaIYOVK3E8bmftBl+wVcEgAqUc6RKGtD9iVJg== +"@asyncapi/react-component@^0.23.0": + version "0.23.0" + resolved "https://registry.npmjs.org/@asyncapi/react-component/-/react-component-0.23.0.tgz#dce9a95cd0fb0d9f0364278088ad31194d411d52" + integrity sha512-mX70k3j5eSM7ets8BBZWnu6RsNXapLKD/7np+dMeXFk7KFbigVQXXr3hS0k2R+c8tRfkhlklicibq5ar1+fjDA== dependencies: - "@asyncapi/avro-schema-parser" "^0.2.0" - "@asyncapi/openapi-schema-parser" "^2.0.0" - "@asyncapi/parser" "^1.4.4" + "@asyncapi/avro-schema-parser" "^0.2.1" + "@asyncapi/openapi-schema-parser" "^2.0.1" + "@asyncapi/parser" "^1.5.0" constate "^1.2.0" dompurify "^2.1.1" markdown-it "^11.0.1" @@ -2961,9 +2961,9 @@ integrity sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg== "@kubernetes/client-node@^0.14.0": - version "0.14.0" - resolved "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.14.0.tgz#a02806f3b6fdb68fb51d451ee8ff01faa446f557" - integrity sha512-/37JHuEUAQ5GQ4kLKBmCYvGgf5W1KZWKreKGWFYH8VvT2Hl/o0aJZasu2w0EHEfmE11JCn0X9arVmOTyVCYvww== + version "0.14.3" + resolved "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.14.3.tgz#5ed9b88873419080547f22cb74eb502bf6671fca" + integrity sha512-9hHGDNm2JEFQcRTpDxVoAVr0fowU+JH/l5atCXY9VXwvFM18pW5wr2LzLP+Q2Rh+uQv7Moz4gEjEKSCgVKykEQ== dependencies: "@types/js-yaml" "^3.12.1" "@types/node" "^10.12.0" @@ -10451,9 +10451,9 @@ core-js-pure@^3.0.0, core-js-pure@^3.0.1: integrity sha512-epIhRLkXdgv32xIUFaaAry2wdxZYBi6bgM7cB136dzzXXa+dFyRLTZeLUJxnd8ShrmyVXBub63n2NHo2JAt8Cw== core-js@3, core-js@^3.0.1, core-js@^3.0.4, core-js@^3.5.0, core-js@^3.6.0, core-js@^3.6.5: - version "3.8.3" - resolved "https://registry.npmjs.org/core-js/-/core-js-3.8.3.tgz#c21906e1f14f3689f93abcc6e26883550dd92dd0" - integrity sha512-KPYXeVZYemC2TkNEkX/01I+7yd+nX3KddKwZ1Ww7SKWdI2wQprSgLmrTddT8nw92AjEklTsPBoSdQBhbI1bQ6Q== + version "3.11.0" + resolved "https://registry.npmjs.org/core-js/-/core-js-3.11.0.tgz#05dac6aa70c0a4ad842261f8957b961d36eb8926" + integrity sha512-bd79DPpx+1Ilh9+30aT5O1sgpQd4Ttg8oqkqi51ZzhedMM1omD2e6IOF48Z/DzDCZ2svp49tN/3vneTK6ZBkXw== core-js@^2.4.0, core-js@^2.5.7, core-js@^2.6.10, core-js@^2.6.5: version "2.6.11" @@ -12518,9 +12518,9 @@ eslint-plugin-import@^2.20.2: tsconfig-paths "^3.9.0" eslint-plugin-jest@^24.1.0: - version "24.1.5" - resolved "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-24.1.5.tgz#1e866a9f0deac587d0a3d5d7cefe99815a580de2" - integrity sha512-FIP3lwC8EzEG+rOs1y96cOJmMVpdFNreoDJv29B5vIupVssRi8zrSY3QadogT0K3h1Y8TMxJ6ZSAzYUmFCp2hg== + version "24.3.6" + resolved "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-24.3.6.tgz#5f0ca019183c3188c5ad3af8e80b41de6c8e9173" + integrity sha512-WOVH4TIaBLIeCX576rLcOgjNXqP+jNlCiEmRgFTfQtJ52DpwnIQKAVGlGPAN7CZ33bW6eNfHD6s8ZbEUTQubJg== dependencies: "@typescript-eslint/experimental-utils" "^4.0.1" @@ -16965,10 +16965,10 @@ json-schema@0.2.3: resolved "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM= -json-schema@^0.2.5: - version "0.2.5" - resolved "https://registry.npmjs.org/json-schema/-/json-schema-0.2.5.tgz#97997f50972dd0500214e208c407efa4b5d7063b" - integrity sha512-gWJOWYFrhQ8j7pVm0EM8Slr+EPVq1Phf6lvzvD/WCeqkrx/f2xBI0xOsRRS9xCn3I4vKtP519dvs3TP09r24wQ== +json-schema@^0.3.0: + version "0.3.0" + resolved "https://registry.npmjs.org/json-schema/-/json-schema-0.3.0.tgz#90a9c5054bd065422c00241851ce8d59475b701b" + integrity sha512-TYfxx36xfl52Rf1LU9HyWSLGPdYLL+SQ8/E/0yVyKG8wCCDaSrhPap0vEdlsZWRaS6tnKKLPGiEJGiREVC8kxQ== json-stable-stringify-without-jsonify@^1.0.1: version "1.0.1" @@ -23538,7 +23538,7 @@ semver@7.0.0: resolved "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== -semver@7.x, semver@^7.0.0, semver@^7.1.1, semver@^7.1.3, semver@^7.2.1, semver@^7.3.2, semver@^7.3.4: +semver@7.x, semver@^7.0.0, semver@^7.1.1, semver@^7.1.3, semver@^7.2.1, semver@^7.3.2, semver@^7.3.4, semver@~7.3.0: version "7.3.5" resolved "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ== @@ -23555,13 +23555,6 @@ semver@~5.3.0: resolved "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" integrity sha1-myzl094C0XxgEq0yaqa00M9U+U8= -semver@~7.3.0: - version "7.3.5" - resolved "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" - integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ== - dependencies: - lru-cache "^6.0.0" - send@0.17.1: version "0.17.1" resolved "https://registry.npmjs.org/send/-/send-0.17.1.tgz#c1d8b059f7900f7466dd4938bdc44e11ddb376c8" @@ -27081,9 +27074,9 @@ xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.1: integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== y18n@^3.2.1: - version "3.2.1" - resolved "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41" - integrity sha1-bRX7qITAhnnA136I53WegR4H+kE= + version "3.2.2" + resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.2.tgz#85c901bd6470ce71fc4bb723ad209b70f7f28696" + integrity sha512-uGZHXkHnhF0XeeAPgnKfPv1bgKAYyVvmNL1xlKsPYZPaIHxGti2hHqvOCQv71XMsLxu1QjergkqogUnms5D3YQ== y18n@^4.0.0: version "4.0.0"