diff --git a/.changeset/giant-rivers-protect.md b/.changeset/giant-rivers-protect.md new file mode 100644 index 0000000000..cbf11a47c0 --- /dev/null +++ b/.changeset/giant-rivers-protect.md @@ -0,0 +1,10 @@ +--- +'@backstage/integration': patch +'@backstage/plugin-catalog-backend': minor +--- + +Updates the `GithubCredentialsProvider` to return the token type, it can either be `token` or `app` depending on the authentication method. + +Update the `GithubOrgReaderProcessor` NOT to query for email addresses if GitHub Apps is used for authentication, this is due to inconsistencies in the GitHub API when using server to server communications and installation tokens. https://github.community/t/api-v4-unable-to-retrieve-email-resource-not-accessible-by-integration/13831/4 for more info. + +**Removes** deprecated GithubOrgReaderProcessor provider configuration(`catalog.processors.githubOrg`). If you're using the deprecated config section make sure to migrate to [integrations](https://backstage.io/docs/integrations/github/locations) instead. diff --git a/.changeset/great-ducks-bow.md b/.changeset/great-ducks-bow.md new file mode 100644 index 0000000000..ff5d3a3458 --- /dev/null +++ b/.changeset/great-ducks-bow.md @@ -0,0 +1,7 @@ +--- +'@backstage/catalog-client': patch +'@backstage/catalog-model': patch +'@backstage/plugin-catalog-backend': patch +--- + +Foundation for standard entity status values diff --git a/.changeset/shaggy-ligers-run.md b/.changeset/shaggy-ligers-run.md new file mode 100644 index 0000000000..b6ad53e418 --- /dev/null +++ b/.changeset/shaggy-ligers-run.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Add support for transforming yaml files in jest with 'yaml-jest' diff --git a/.changeset/shiny-keys-dream.md b/.changeset/shiny-keys-dream.md new file mode 100644 index 0000000000..3e28f5d431 --- /dev/null +++ b/.changeset/shiny-keys-dream.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Set admonition font size to 1rem in TechDocs to align with the rest of the document's font sizes. +Fixes #5448 and #5541. diff --git a/.changeset/techdocs-are-readable.md b/.changeset/techdocs-are-readable.md new file mode 100644 index 0000000000..b780d4f2fe --- /dev/null +++ b/.changeset/techdocs-are-readable.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Fixes #5529, a bug that prevented TechDocs from rendering pages containing malformed links. diff --git a/.github/stale.yml b/.github/stale.yml index 1b3792ab5e..3cd0d53720 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -9,6 +9,7 @@ exemptLabels: - plugin - help wanted - good first issue + - rfc # Label to use when marking an issue as stale staleLabel: stale # Comment to post when marking an issue as stale. Set to `false` to disable diff --git a/docs/auth/gitlab/provider.md b/docs/auth/gitlab/provider.md index 8e9c499dc4..c7ac2cedc9 100644 --- a/docs/auth/gitlab/provider.md +++ b/docs/auth/gitlab/provider.md @@ -31,8 +31,8 @@ auth: providers: gitlab: development: - clientId: ${AUTH_GITLAB_APPLICATION_ID} - clientSecret: ${AUTH_GITLAB_SECRET} + clientId: ${AUTH_GITLAB_CLIENT_ID} + clientSecret: ${AUTH_GITLAB_CLIENT_SECRET} ## uncomment if using self-hosted GitLab # audience: https://gitlab.company.com ``` diff --git a/docs/auth/index.md b/docs/auth/index.md index f3783c79f3..000c57a50c 100644 --- a/docs/auth/index.md +++ b/docs/auth/index.md @@ -77,7 +77,6 @@ built-in providers: + const app = createApp({ apis, - plugins: Object.values(plugins), + components: { + SignInPage: props => ( + ( + "We would like to convey entity statuses through the catalog in a generic way, > as an integration layer. Our monitoring and alerting system has a plugin with > Backstage, and it would be useful if the entity's status field contained the -> current alert state close to the actual entity data for anyone to consume. +> current alert state close to the actual entity data for anyone to consume. We +> find the `status.items` semantics a poor fit, so we would prefer to make our +> own custom field under `status` for these purposes." -While we are considering a mechanism for contributing generic statuses to -entities, no such mechanism has yet been built. If you are interested in that -topic, [this issue](https://github.com/backstage/backstage/issues/2292) contains +We have not yet ventured to define any generic semantics for the `status` +object. We recommend sticking with the `status.items` mechanism where possible +(see below), since third party consumers will not be able to consume your status +information otherwise. Please reach out to the maintainers on Discord or by +making a GitHub issue describing your use case if you are interested in this +topic. + +## Adding a New Status Item Type + +Example intent: + +> "The semantics of the entity `status.items` field are fine for our needs, but +> we want to contribute our own type of status into that array instead of the +> catalog specific one." + +This is a simple, low risk way of adding your own status information to +entities. Consumers will be able to easily track and display the status together +with other types / sources. + +We recommend that any status type that are not strictly private within the +organization be namespaced to avoid collisions. Statuses emitted by Backstage +core processes will for example be prefixed with `backstage.io/`, your +organization may prefix with `my-org.net/`, and `pagerduty.com/active-alerts` +could be a sensible complete status item type for that particular external +system. + +The mechanics for how to emit custom statuses is not in place yet, so if this is +of interest to you, you might consider contacting the maintainers on Discord or +my making a GitHub issue describing your use case. +[This issue](https://github.com/backstage/backstage/issues/2292) also contains more context. - -But in general, errors emitted (and exceptions thrown) by any processor -including custom ones, end up in the [well known key](well-known-statuses.md) -for ingestion status. diff --git a/docs/features/software-catalog/well-known-statuses.md b/docs/features/software-catalog/well-known-statuses.md index 7ddf320d26..d293ae5062 100644 --- a/docs/features/software-catalog/well-known-statuses.md +++ b/docs/features/software-catalog/well-known-statuses.md @@ -6,35 +6,37 @@ sidebar_label: Well-known Statuses description: Lists a number of well known entity statuses, that have defined semantics. They can be attached to catalog entities and consumed by plugins as needed. --- -This section lists a number of well known -[entity status fields](descriptor-format.md#common-to-all-kinds-status), that -have defined semantics. They can be attached to catalog entities and consumed by +This section lists well known +[entity statuses](descriptor-format.md#common-to-all-kinds-status), that have +defined semantics. They can be attached to catalog entities and consumed by plugins as needed. -If you are looking to extend the set of statuses, see +If you are looking to extend the statuses, see [Extending the model](extending-the-model.md). ## Common Fields -The values of statuses are currently left unrestricted, except that they must be -objects. They therefore currently formally have no common fields. +The `status` object of an entity is currently left unrestricted, except for the +`items` field. Its structure is defined in the +[descriptor format](descriptor-format.md#common-to-all-kinds-status) section. -We reserve the right to extend this model in the future, such that some fields -of those value objects gain standardized meaning. We may for example want to add -a standard concept of "severity" or "level" to these. +We reserve the right to extend this model in the future. This status is in +active development and its format will change unexpectedly. Do not consume it in +your own code until such a time that this documentation has been updated. -## Statuses +## Status Item Types -This is a (non-exhaustive) list of statuses that are known to be in active use. +This is a (non-exhaustive) list of `status.items.[].type` values that are known +to be in active use. ### `backstage.io/catalog-processing` -Contains the current status of the catalog's ingestion of this entity. Errors -that may appear here include inability to read from the remote SCM provider, -syntax errors in the YAML file, and similar. +Expresses an aspect of the current status of the catalog's ingestion of this +entity. Errors that may appear here include inability to read from the remote +SCM provider, syntax errors in the YAML file, and similar. Note that the entity data itself may be of an older version, when errors are -present. The ingestion system keeps the old, valid entity data untouched when +present. The ingestion system keeps the old valid entity data untouched when possible, so the errors described in this state may not seem to align with the rest of the entity, because they pertain to a remote that could not be successfully ingested. This is normal. @@ -42,10 +44,12 @@ successfully ingested. This is normal. ```yaml # Example: status: - backstage.io/catalog-processing: - errors: [] + items: + - type: backstage.io/catalog-processing + level: error + message: 'NotFoundError: File not found' + error: + name: NotFoundError + message: File not found + stack: ... ``` - -This status is in active development and its format will change unexpectedly. Do -not consume it in your own code until such a time that this documentation has -been updated. diff --git a/docs/integrations/github/email.png b/docs/integrations/github/email.png new file mode 100644 index 0000000000..763cfbcd58 Binary files /dev/null and b/docs/integrations/github/email.png differ diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 4a1bf88932..5ee650e29c 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -53,6 +53,16 @@ catalog: token: ${GITHUB_TOKEN} ``` +If Backstage is configured to use GitHub Apps authentication you must grant +`Read-Only` access for `Members` under `Organization` in order to ingest users +correctly. You can modify the app's permissions under the organization settings, +`https://github.com/organizations/{ORG}/settings/apps/{APP_NAME}/permissions`. +![permissions](./permissions.png) + +**Please note that when you change permissions, the app owner will get an email +that must be approved first before the changes are applied.** +![email](./email.png) + Locations point out the specific org(s) you want to import. The `type` of these locations must be `github-org`, and the `target` must point to the exact URL of some organization. You can have several such location entries if you want, but diff --git a/docs/integrations/github/permissions.png b/docs/integrations/github/permissions.png new file mode 100644 index 0000000000..4928b184ac Binary files /dev/null and b/docs/integrations/github/permissions.png differ diff --git a/docs/tutorials/quickstart-app-auth.md b/docs/tutorials/quickstart-app-auth.md index 510b697221..d54b21f7c4 100644 --- a/docs/tutorials/quickstart-app-auth.md +++ b/docs/tutorials/quickstart-app-auth.md @@ -329,12 +329,12 @@ above: ### 6. In the same file, modify createApp -Remember to modify the provider information based on the table above. +Remember to add or modify a `SignInPage` component for `createApp`, using +provider information based on the table above. ```tsx const app = createApp({ apis, - plugins: Object.values(plugins), components: { SignInPage: props => ( = { items: T[]; }; +// @public +export const ENTITY_STATUS_CATALOG_PROCESSING_TYPE = "backstage.io/catalog-processing"; + // (No @packageDocumentation comment for this package) diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 359e3a2c60..98b378f5b0 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -18,7 +18,8 @@ import { Entity } from '@backstage/catalog-model'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { CatalogClient } from './CatalogClient'; -import { CatalogListResponse, DiscoveryApi } from './types'; +import { CatalogListResponse } from './types/api'; +import { DiscoveryApi } from './types/discovery'; const server = setupServer(); const token = 'fake-token'; diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 063db5a6a3..7c9b313ce7 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -31,8 +31,8 @@ import { CatalogEntitiesRequest, CatalogListResponse, CatalogRequestOptions, - DiscoveryApi, -} from './types'; +} from './types/api'; +import { DiscoveryApi } from './types/discovery'; export class CatalogClient implements CatalogApi { private readonly discoveryApi: DiscoveryApi; diff --git a/packages/catalog-client/src/index.ts b/packages/catalog-client/src/index.ts index c5a626e25b..59c652f9fd 100644 --- a/packages/catalog-client/src/index.ts +++ b/packages/catalog-client/src/index.ts @@ -15,10 +15,4 @@ */ export { CatalogClient } from './CatalogClient'; -export type { - AddLocationRequest, - AddLocationResponse, - CatalogApi, - CatalogEntitiesRequest, - CatalogListResponse, -} from './types'; +export * from './types'; diff --git a/packages/catalog-client/src/types.ts b/packages/catalog-client/src/types/api.ts similarity index 91% rename from packages/catalog-client/src/types.ts rename to packages/catalog-client/src/types/api.ts index ef907eafa9..f9ba2e9d8d 100644 --- a/packages/catalog-client/src/types.ts +++ b/packages/catalog-client/src/types/api.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 Spotify AB + * Copyright 2021 Spotify AB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -81,10 +81,3 @@ export type AddLocationResponse = { location: Location; entities: Entity[]; }; - -/** - * This is a copy of the core DiscoveryApi, to avoid importing core. - */ -export type DiscoveryApi = { - getBaseUrl(pluginId: string): Promise; -}; diff --git a/packages/catalog-client/src/types/discovery.ts b/packages/catalog-client/src/types/discovery.ts new file mode 100644 index 0000000000..90eb748b2f --- /dev/null +++ b/packages/catalog-client/src/types/discovery.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This is a copy of the core DiscoveryApi, to avoid importing core. + */ +export type DiscoveryApi = { + getBaseUrl(pluginId: string): Promise; +}; diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts new file mode 100644 index 0000000000..c1670659ec --- /dev/null +++ b/packages/catalog-client/src/types/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type { + AddLocationRequest, + AddLocationResponse, + CatalogApi, + CatalogEntitiesRequest, + CatalogListResponse, +} from './api'; +export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/packages/catalog-client/src/types/status.ts b/packages/catalog-client/src/types/status.ts new file mode 100644 index 0000000000..7990c4b121 --- /dev/null +++ b/packages/catalog-client/src/types/status.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The entity `status.items[].type` for the status of the processing engine in + * regards to an entity. + */ +export const ENTITY_STATUS_CATALOG_PROCESSING_TYPE = + 'backstage.io/catalog-processing'; diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 890a45b6e7..7b5542fcd6 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -7,6 +7,7 @@ import { JsonObject } from '@backstage/config'; import { JSONSchema7 } from 'json-schema'; import { JsonValue } from '@backstage/config'; +import { SerializedError } from '@backstage/errors'; import * as yup from 'yup'; // @public (undocumented) @@ -112,7 +113,7 @@ export type Entity = { metadata: EntityMeta; spec?: JsonObject; relations?: EntityRelation[]; - status?: Record; + status?: UNSTABLE_EntityStatus; }; // @public @@ -528,6 +529,22 @@ export interface TemplateEntityV1beta2 extends Entity { // @public (undocumented) export const templateEntityV1beta2Validator: KindValidator; +// @alpha +export type UNSTABLE_EntityStatus = { + items?: UNSTABLE_EntityStatusItem[]; +}; + +// @alpha +export type UNSTABLE_EntityStatusItem = { + type: string; + level: UNSTABLE_EntityStatusLevel; + message: string; + error?: SerializedError; +}; + +// @alpha +export type UNSTABLE_EntityStatusLevel = 'info' | 'warning' | 'error'; + // @public (undocumented) interface UserEntityV1alpha1 extends Entity { // (undocumented) diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json index 7a19be8708..97ce0e6568 100644 --- a/packages/catalog-model/package.json +++ b/packages/catalog-model/package.json @@ -30,6 +30,7 @@ }, "dependencies": { "@backstage/config": "^0.1.5", + "@backstage/errors": "^0.1.1", "@types/json-schema": "^7.0.5", "@types/yup": "^0.29.8", "ajv": "^7.0.3", diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts index 1b5d38dee9..3b5a0621bb 100644 --- a/packages/catalog-model/src/entity/Entity.ts +++ b/packages/catalog-model/src/entity/Entity.ts @@ -16,6 +16,7 @@ import { JsonObject } from '@backstage/config'; import { EntityName } from '../types'; +import { UNSTABLE_EntityStatus } from './EntityStatus'; /** * The format envelope that's common to all versions/kinds of entity. @@ -55,7 +56,7 @@ export type Entity = { * The keys are implementation defined and the values can be any JSON object * with semantics that match that implementation. */ - status?: Record; + status?: UNSTABLE_EntityStatus; }; /** diff --git a/packages/catalog-model/src/entity/EntityStatus.ts b/packages/catalog-model/src/entity/EntityStatus.ts new file mode 100644 index 0000000000..67090f2bc2 --- /dev/null +++ b/packages/catalog-model/src/entity/EntityStatus.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SerializedError } from '@backstage/errors'; + +/** + * The current status of the entity, as claimed by various sources. + * @alpha + */ +export type UNSTABLE_EntityStatus = { + /** + * Specific status item on a well known format. + */ + items?: UNSTABLE_EntityStatusItem[]; +}; + +/** + * A specific status item on a well known format. + * @alpha + */ +export type UNSTABLE_EntityStatusItem = { + /** + * The type of status as a unique key per source. + */ + type: string; + /** + * The level / severity of the status item. If the level is "error", the + * processing of the entity may be entirely blocked. In this case the status + * entry may apply to a different, newer version of the data than what is + * being returned in the catalog response. + */ + level: UNSTABLE_EntityStatusLevel; + /** + * A brief message describing the status, intended for human consumption. + */ + message: string; + /** + * An optional serialized error object related to the status. + */ + error?: SerializedError; +}; + +/** + * Each entity status item has a level, describing its severity. + * @alpha + */ +export type UNSTABLE_EntityStatusLevel = + | 'info' // Only informative data + | 'warning' // Warnings were found + | 'error'; // Errors were found diff --git a/packages/catalog-model/src/entity/index.ts b/packages/catalog-model/src/entity/index.ts index 572df63557..2ccd205d85 100644 --- a/packages/catalog-model/src/entity/index.ts +++ b/packages/catalog-model/src/entity/index.ts @@ -15,10 +15,10 @@ */ export { + EDIT_URL_ANNOTATION, ENTITY_DEFAULT_NAMESPACE, ENTITY_META_GENERATED_FIELDS, VIEW_URL_ANNOTATION, - EDIT_URL_ANNOTATION, } from './constants'; export type { Entity, @@ -27,6 +27,11 @@ export type { EntityRelation, EntityRelationSpec, } from './Entity'; +export type { + UNSTABLE_EntityStatus, + UNSTABLE_EntityStatusItem, + UNSTABLE_EntityStatusLevel, +} from './EntityStatus'; export * from './policies'; export { compareEntityToRef, diff --git a/packages/catalog-model/src/schema/Entity.schema.json b/packages/catalog-model/src/schema/Entity.schema.json index e6b004dfd0..b8c43dda9e 100644 --- a/packages/catalog-model/src/schema/Entity.schema.json +++ b/packages/catalog-model/src/schema/Entity.schema.json @@ -64,13 +64,7 @@ } }, "status": { - "type": "object", - "description": "The current status of the entity, as claimed by various sources.", - "patternProperties": { - "^.+$": { - "$ref": "common#status" - } - } + "$ref": "common#status" } } } diff --git a/packages/catalog-model/src/schema/shared/common.schema.json b/packages/catalog-model/src/schema/shared/common.schema.json index cb4d82cdb9..efb8b6d1bb 100644 --- a/packages/catalog-model/src/schema/shared/common.schema.json +++ b/packages/catalog-model/src/schema/shared/common.schema.json @@ -46,8 +46,53 @@ "status": { "$id": "#status", "type": "object", - "description": "A specific status of an entity.", - "additionalProperties": true + "description": "The current status of the entity, as claimed by various sources.", + "required": [], + "additionalProperties": true, + "properties": { + "items": { + "$ref": "#statusItem" + } + } + }, + "statusItem": { + "$id": "#statusItem", + "type": "object", + "description": "A specific status item on a well known format.", + "required": ["type", "level", "message"], + "additionalProperties": true, + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "level": { + "$ref": "#statusLevel", + "description": "The status level / severity of the status item." + }, + "message": { + "type": "string", + "description": "A brief message describing the status, intended for human consumption." + }, + "error": { + "$ref": "#error", + "description": "An optional serialized error object related to the status." + } + } + }, + "statusLevel": { + "$id": "#statusLevel", + "type": "string", + "description": "A status level / severity.", + "enum": ["info", "warning", "error"] + }, + "error": { + "$id": "#error", + "type": "object", + "description": "A serialized error object.", + "required": [], + "additionalProperties": true, + "properties": {} } } } diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index 94cf059cbd..4a2ae7119c 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -89,6 +89,7 @@ async function getConfig() { '\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg)$': require.resolve( './jestFileTransform.js', ), + '\\.(yaml)$': require.resolve('yaml-jest'), }, // A bit more opinionated diff --git a/packages/cli/package.json b/packages/cli/package.json index a72ef48bf3..12d8373ba5 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -113,6 +113,7 @@ "webpack-dev-server": "3.11.0", "webpack-node-externals": "^2.5.0", "yaml": "^1.10.0", + "yaml-jest": "^1.0.5", "yml-loader": "^2.1.0", "yn": "^4.0.0" }, diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 5387eb12db..31cbc8b5d8 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -115,6 +115,9 @@ export class GithubCredentialsProvider { }): Promise; } +// @public (undocumented) +export type GithubCredentialType = 'app' | 'token'; + // @public (undocumented) export class GitHubIntegration implements ScmIntegration { constructor(integrationConfig: GitHubIntegrationConfig); diff --git a/packages/integration/src/github/GithubCredentialsProvider.test.ts b/packages/integration/src/github/GithubCredentialsProvider.test.ts index d295677c55..271d4b9920 100644 --- a/packages/integration/src/github/GithubCredentialsProvider.test.ts +++ b/packages/integration/src/github/GithubCredentialsProvider.test.ts @@ -81,13 +81,13 @@ describe('GithubCredentialsProvider tests', () => { }, } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); - const { token, headers } = await github.getCredentials({ + const { token, headers, type } = await github.getCredentials({ url: 'https://github.com/backstage/foobar', }); const { token: accessToken2 } = await github.getCredentials({ url: 'https://github.com/backstage/foobar', }); - + expect(type).toEqual('app'); expect(token).toEqual('secret_token'); expect(token).toEqual(accessToken2); expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); @@ -102,6 +102,7 @@ describe('GithubCredentialsProvider tests', () => { Authorization: 'Bearer hardcoded_token', }, token: 'hardcoded_token', + type: 'token', }); }); @@ -260,6 +261,6 @@ describe('GithubCredentialsProvider tests', () => { githubProvider.getCredentials({ url: 'https://github.com/backstage', }), - ).resolves.toEqual({ headers: undefined, token: undefined }); + ).resolves.toEqual({ headers: undefined, token: undefined, type: 'token' }); }); }); diff --git a/packages/integration/src/github/GithubCredentialsProvider.ts b/packages/integration/src/github/GithubCredentialsProvider.ts index 0bf217c824..9bc09deab3 100644 --- a/packages/integration/src/github/GithubCredentialsProvider.ts +++ b/packages/integration/src/github/GithubCredentialsProvider.ts @@ -192,9 +192,12 @@ export class GithubAppCredentialsMux { } } +export type GithubCredentialType = 'app' | 'token'; + export type GithubCredentials = { headers?: { [name: string]: string }; token?: string; + type: GithubCredentialType; }; // TODO: Possibly move this to a backend only package so that it's not used in the frontend by mistake @@ -226,8 +229,10 @@ export class GithubCredentialsProvider { const owner = parsed.owner || parsed.name; const repo = parsed.owner ? parsed.name : undefined; + let type: GithubCredentialType = 'app'; let token = await this.githubAppCredentialsMux.getAppToken(owner, repo); if (!token) { + type = 'token'; token = this.token; } @@ -238,6 +243,7 @@ export class GithubCredentialsProvider { } : undefined, token, + type, }; } } diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index 99d2f56d0f..ee92b7d934 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -21,4 +21,5 @@ export { export type { GitHubIntegrationConfig } from './config'; export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core'; export { GithubCredentialsProvider } from './GithubCredentialsProvider'; +export type { GithubCredentialType } from './GithubCredentialsProvider'; export { GitHubIntegration } from './GitHubIntegration'; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index f55ca2c13f..2f0164ace5 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -31,6 +31,7 @@ "dependencies": { "@azure/msal-node": "^1.0.0-beta.3", "@backstage/backend-common": "^0.8.1", + "@backstage/catalog-client": "^0.3.11", "@backstage/catalog-model": "^0.7.10", "@backstage/config": "^0.1.5", "@backstage/errors": "^0.1.1", diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts index 9b724def28..dc6a2e5ec2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts @@ -13,15 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +jest.mock('@octokit/graphql'); import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { - GitHubIntegration, ScmIntegrations, - ScmIntegrationsGroup, + GithubCredentialsProvider, } from '@backstage/integration'; import { GithubOrgReaderProcessor, parseUrl } from './GithubOrgReaderProcessor'; +import { graphql } from '@octokit/graphql'; +import { ConfigReader } from '@backstage/config'; describe('GithubOrgReaderProcessor', () => { describe('parseUrl', () => { @@ -34,67 +35,27 @@ describe('GithubOrgReaderProcessor', () => { }); describe('implementation', () => { - let integrations: ScmIntegrations; - let github: jest.Mocked>; + const logger = getVoidLogger(); + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + }, + ], + }, + }), + ); beforeEach(() => { - github = { - byHost: jest.fn(), - byUrl: jest.fn(), - list: jest.fn(), - }; - integrations = ({ - github, - } as Partial) as ScmIntegrations; - }); - - it('rejects unknown types', async () => { - const processor = new GithubOrgReaderProcessor({ - providers: [ - { - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], - integrations, - logger: getVoidLogger(), - }); - const location: LocationSpec = { - type: 'not-github-org', - target: 'https://github.com', - }; - await expect( - processor.readLocation(location, false, () => {}), - ).resolves.toBeFalsy(); - }); - - it('rejects unknown targets from providers', async () => { - const processor = new GithubOrgReaderProcessor({ - providers: [ - { - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], - integrations, - logger: getVoidLogger(), - }); - const location: LocationSpec = { - type: 'github-org', - target: 'https://not.github.com/apa', - }; - await expect( - processor.readLocation(location, false, () => {}), - ).rejects.toThrow( - /There is no GitHub Org provider that matches https:\/\/not.github.com\/apa/, - ); + jest.resetAllMocks(); }); it('rejects unknown targets from integrations', async () => { const processor = new GithubOrgReaderProcessor({ - providers: [], integrations, - logger: getVoidLogger(), + logger, }); const location: LocationSpec = { type: 'github-org', @@ -106,5 +67,101 @@ describe('GithubOrgReaderProcessor', () => { /There is no GitHub Org provider that matches https:\/\/not.github.com\/apa/, ); }); + + it('should not query for email addresses when GitHub Apps is used for authentication', async () => { + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { pageInfo: { hasNextPage: false }, nodes: [{}] }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { members: { pageInfo: { hasNextPage: false }, nodes: [{}] } }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + jest.spyOn(GithubCredentialsProvider, 'create').mockReturnValue({ + getCredentials: mockGetCredentials, + } as any); + + const processor = new GithubOrgReaderProcessor({ + integrations, + logger, + }); + const location: LocationSpec = { + type: 'github-org', + target: 'https://github.com/backstage', + }; + + await processor.readLocation(location, false, () => {}); + + expect(mockClient).toHaveBeenCalledWith( + expect.stringContaining('@include(if: $email)'), + expect.objectContaining({ email: false }), + ); + }); + + it('should query for email addresses when token is used for authentication', async () => { + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'token', + }); + + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { pageInfo: { hasNextPage: false }, nodes: [{}] }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { members: { pageInfo: { hasNextPage: false }, nodes: [{}] } }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + jest.spyOn(GithubCredentialsProvider, 'create').mockReturnValue({ + getCredentials: mockGetCredentials, + } as any); + + const processor = new GithubOrgReaderProcessor({ + integrations, + logger, + }); + const location: LocationSpec = { + type: 'github-org', + target: 'https://github.com/backstage', + }; + + await processor.readLocation(location, false, () => {}); + + expect(mockClient).toHaveBeenCalledWith( + expect.stringContaining('@include(if: $email)'), + expect.objectContaining({ email: true }), + ); + }); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 14204adde7..6a2b4bd8b1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -18,16 +18,12 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { GithubCredentialsProvider, + GithubCredentialType, ScmIntegrations, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; -import { - getOrganizationTeams, - getOrganizationUsers, - ProviderConfig, - readGithubConfig, -} from './github'; +import { getOrganizationTeams, getOrganizationUsers } from './github'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; import { buildOrgHierarchy } from './util/org'; @@ -38,7 +34,6 @@ type GraphQL = typeof graphql; * Extracts teams and users out of a GitHub org. */ export class GithubOrgReaderProcessor implements CatalogProcessor { - private readonly providers: ProviderConfig[]; private readonly integrations: ScmIntegrations; private readonly logger: Logger; @@ -47,17 +42,11 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { return new GithubOrgReaderProcessor({ ...options, - providers: readGithubConfig(config), integrations, }); } - constructor(options: { - providers: ProviderConfig[]; - integrations: ScmIntegrations; - logger: Logger; - }) { - this.providers = options.providers; + constructor(options: { integrations: ScmIntegrations; logger: Logger }) { this.integrations = options.integrations; this.logger = options.logger; } @@ -71,14 +60,14 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { return false; } - const client = await this.createClient(location.target); + const { client, tokenType } = await this.createClient(location.target); const { org } = parseUrl(location.target); // Read out all of the raw data const startTimestamp = Date.now(); this.logger.info('Reading GitHub users and groups'); - const { users } = await getOrganizationUsers(client, org); + const { users } = await getOrganizationUsers(client, org, tokenType); const { groups, groupMemberUsers } = await getOrganizationTeams( client, org, @@ -112,65 +101,31 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { return true; } - private async createClient(orgUrl: string): Promise { - let client = await this.createClientFromIntegrations(orgUrl); + private async createClient( + orgUrl: string, + ): Promise<{ client: GraphQL; tokenType: GithubCredentialType }> { + const gitHubConfig = this.integrations.github.byUrl(orgUrl)?.config; - if (!client) { - client = await this.createClientFromProvider(orgUrl); - } - - if (!client) { + if (!gitHubConfig) { throw new Error( - `There is no GitHub Org provider that matches ${orgUrl}. Please add a configuration for an integration or add an entry for it under catalog.processors.githubOrg.providers.`, + `There is no GitHub Org provider that matches ${orgUrl}. Please add a configuration for an integration.`, ); } - return client; - } - - private async createClientFromProvider( - orgUrl: string, - ): Promise { - const provider = this.providers.find(p => - orgUrl.startsWith(`${p.target}/`), - ); - - if (!provider) { - return undefined; - } - - this.logger.warn( - 'GithubOrgReaderProcessor uses provider defined in catalog.processors.githubOrg.providers, migrate to integrations instead. See https://backstage.io/docs/integrations/github/locations', - ); - - return !provider.token - ? graphql - : graphql.defaults({ - baseUrl: provider.apiBaseUrl, - headers: { - authorization: `token ${provider.token}`, - }, - }); - } - - private async createClientFromIntegrations( - orgUrl: string, - ): Promise { - const gitHubConfig = this.integrations.github.byUrl(orgUrl)?.config; - if (!gitHubConfig) { - return undefined; - } - const credentialsProvider = GithubCredentialsProvider.create(gitHubConfig); - - const { headers } = await credentialsProvider.getCredentials({ + const { + headers, + type: tokenType, + } = await credentialsProvider.getCredentials({ url: orgUrl, }); - return graphql.defaults({ + const client = graphql.defaults({ baseUrl: gitHubConfig.apiBaseUrl, headers, }); + + return { client, tokenType }; } } diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts index 88bddc93fb..c5094230f8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts @@ -65,7 +65,9 @@ describe('github', () => { graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))), ); - await expect(getOrganizationUsers(graphql, 'a')).resolves.toEqual(output); + await expect( + getOrganizationUsers(graphql, 'a', 'token'), + ).resolves.toEqual(output); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index f6cfcb8901..2cc97d2ec5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -15,6 +15,7 @@ */ import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { GithubCredentialType } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; // Graphql types @@ -77,13 +78,20 @@ export type Connection = { export async function getOrganizationUsers( client: typeof graphql, org: string, + tokenType: GithubCredentialType, ): Promise<{ users: UserEntity[] }> { const query = ` - query users($org: String!, $cursor: String) { + query users($org: String!, $email: Boolean!, $cursor: String) { organization(login: $org) { membersWithRole(first: 100, after: $cursor) { pageInfo { hasNextPage, endCursor } - nodes { avatarUrl, bio, email, login, name } + nodes { + avatarUrl, + bio, + email @include(if: $email), + login, + name + } } } }`; @@ -119,7 +127,7 @@ export async function getOrganizationUsers( query, r => r.organization?.membersWithRole, mapper, - { org }, + { org, email: tokenType === 'token' }, ); return { users }; diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index 55cb8e2029..b7b6dfa297 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -15,6 +15,7 @@ */ import { stringifyEntityRef } from '@backstage/catalog-model'; +import { serializeError } from '@backstage/errors'; import { Logger } from 'winston'; import { ProcessingDatabase } from './database/types'; import { Stitcher } from './Stitcher'; @@ -118,7 +119,27 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { for (const error of result.errors) { this.logger.warn(error.message); } + const errorsString = JSON.stringify( + result.errors.map(e => serializeError(e)), + ); + + // If the result was marked as not OK, it signals that some part of the + // processing pipeline threw an exception. This can happen both as part of + // non-catastrophic things such as due to validation errors, as well as if + // something fatal happens inside the processing for other reasons. In any + // case, this means we can't trust that anything in the output is okay. So + // just store the errors and trigger a stich so that they become visible to + // the outside. if (!result.ok) { + await this.processingDatabase.transaction(async tx => { + await this.processingDatabase.updateProcessedEntityErrors(tx, { + id, + errors: errorsString, + }); + }); + await this.stitcher.stitch( + new Set([stringifyEntityRef(unprocessedEntity)]), + ); return; } @@ -128,7 +149,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { id, processedEntity: result.completedEntity, state: result.state, - errors: JSON.stringify(result.errors), + errors: errorsString, relations: result.relations, deferredEntities: result.deferredEntities, }); diff --git a/plugins/catalog-backend/src/next/Stitcher.test.ts b/plugins/catalog-backend/src/next/Stitcher.test.ts index 644284f3e4..f38af384fa 100644 --- a/plugins/catalog-backend/src/next/Stitcher.test.ts +++ b/plugins/catalog-backend/src/next/Stitcher.test.ts @@ -91,9 +91,6 @@ describe('Stitcher', () => { }, }, ], - status: { - 'backstage.io/catalog-processing': {}, - }, apiVersion: 'a', kind: 'k', metadata: { @@ -174,9 +171,6 @@ describe('Stitcher', () => { }, }, ]), - status: { - 'backstage.io/catalog-processing': {}, - }, apiVersion: 'a', kind: 'k', metadata: { diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index d7d4325ad0..0281591fb1 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -14,9 +14,13 @@ * limitations under the License. */ -import { Entity, parseEntityRef } from '@backstage/catalog-model'; -import { JsonObject } from '@backstage/config'; -import { ConflictError } from '@backstage/errors'; +import { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from '@backstage/catalog-client'; +import { + Entity, + parseEntityRef, + UNSTABLE_EntityStatusItem, +} from '@backstage/catalog-model'; +import { ConflictError, SerializedError } from '@backstage/errors'; import { createHash } from 'crypto'; import stableStringify from 'fast-json-stable-stringify'; import { Knex } from 'knex'; @@ -36,10 +40,6 @@ export type DbFinalEntitiesRow = { final_entity: string; }; -type ProcessingStatus = { - errors?: JsonObject[]; -}; - function generateStableHash(entity: Entity) { return createHash('sha1') .update(stableStringify({ ...entity })) @@ -139,7 +139,7 @@ export class Stitcher { // it const entity = JSON.parse(processedEntity) as Entity; const isOrphan = Number(incomingReferenceCount) === 0; - const processingStatus: ProcessingStatus = {}; + let statusItems: UNSTABLE_EntityStatusItem[] = []; if (isOrphan) { this.logger.debug(`${entityRef} is an orphan`); @@ -149,9 +149,14 @@ export class Stitcher { }; } if (errors) { - const parsedErrors = JSON.parse(errors); + const parsedErrors = JSON.parse(errors) as SerializedError[]; if (Array.isArray(parsedErrors) && parsedErrors.length) { - processingStatus.errors = parsedErrors; + statusItems = parsedErrors.map(e => ({ + type: ENTITY_STATUS_CATALOG_PROCESSING_TYPE, + level: 'error', + message: e.toString(), + error: e, + })); } } @@ -163,10 +168,12 @@ export class Stitcher { type: row.relationType!, target: parseEntityRef(row.relationTarget!), })); - entity.status = { - ...entity.status, - 'backstage.io/catalog-processing': processingStatus, - }; + if (statusItems.length) { + entity.status = { + ...entity.status, + items: [...(entity.status?.items ?? []), ...statusItems], + }; + } // If the output entity was actually not changed, just abort const hash = generateStableHash(entity); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 8588ba5e52..cd3ddf037b 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -123,6 +123,20 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ); } + async updateProcessedEntityErrors( + txOpaque: Transaction, + options: UpdateProcessedEntityOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + const { id, errors } = options; + + await tx('refresh_state') + .update({ + errors, + }) + .where('entity_id', id); + } + private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] { return lodash.uniqBy( rows, diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index d7a1dcd20b..f65c796a00 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -34,6 +34,11 @@ export type UpdateProcessedEntityOptions = { deferredEntities: Entity[]; }; +export type UpdateProcessedEntityErrorsOptions = { + id: string; + errors?: string; +}; + export type RefreshStateItem = { id: string; entityRef: string; @@ -80,10 +85,18 @@ export interface ProcessingDatabase { ): Promise; /** - * Updates the + * Updates a processed entity */ updateProcessedEntity( txOpaque: Transaction, options: UpdateProcessedEntityOptions, ): Promise; + + /** + * Updates only the errors of a processed entity + */ + updateProcessedEntityErrors( + txOpaque: Transaction, + options: UpdateProcessedEntityErrorsOptions, + ): Promise; } diff --git a/plugins/techdocs/src/reader/components/Reader.tsx b/plugins/techdocs/src/reader/components/Reader.tsx index 09cd3abec8..c779985364 100644 --- a/plugins/techdocs/src/reader/components/Reader.tsx +++ b/plugins/techdocs/src/reader/components/Reader.tsx @@ -183,21 +183,24 @@ export const Reader = ({ entityId, onReady }: Props) => { border-bottom: 1px solid ${theme.palette.text.primary}; } .md-typeset table:not([class]) th { font-weight: bold; } + .md-typeset .admonition, .md-typeset details { + font-size: 1rem; + } @media screen and (max-width: 76.1875em) { - .md-nav { - background-color: ${theme.palette.background.default}; + .md-nav { + background-color: ${theme.palette.background.default}; transition: none !important } .md-sidebar--secondary { display: none; } .md-sidebar--primary { left: 72px; width: 10rem } .md-content { margin-left: 10rem; max-width: calc(100% - 10rem); } .md-content__inner { font-size: 0.9rem } - .md-footer { - position: static; - margin-left: 10rem; - width: calc(100% - 10rem); + .md-footer { + position: static; + margin-left: 10rem; + width: calc(100% - 10rem); } - .md-nav--primary .md-nav__title { + .md-nav--primary .md-nav__title { white-space: normal; height: auto; line-height: 1rem; diff --git a/plugins/techdocs/src/reader/transformers/rewriteDocLinks.test.ts b/plugins/techdocs/src/reader/transformers/rewriteDocLinks.test.ts index ca85c64fbe..01467f665d 100644 --- a/plugins/techdocs/src/reader/transformers/rewriteDocLinks.test.ts +++ b/plugins/techdocs/src/reader/transformers/rewriteDocLinks.test.ts @@ -56,6 +56,21 @@ describe('rewriteDocLinks', () => { 'http://localhost/example-docs/example-page', ]); }); + + it('should rewrite non-parseable URLs as text', () => { + const expectedText = `www.my-internet.[top-level-domain]/pathname/[URLkey]`; + const shadowDom = createTestShadowDom( + `${expectedText}`, + { + preTransformers: [rewriteDocLinks()], + postTransformers: [], + }, + ); + + // There should be no tags, but the link text should remain. + expect(getSample(shadowDom, 'a', 'href')).toEqual([]); + expect(shadowDom.innerHTML).toContain(expectedText); + }); }); describe('normalizeUrl', () => { diff --git a/plugins/techdocs/src/reader/transformers/rewriteDocLinks.ts b/plugins/techdocs/src/reader/transformers/rewriteDocLinks.ts index 5c4a730963..8c4bc29201 100644 --- a/plugins/techdocs/src/reader/transformers/rewriteDocLinks.ts +++ b/plugins/techdocs/src/reader/transformers/rewriteDocLinks.ts @@ -31,12 +31,19 @@ export const rewriteDocLinks = (): Transformer => { if (elemAttribute.match(/^https?:\/\//i)) { elem.setAttribute('target', '_blank'); } - const normalizedWindowLocation = normalizeUrl(window.location.href); - elem.setAttribute( - attributeName, - new URL(elemAttribute, normalizedWindowLocation).toString(), - ); + try { + const normalizedWindowLocation = normalizeUrl( + window.location.href, + ); + elem.setAttribute( + attributeName, + new URL(elemAttribute, normalizedWindowLocation).toString(), + ); + } catch (_e) { + // Non-parseable links should be re-written as plain text. + elem.replaceWith(elem.textContent || elemAttribute); + } } }); }; diff --git a/yarn.lock b/yarn.lock index befc88cefa..64aeb6eedc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5734,9 +5734,11 @@ integrity sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w== "@types/classnames@^2.2.9": - version "2.2.11" - resolved "https://registry.npmjs.org/@types/classnames/-/classnames-2.2.11.tgz#2521cc86f69d15c5b90664e4829d84566052c1cf" - integrity sha512-2koNhpWm3DgWRp5tpkiJ8JGc1xTn2q0l+jUNUE7oMKXUf5NpI9AIdC4kbjGNFBdHtcxBD18LAksoudAVhFKCjw== + version "2.3.1" + resolved "https://registry.npmjs.org/@types/classnames/-/classnames-2.3.1.tgz#3c2467aa0f1a93f1f021e3b9bcf938bd5dfdc0dd" + integrity sha512-zeOWb0JGBoVmlQoznvqXbE0tEC/HONsnoUNH19Hc96NFsTAwTXbTqb8FMYkru1F/iqp7a18Ws3nWJvtA1sHD1A== + dependencies: + classnames "*" "@types/clean-css@*": version "4.2.1" @@ -9409,20 +9411,10 @@ caniuse-api@^3.0.0: lodash.memoize "^4.1.2" lodash.uniq "^4.5.0" -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001035, caniuse-lite@^1.0.30001109: - version "1.0.30001113" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001113.tgz#22016ab55b5a8b04fa00ca342d9ee1b98df48065" - integrity sha512-qMvjHiKH21zzM/VDZr6oosO6Ri3U0V2tC015jRXjOecwQCJtsU5zklTNTk31jQbIOP8gha0h1ccM/g0ECP+4BA== - -caniuse-lite@^1.0.30001125: - version "1.0.30001198" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001198.tgz#ed2d9b5f060322ba2efa42afdc56dee3255473f4" - integrity sha512-r5GGgESqOPZzwvdLVER374FpQu2WluCF1Z2DSiFJ89KSmGjT0LVKjgv4NcAqHmGWF9ihNpqRI9KXO9Ex4sKsgA== - -caniuse-lite@^1.0.30001164: - version "1.0.30001165" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001165.tgz#32955490d2f60290bb186bb754f2981917fa744f" - integrity sha512-8cEsSMwXfx7lWSUMA2s08z9dIgsnR5NAqjXP23stdsU3AUWkCr/rr4s4OFtHXn5XXr6+7kam3QFVoYyXNPdJPA== +caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001035, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001125, caniuse-lite@^1.0.30001164: + version "1.0.30001228" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001228.tgz" + integrity sha512-QQmLOGJ3DEgokHbMSA8cj2a+geXqmnpyOFT0lhQV6P3/YOJvGDEwoedcwxEQ30gJIwIIunHIicunJ2rzK5gB2A== canvas@^2.6.1: version "2.7.0" @@ -9681,7 +9673,7 @@ class-utils@^0.3.5: isobject "^3.0.0" static-extend "^0.1.1" -classnames@^2.2.5, classnames@^2.2.6, classnames@^2.3.1: +classnames@*, classnames@^2.2.5, classnames@^2.2.6, classnames@^2.3.1: version "2.3.1" resolved "https://registry.npmjs.org/classnames/-/classnames-2.3.1.tgz#dfcfa3891e306ec1dad105d0e88f4417b8535e8e" integrity sha512-OlQdbZ7gLfGarSqxesMesDa5uz7KFbID8Kpq/SxIoNGDqY8lSYs0D+hhtBXhcdB3rcbXArFr7vlHheLk1voeNA== @@ -16809,6 +16801,14 @@ js-yaml@^3.13.0, js-yaml@^3.13.1, js-yaml@^3.14.0, js-yaml@^3.6.1, js-yaml@^3.8. argparse "^1.0.7" esprima "^4.0.0" +js-yaml@^3.7.0: + version "3.14.1" + resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== + dependencies: + argparse "^1.0.7" + esprima "^4.0.0" + js-yaml@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz#f426bc0ff4b4051926cd588c71113183409a121f" @@ -18740,6 +18740,7 @@ minipass-fetch@^1.3.0, minipass-fetch@^1.3.2: resolved "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-1.3.3.tgz#34c7cea038c817a8658461bf35174551dce17a0a" integrity sha512-akCrLDWfbdAWkMLBxJEeWTdNsjML+dt5YgOI4gJ53vuO0vrmYQkUPxa6j6V65s9CcePIr2SSWqjT2EcrNseryQ== dependencies: + encoding "^0.1.12" minipass "^3.1.0" minipass-sized "^1.0.3" minizlib "^2.0.0" @@ -27119,6 +27120,13 @@ yaml-ast-parser@0.0.43, yaml-ast-parser@^0.0.43: resolved "https://registry.npmjs.org/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz#e8a23e6fb4c38076ab92995c5dca33f3d3d7c9bb" integrity sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A== +yaml-jest@^1.0.5: + version "1.0.5" + resolved "https://registry.npmjs.org/yaml-jest/-/yaml-jest-1.0.5.tgz#288e541bae1b551d113d4864d6cb8f316e1bd331" + integrity sha1-KI5UG64bVR0RPUhk1suPMW4b0zE= + dependencies: + js-yaml "^3.7.0" + yaml@^1.10.0, yaml@^1.7.2, yaml@^1.9.2: version "1.10.2" resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"