From 1d5e42655cdbd4f23b49fbb06e87f29557659f2a Mon Sep 17 00:00:00 2001 From: Alex Crome Date: Mon, 1 May 2023 10:24:22 +0100 Subject: [PATCH 01/32] Correct command to create new plugins in create-app readme Signed-off-by: Alex Crome --- .changeset/chilled-goats-jump.md | 5 +++++ packages/create-app/templates/default-app/plugins/README.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/chilled-goats-jump.md diff --git a/.changeset/chilled-goats-jump.md b/.changeset/chilled-goats-jump.md new file mode 100644 index 0000000000..3dd152712a --- /dev/null +++ b/.changeset/chilled-goats-jump.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': patch +--- + +Correct command to create new plugins diff --git a/packages/create-app/templates/default-app/plugins/README.md b/packages/create-app/templates/default-app/plugins/README.md index 58dc32cb08..d7865fdba3 100644 --- a/packages/create-app/templates/default-app/plugins/README.md +++ b/packages/create-app/templates/default-app/plugins/README.md @@ -4,6 +4,6 @@ This is where your own plugins and their associated modules live, each in a separate folder of its own. If you want to create a new plugin here, go to your project root directory, run -the command `yarn backstage-cli create`, and follow the on-screen instructions. +the command `yarn new`, and follow the on-screen instructions. You can also check out existing plugins on [the plugin marketplace](https://backstage.io/plugins)! From 3d72bdb41c7e59f6f44e587729ae20e095423933 Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Tue, 2 May 2023 13:45:16 -0500 Subject: [PATCH 02/32] upgrade aws-sdk to v3 for search-backend-module-elasticsearch This also enables opensearch serverless support Signed-off-by: Andrew Ochsner --- .changeset/fluffy-crabs-reply.md | 5 + .../api-report.md | 8 +- .../package.json | 6 +- .../src/engines/ElasticSearchClientOptions.ts | 2 + .../ElasticSearchClientWrapper.test.ts | 1 + .../src/engines/ElasticSearchSearchEngine.ts | 24 +- yarn.lock | 242 ++++++++++++++++-- 7 files changed, 256 insertions(+), 32 deletions(-) create mode 100644 .changeset/fluffy-crabs-reply.md diff --git a/.changeset/fluffy-crabs-reply.md b/.changeset/fluffy-crabs-reply.md new file mode 100644 index 0000000000..5570ef0492 --- /dev/null +++ b/.changeset/fluffy-crabs-reply.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': minor +--- + +Upgrade to aws-sdk v3 and include Opensearch Serverless support diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index c7add1e8cb..b567414b22 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -8,8 +8,8 @@ import { ApiResponse } from '@opensearch-project/opensearch'; import { ApiResponse as ApiResponse_2 } from '@elastic/elasticsearch'; import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; -import { BulkHelper } from '@opensearch-project/opensearch/lib/Helpers'; -import { BulkStats } from '@opensearch-project/opensearch/lib/Helpers'; +import { BulkHelper } from '@elastic/elasticsearch/lib/Helpers'; +import { BulkStats } from '@elastic/elasticsearch/lib/Helpers'; import { Config } from '@backstage/config'; import type { ConnectionOptions } from 'tls'; import { IndexableDocument } from '@backstage/plugin-search-common'; @@ -424,6 +424,10 @@ export interface OpenSearchElasticSearchClientOptions nodes?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[]; // (undocumented) provider?: 'aws' | 'opensearch'; + // (undocumented) + region?: string; + // (undocumented) + service?: 'es' | 'aoss'; } // @public (undocumented) diff --git a/plugins/search-backend-module-elasticsearch/package.json b/plugins/search-backend-module-elasticsearch/package.json index 73c3a5d7dc..76f3265c31 100644 --- a/plugins/search-backend-module-elasticsearch/package.json +++ b/plugins/search-backend-module-elasticsearch/package.json @@ -36,15 +36,15 @@ "clean": "backstage-cli package clean" }, "dependencies": { + "@aws-sdk/credential-provider-node": "^3.321.1", "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@elastic/elasticsearch": "^7.13.0", - "@opensearch-project/opensearch": "^2.0.0", - "aws-os-connection": "^0.2.0", - "aws-sdk": "^2.948.0", + "@opensearch-project/opensearch": "^2.2.1", + "aws4": "^1.12.0", "elastic-builder": "^2.16.0", "lodash": "^4.17.21", "uuid": "^8.3.2", diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts index 8cb2f67fce..2974f03331 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts @@ -51,6 +51,8 @@ export type ElasticSearchClientOptions = export interface OpenSearchElasticSearchClientOptions extends BaseElasticSearchClientOptions { provider?: 'aws' | 'opensearch'; + region?: string; + service?: 'es' | 'aoss'; auth?: OpenSearchAuth; connection?: OpenSearchConnectionConstructor; node?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[]; diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts index 7355daf3b9..620d894735 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts @@ -229,6 +229,7 @@ describe('ElasticSearchClientWrapper', () => { osOptions = { provider: 'aws', node: 'https://my-es-cluster.eu-west-1.es.amazonaws.com', + region: 'eu-west-1', // todo(backstage/techdocs-core): Remove the following ts-ignore when // @short.io/opensearch-mock is updated to work w/opensearch >= 2.0.0 // @ts-ignore diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 3294ca9d33..a13fbaf92a 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -21,9 +21,11 @@ import { SearchEngine, SearchQuery, } from '@backstage/plugin-search-common'; -import { awsGetCredentials, createAWSConnection } from 'aws-os-connection'; import { isEmpty, isNumber, isNaN as nan } from 'lodash'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; // V3 SDK. +import { AwsSigv4Signer } from '@opensearch-project/opensearch/aws'; +import { RequestSigner } from 'aws4'; import { Config } from '@backstage/config'; import { ElasticSearchClientOptions } from './ElasticSearchClientOptions'; import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; @@ -462,15 +464,14 @@ export async function createElasticSearchClientOptions( }; } if (config.getOptionalString('provider') === 'aws') { - const awsCredentials = await awsGetCredentials(); - const AWSConnection = createAWSConnection(awsCredentials); + const requestSigner = new RequestSigner(config.getString('node')); + const service = + config.getOptionalString('service') ?? requestSigner.service; + if (service !== 'es' && service !== 'aoss') + throw new Error(`Unrecognized serivce type: ${service}`); return { provider: 'aws', - // todo(backstage/techdocs-core): Remove the following ts-ignore when - // aws-os-connection is updated to work with opensearch >= 2.0.0 - // @ts-ignore node: config.getString('node'), - ...AWSConnection, ...(sslConfig ? { ssl: { @@ -479,6 +480,15 @@ export async function createElasticSearchClientOptions( }, } : {}), + ...AwsSigv4Signer({ + region: config.getOptionalString('region') ?? requestSigner.region, // for backwards compatibility + service: service, + getCredentials: () => { + // Any other method to acquire a new Credentials object can be used. + const credentialsProvider = defaultProvider(); + return credentialsProvider(); + }, + }), }; } if (config.getOptionalString('provider') === 'opensearch') { diff --git a/yarn.lock b/yarn.lock index 0bd97d81f8..27ee93b8ca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -741,6 +741,46 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-sso-oidc@npm:3.321.1": + version: 3.321.1 + resolution: "@aws-sdk/client-sso-oidc@npm:3.321.1" + dependencies: + "@aws-crypto/sha256-browser": 3.0.0 + "@aws-crypto/sha256-js": 3.0.0 + "@aws-sdk/config-resolver": 3.310.0 + "@aws-sdk/fetch-http-handler": 3.310.0 + "@aws-sdk/hash-node": 3.310.0 + "@aws-sdk/invalid-dependency": 3.310.0 + "@aws-sdk/middleware-content-length": 3.310.0 + "@aws-sdk/middleware-endpoint": 3.310.0 + "@aws-sdk/middleware-host-header": 3.310.0 + "@aws-sdk/middleware-logger": 3.310.0 + "@aws-sdk/middleware-recursion-detection": 3.310.0 + "@aws-sdk/middleware-retry": 3.310.0 + "@aws-sdk/middleware-serde": 3.310.0 + "@aws-sdk/middleware-stack": 3.310.0 + "@aws-sdk/middleware-user-agent": 3.319.0 + "@aws-sdk/node-config-provider": 3.310.0 + "@aws-sdk/node-http-handler": 3.321.1 + "@aws-sdk/protocol-http": 3.310.0 + "@aws-sdk/smithy-client": 3.316.0 + "@aws-sdk/types": 3.310.0 + "@aws-sdk/url-parser": 3.310.0 + "@aws-sdk/util-base64": 3.310.0 + "@aws-sdk/util-body-length-browser": 3.310.0 + "@aws-sdk/util-body-length-node": 3.310.0 + "@aws-sdk/util-defaults-mode-browser": 3.316.0 + "@aws-sdk/util-defaults-mode-node": 3.316.0 + "@aws-sdk/util-endpoints": 3.319.0 + "@aws-sdk/util-retry": 3.310.0 + "@aws-sdk/util-user-agent-browser": 3.310.0 + "@aws-sdk/util-user-agent-node": 3.310.0 + "@aws-sdk/util-utf8": 3.310.0 + tslib: ^2.5.0 + checksum: 25653c4d788b1cf95fa1246e82c6be4eba588bc5ff150c2b6b7495b73b08ad89efa75a9d52312079326feea56ebfd8f715150bd4a1bcf59dd2e0a282397ff09f + languageName: node + linkType: hard + "@aws-sdk/client-sso@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/client-sso@npm:3.310.0" @@ -781,6 +821,46 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-sso@npm:3.321.1": + version: 3.321.1 + resolution: "@aws-sdk/client-sso@npm:3.321.1" + dependencies: + "@aws-crypto/sha256-browser": 3.0.0 + "@aws-crypto/sha256-js": 3.0.0 + "@aws-sdk/config-resolver": 3.310.0 + "@aws-sdk/fetch-http-handler": 3.310.0 + "@aws-sdk/hash-node": 3.310.0 + "@aws-sdk/invalid-dependency": 3.310.0 + "@aws-sdk/middleware-content-length": 3.310.0 + "@aws-sdk/middleware-endpoint": 3.310.0 + "@aws-sdk/middleware-host-header": 3.310.0 + "@aws-sdk/middleware-logger": 3.310.0 + "@aws-sdk/middleware-recursion-detection": 3.310.0 + "@aws-sdk/middleware-retry": 3.310.0 + "@aws-sdk/middleware-serde": 3.310.0 + "@aws-sdk/middleware-stack": 3.310.0 + "@aws-sdk/middleware-user-agent": 3.319.0 + "@aws-sdk/node-config-provider": 3.310.0 + "@aws-sdk/node-http-handler": 3.321.1 + "@aws-sdk/protocol-http": 3.310.0 + "@aws-sdk/smithy-client": 3.316.0 + "@aws-sdk/types": 3.310.0 + "@aws-sdk/url-parser": 3.310.0 + "@aws-sdk/util-base64": 3.310.0 + "@aws-sdk/util-body-length-browser": 3.310.0 + "@aws-sdk/util-body-length-node": 3.310.0 + "@aws-sdk/util-defaults-mode-browser": 3.316.0 + "@aws-sdk/util-defaults-mode-node": 3.316.0 + "@aws-sdk/util-endpoints": 3.319.0 + "@aws-sdk/util-retry": 3.310.0 + "@aws-sdk/util-user-agent-browser": 3.310.0 + "@aws-sdk/util-user-agent-node": 3.310.0 + "@aws-sdk/util-utf8": 3.310.0 + tslib: ^2.5.0 + checksum: be867824663577684520e1abefb3d7989affabd8a89cbabd3bf0eb4d60bb7d47488b8e8fce4ac29b36c479fa97d27f2de6169d5ef98e748c7ac700fdedb41580 + languageName: node + linkType: hard + "@aws-sdk/client-sts@npm:3.312.0, @aws-sdk/client-sts@npm:^3.208.0": version: 3.312.0 resolution: "@aws-sdk/client-sts@npm:3.312.0" @@ -890,7 +970,24 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/credential-provider-node@npm:3.310.0, @aws-sdk/credential-provider-node@npm:^3.208.0": +"@aws-sdk/credential-provider-ini@npm:3.321.1": + version: 3.321.1 + resolution: "@aws-sdk/credential-provider-ini@npm:3.321.1" + dependencies: + "@aws-sdk/credential-provider-env": 3.310.0 + "@aws-sdk/credential-provider-imds": 3.310.0 + "@aws-sdk/credential-provider-process": 3.310.0 + "@aws-sdk/credential-provider-sso": 3.321.1 + "@aws-sdk/credential-provider-web-identity": 3.310.0 + "@aws-sdk/property-provider": 3.310.0 + "@aws-sdk/shared-ini-file-loader": 3.310.0 + "@aws-sdk/types": 3.310.0 + tslib: ^2.5.0 + checksum: ee3d615535efca5cfdd4749f23b02782a5f685289a114799dae46c84e03ba1ebf24ee3b846daaf3b042df43943bd132a83559cc76c4f261cdaadd1150f02a08f + languageName: node + linkType: hard + +"@aws-sdk/credential-provider-node@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/credential-provider-node@npm:3.310.0" dependencies: @@ -908,6 +1005,24 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-node@npm:^3.208.0, @aws-sdk/credential-provider-node@npm:^3.321.1": + version: 3.321.1 + resolution: "@aws-sdk/credential-provider-node@npm:3.321.1" + dependencies: + "@aws-sdk/credential-provider-env": 3.310.0 + "@aws-sdk/credential-provider-imds": 3.310.0 + "@aws-sdk/credential-provider-ini": 3.321.1 + "@aws-sdk/credential-provider-process": 3.310.0 + "@aws-sdk/credential-provider-sso": 3.321.1 + "@aws-sdk/credential-provider-web-identity": 3.310.0 + "@aws-sdk/property-provider": 3.310.0 + "@aws-sdk/shared-ini-file-loader": 3.310.0 + "@aws-sdk/types": 3.310.0 + tslib: ^2.5.0 + checksum: 899af9f4636b60be7a5dc022bf03d00bb5df8ff1b39214604d2157322766fc368c69457435bfb21a56fc5997a4b13feca32ec8bb53a57562ee2994a9cee4a968 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-process@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/credential-provider-process@npm:3.310.0" @@ -934,6 +1049,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-sso@npm:3.321.1": + version: 3.321.1 + resolution: "@aws-sdk/credential-provider-sso@npm:3.321.1" + dependencies: + "@aws-sdk/client-sso": 3.321.1 + "@aws-sdk/property-provider": 3.310.0 + "@aws-sdk/shared-ini-file-loader": 3.310.0 + "@aws-sdk/token-providers": 3.321.1 + "@aws-sdk/types": 3.310.0 + tslib: ^2.5.0 + checksum: a7c588394dbe29a2c9ade15dd5d151131963d1da4d14272432f397aa73aa551e5eb7bb5fb7829a628f136927ee689bfad579b7adc18bbc02d7f47e37d745105e + languageName: node + linkType: hard + "@aws-sdk/credential-provider-web-identity@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/credential-provider-web-identity@npm:3.310.0" @@ -1326,6 +1455,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/middleware-user-agent@npm:3.319.0": + version: 3.319.0 + resolution: "@aws-sdk/middleware-user-agent@npm:3.319.0" + dependencies: + "@aws-sdk/protocol-http": 3.310.0 + "@aws-sdk/types": 3.310.0 + "@aws-sdk/util-endpoints": 3.319.0 + tslib: ^2.5.0 + checksum: a8bcb55eb774a96bc7018fe59ac5c3728743e538f7202815887daeebf3186b102835ac5b3bfd7addea1c3215f04ad8017d19d3f00cfe5635cf0eb4645b379c40 + languageName: node + linkType: hard + "@aws-sdk/node-config-provider@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/node-config-provider@npm:3.310.0" @@ -1338,7 +1479,7 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/node-http-handler@npm:3.310.0, @aws-sdk/node-http-handler@npm:^3.208.0": +"@aws-sdk/node-http-handler@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/node-http-handler@npm:3.310.0" dependencies: @@ -1351,6 +1492,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/node-http-handler@npm:3.321.1, @aws-sdk/node-http-handler@npm:^3.208.0": + version: 3.321.1 + resolution: "@aws-sdk/node-http-handler@npm:3.321.1" + dependencies: + "@aws-sdk/abort-controller": 3.310.0 + "@aws-sdk/protocol-http": 3.310.0 + "@aws-sdk/querystring-builder": 3.310.0 + "@aws-sdk/types": 3.310.0 + tslib: ^2.5.0 + checksum: b8546b1b20572e1ca02d3792319967712b40c925e4e06f1fb098d70ce55e32a15e514d655e8809650349567e1f46056dbbc3fb9f55df1257c8619127ca01f8fe + languageName: node + linkType: hard + "@aws-sdk/property-provider@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/property-provider@npm:3.310.0" @@ -1452,6 +1606,17 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/smithy-client@npm:3.316.0": + version: 3.316.0 + resolution: "@aws-sdk/smithy-client@npm:3.316.0" + dependencies: + "@aws-sdk/middleware-stack": 3.310.0 + "@aws-sdk/types": 3.310.0 + tslib: ^2.5.0 + checksum: 2a42969efdf6f3e2383aae3f400f9d21623864ffefb4b83da275866621cfa216315e13e4629c5683a280ee61e3bd9908e90b1f4623fbb51fab212fa492a68226 + languageName: node + linkType: hard + "@aws-sdk/token-providers@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/token-providers@npm:3.310.0" @@ -1465,6 +1630,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/token-providers@npm:3.321.1": + version: 3.321.1 + resolution: "@aws-sdk/token-providers@npm:3.321.1" + dependencies: + "@aws-sdk/client-sso-oidc": 3.321.1 + "@aws-sdk/property-provider": 3.310.0 + "@aws-sdk/shared-ini-file-loader": 3.310.0 + "@aws-sdk/types": 3.310.0 + tslib: ^2.5.0 + checksum: 2a4af584a5bdaa131ee85d62f67cef210d096f558064923c109f8773b402ef2c4830bba3486422b55f33e31371c794dbaecb07cf3d889250704bd815491569df + languageName: node + linkType: hard + "@aws-sdk/types@npm:3.310.0, @aws-sdk/types@npm:^3.208.0, @aws-sdk/types@npm:^3.222.0": version: 3.310.0 resolution: "@aws-sdk/types@npm:3.310.0" @@ -1553,6 +1731,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/util-defaults-mode-browser@npm:3.316.0": + version: 3.316.0 + resolution: "@aws-sdk/util-defaults-mode-browser@npm:3.316.0" + dependencies: + "@aws-sdk/property-provider": 3.310.0 + "@aws-sdk/types": 3.310.0 + bowser: ^2.11.0 + tslib: ^2.5.0 + checksum: abd989642a7f04dd87fef911623820e2f2e80932a663b157a716eb87907fbea99c28cbf706c5284b019db71641b464c8ae1448dc22ebfa9e4b3e6d627f800522 + languageName: node + linkType: hard + "@aws-sdk/util-defaults-mode-node@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/util-defaults-mode-node@npm:3.310.0" @@ -1567,6 +1757,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/util-defaults-mode-node@npm:3.316.0": + version: 3.316.0 + resolution: "@aws-sdk/util-defaults-mode-node@npm:3.316.0" + dependencies: + "@aws-sdk/config-resolver": 3.310.0 + "@aws-sdk/credential-provider-imds": 3.310.0 + "@aws-sdk/node-config-provider": 3.310.0 + "@aws-sdk/property-provider": 3.310.0 + "@aws-sdk/types": 3.310.0 + tslib: ^2.5.0 + checksum: 28408b0ab9717750774d642d30ddf2b547093837bb69706bd2d0d9dc11f1d331c98c0f44412fc1b361b9a17b36f9bb970225b45ed0ec42ed17d2af242d864106 + languageName: node + linkType: hard + "@aws-sdk/util-endpoints@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/util-endpoints@npm:3.310.0" @@ -1577,6 +1781,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/util-endpoints@npm:3.319.0": + version: 3.319.0 + resolution: "@aws-sdk/util-endpoints@npm:3.319.0" + dependencies: + "@aws-sdk/types": 3.310.0 + tslib: ^2.5.0 + checksum: 7b3e67494bab98f7e979fed0e172ea2179f29184fffea0954e0173f89128367996c09ee9bb2a86b500f8d44ad024b4efc550610b2daa943e223b2050c7a32b56 + languageName: node + linkType: hard + "@aws-sdk/util-hex-encoding@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/util-hex-encoding@npm:3.310.0" @@ -8402,6 +8616,7 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-search-backend-module-elasticsearch@workspace:plugins/search-backend-module-elasticsearch" dependencies: + "@aws-sdk/credential-provider-node": ^3.321.1 "@backstage/backend-common": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" "@backstage/cli": "workspace:^" @@ -8410,10 +8625,9 @@ __metadata: "@backstage/plugin-search-common": "workspace:^" "@elastic/elasticsearch": ^7.13.0 "@elastic/elasticsearch-mock": ^1.0.0 - "@opensearch-project/opensearch": ^2.0.0 + "@opensearch-project/opensearch": ^2.2.1 "@short.io/opensearch-mock": ^0.3.1 - aws-os-connection: ^0.2.0 - aws-sdk: ^2.948.0 + aws4: ^1.12.0 elastic-builder: ^2.16.0 lodash: ^4.17.21 uuid: ^8.3.2 @@ -13447,7 +13661,7 @@ __metadata: languageName: node linkType: hard -"@opensearch-project/opensearch@npm:^2.0.0": +"@opensearch-project/opensearch@npm:^2.2.1": version: 2.2.1 resolution: "@opensearch-project/opensearch@npm:2.2.1" dependencies: @@ -18377,18 +18591,6 @@ __metadata: languageName: node linkType: hard -"aws-os-connection@npm:^0.2.0": - version: 0.2.0 - resolution: "aws-os-connection@npm:0.2.0" - dependencies: - aws4: ^1.11.0 - peerDependencies: - "@opensearch-project/opensearch": 2 - aws-sdk: ">=2" - checksum: 1708ffd9bdc5f7bd6b8447f97859936a8657f4df567f61f12dacd040e6f1a25020bbea8027bb28f19fe27d171a0d3a5090c2d84d6f752bb04e387640413fec7c - languageName: node - linkType: hard - "aws-sdk-client-mock-jest@npm:^2.0.0": version: 2.1.1 resolution: "aws-sdk-client-mock-jest@npm:2.1.1" @@ -18423,7 +18625,7 @@ __metadata: languageName: node linkType: hard -"aws-sdk@npm:^2.1231.0, aws-sdk@npm:^2.840.0, aws-sdk@npm:^2.948.0": +"aws-sdk@npm:^2.1231.0, aws-sdk@npm:^2.840.0": version: 2.1279.0 resolution: "aws-sdk@npm:2.1279.0" dependencies: @@ -18448,7 +18650,7 @@ __metadata: languageName: node linkType: hard -"aws4@npm:^1.11.0, aws4@npm:^1.8.0": +"aws4@npm:^1.11.0, aws4@npm:^1.12.0, aws4@npm:^1.8.0": version: 1.12.0 resolution: "aws4@npm:1.12.0" checksum: 68f79708ac7c335992730bf638286a3ee0a645cf12575d557860100767c500c08b30e24726b9f03265d74116417f628af78509e1333575e9f8d52a80edfe8cbc From 0f5ef03794e34e30644ba5c12c2e93a7919dec40 Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Tue, 2 May 2023 14:23:29 -0500 Subject: [PATCH 03/32] spell OpenSearch and accept it Signed-off-by: Andrew Ochsner --- .changeset/fluffy-crabs-reply.md | 2 +- .github/vale/Vocab/Backstage/accept.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.changeset/fluffy-crabs-reply.md b/.changeset/fluffy-crabs-reply.md index 5570ef0492..0665dd864d 100644 --- a/.changeset/fluffy-crabs-reply.md +++ b/.changeset/fluffy-crabs-reply.md @@ -2,4 +2,4 @@ '@backstage/plugin-search-backend-module-elasticsearch': minor --- -Upgrade to aws-sdk v3 and include Opensearch Serverless support +Upgrade to aws-sdk v3 and include OpenSearch Serverless support diff --git a/.github/vale/Vocab/Backstage/accept.txt b/.github/vale/Vocab/Backstage/accept.txt index 83fba8aa56..cbb2279e1b 100644 --- a/.github/vale/Vocab/Backstage/accept.txt +++ b/.github/vale/Vocab/Backstage/accept.txt @@ -247,6 +247,7 @@ Olausson Oldsberg onboarding Onboarding +OpenSearch OpenShift orgs padding From 12a345317ab19b5489b8bdaadd74e557fa398aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 3 May 2023 13:17:03 +0200 Subject: [PATCH 04/32] Slightly optimize the facets request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/honest-turkeys-learn.md | 5 +++++ .../src/service/DefaultEntitiesCatalog.test.ts | 11 +++++++++++ .../src/service/DefaultEntitiesCatalog.ts | 5 ++--- 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 .changeset/honest-turkeys-learn.md diff --git a/.changeset/honest-turkeys-learn.md b/.changeset/honest-turkeys-learn.md new file mode 100644 index 0000000000..e4e6aae5fc --- /dev/null +++ b/.changeset/honest-turkeys-learn.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Remove unnecessary join in the entity facets endpoint, exclude nulls diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 93a37fb04a..4f011813e6 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -1729,6 +1729,17 @@ describe('DefaultEntitiesCatalog', () => { ], }, }); + + await expect( + catalog.facets({ + facets: ['kind'], + filter: { not: { key: 'metadata.name', values: ['two'] } }, + }), + ).resolves.toEqual({ + facets: { + kind: [{ value: 'k', count: 1 }], + }, + }); }, ); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 5fca950cac..8affa78991 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -678,10 +678,9 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { for (const facet of request.facets) { const dbQuery = db('search') - .join('final_entities', 'search.entity_id', 'final_entities.entity_id') .where('search.key', facet.toLocaleLowerCase('en-US')) - .count('search.entity_id as count') - .select({ value: 'search.original_value' }) + .whereNotNull('search.original_value') + .select({ value: 'search.original_value', count: db.raw('count(*)') }) .groupBy('search.original_value'); if (request?.filter) { From b01684d280ff1843f3aab29a3ba531332bf58df2 Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Wed, 3 May 2023 16:34:33 -0500 Subject: [PATCH 05/32] use integration-aws-node credential provider Signed-off-by: Andrew Ochsner --- .../package.json | 1 + .../ElasticSearchClientWrapper.test.ts | 14 +- .../src/engines/ElasticSearchSearchEngine.ts | 214 +++++++++--------- yarn.lock | 1 + 4 files changed, 115 insertions(+), 115 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/package.json b/plugins/search-backend-module-elasticsearch/package.json index 76f3265c31..0d4f08e46c 100644 --- a/plugins/search-backend-module-elasticsearch/package.json +++ b/plugins/search-backend-module-elasticsearch/package.json @@ -40,6 +40,7 @@ "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/integration-aws-node": "workspace:^", "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@elastic/elasticsearch": "^7.13.0", diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts index 620d894735..75f2e36b2b 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts @@ -14,17 +14,13 @@ * limitations under the License. */ -import { ConfigReader } from '@backstage/config'; import { Client as ElasticSearchClient } from '@elastic/elasticsearch'; import { Client as OpenSearchClient } from '@opensearch-project/opensearch'; import Mock from '@short.io/opensearch-mock'; import { Readable } from 'stream'; import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; -import { - createElasticSearchClientOptions, - ElasticSearchClientOptions, -} from './ElasticSearchSearchEngine'; +import { ElasticSearchClientOptions } from './ElasticSearchSearchEngine'; jest.mock('@elastic/elasticsearch', () => ({ ...jest.requireActual('@elastic/elasticsearch'), @@ -103,11 +99,9 @@ describe('ElasticSearchClientWrapper', () => { let esOptions: ElasticSearchClientOptions; beforeEach(async () => { - esOptions = await createElasticSearchClientOptions( - new ConfigReader({ - node: 'http://localhost:9200', - }), - ); + esOptions = { + node: 'http://localhost:9200', + }; jest.clearAllMocks(); }); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index a13fbaf92a..baeeffd765 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -23,7 +23,6 @@ import { } from '@backstage/plugin-search-common'; import { isEmpty, isNumber, isNaN as nan } from 'lodash'; -import { defaultProvider } from '@aws-sdk/credential-provider-node'; // V3 SDK. import { AwsSigv4Signer } from '@opensearch-project/opensearch/aws'; import { RequestSigner } from 'aws4'; import { Config } from '@backstage/config'; @@ -35,6 +34,10 @@ import { Logger } from 'winston'; import { MissingIndexError } from '@backstage/plugin-search-backend-node'; import esb from 'elastic-builder'; import { v4 as uuid } from 'uuid'; +import { + AwsCredentialsManager, + DefaultAwsCredentialsManager, +} from '@backstage/integration-aws-node'; export type { ElasticSearchClientOptions }; @@ -118,6 +121,7 @@ const DEFAULT_INDEXER_BATCH_SIZE = 1000; export class ElasticSearchSearchEngine implements SearchEngine { private readonly elasticSearchClientWrapper: ElasticSearchClientWrapper; private readonly highlightOptions: ElasticSearchHighlightConfig; + private static credentialProvider: AwsCredentialsManager; constructor( private readonly elasticSearchClientOptions: ElasticSearchClientOptions, @@ -147,8 +151,8 @@ export class ElasticSearchSearchEngine implements SearchEngine { aliasPostfix = `search`, indexPrefix = ``, } = options; - - const clientOptions = await createElasticSearchClientOptions( + this.credentialProvider = DefaultAwsCredentialsManager.fromConfig(config); + const clientOptions = await this.createElasticSearchClientOptions( config.getConfig('search.elasticsearch'), ); if (clientOptions.provider === 'elastic') { @@ -414,6 +418,108 @@ export class ElasticSearchSearchEngine implements SearchEngine { const postFix = this.aliasPostfix ? `__${this.aliasPostfix}` : ''; return `${this.indexPrefix}${type}${postFix}`; } + + private static async createElasticSearchClientOptions( + config?: Config, + ): Promise { + if (!config) { + throw new Error('No elastic search config found'); + } + const clientOptionsConfig = config.getOptionalConfig('clientOptions'); + const sslConfig = clientOptionsConfig?.getOptionalConfig('ssl'); + + if (config.getOptionalString('provider') === 'elastic') { + const authConfig = config.getConfig('auth'); + return { + provider: 'elastic', + cloud: { + id: config.getString('cloudId'), + }, + auth: { + username: authConfig.getString('username'), + password: authConfig.getString('password'), + }, + ...(sslConfig + ? { + ssl: { + rejectUnauthorized: + sslConfig?.getOptionalBoolean('rejectUnauthorized'), + }, + } + : {}), + }; + } + if (config.getOptionalString('provider') === 'aws') { + const requestSigner = new RequestSigner(config.getString('node')); + const service = + config.getOptionalString('service') ?? requestSigner.service; + if (service !== 'es' && service !== 'aoss') + throw new Error(`Unrecognized serivce type: ${service}`); + return { + provider: 'aws', + node: config.getString('node'), + ...(sslConfig + ? { + ssl: { + rejectUnauthorized: + sslConfig?.getOptionalBoolean('rejectUnauthorized'), + }, + } + : {}), + ...AwsSigv4Signer({ + region: config.getOptionalString('region') ?? requestSigner.region, // for backwards compatibility + service: service, + getCredentials: async () => { + const provider = + await this.credentialProvider.getCredentialProvider(); + return await provider.sdkCredentialProvider(); + }, + }), + }; + } + if (config.getOptionalString('provider') === 'opensearch') { + const authConfig = config.getConfig('auth'); + return { + provider: 'opensearch', + node: config.getString('node'), + auth: { + username: authConfig.getString('username'), + password: authConfig.getString('password'), + }, + ...(sslConfig + ? { + ssl: { + rejectUnauthorized: + sslConfig?.getOptionalBoolean('rejectUnauthorized'), + }, + } + : {}), + }; + } + const authConfig = config.getOptionalConfig('auth'); + const auth = + authConfig && + (authConfig.has('apiKey') + ? { + apiKey: authConfig.getString('apiKey'), + } + : { + username: authConfig.getString('username'), + password: authConfig.getString('password'), + }); + return { + node: config.getString('node'), + auth, + ...(sslConfig + ? { + ssl: { + rejectUnauthorized: + sslConfig?.getOptionalBoolean('rejectUnauthorized'), + }, + } + : {}), + }; + } } /** @@ -432,105 +538,3 @@ export function decodePageCursor(pageCursor?: string): { page: number } { export function encodePageCursor({ page }: { page: number }): string { return Buffer.from(`${page}`, 'utf-8').toString('base64'); } - -export async function createElasticSearchClientOptions( - config?: Config, -): Promise { - if (!config) { - throw new Error('No elastic search config found'); - } - const clientOptionsConfig = config.getOptionalConfig('clientOptions'); - const sslConfig = clientOptionsConfig?.getOptionalConfig('ssl'); - - if (config.getOptionalString('provider') === 'elastic') { - const authConfig = config.getConfig('auth'); - return { - provider: 'elastic', - cloud: { - id: config.getString('cloudId'), - }, - auth: { - username: authConfig.getString('username'), - password: authConfig.getString('password'), - }, - ...(sslConfig - ? { - ssl: { - rejectUnauthorized: - sslConfig?.getOptionalBoolean('rejectUnauthorized'), - }, - } - : {}), - }; - } - if (config.getOptionalString('provider') === 'aws') { - const requestSigner = new RequestSigner(config.getString('node')); - const service = - config.getOptionalString('service') ?? requestSigner.service; - if (service !== 'es' && service !== 'aoss') - throw new Error(`Unrecognized serivce type: ${service}`); - return { - provider: 'aws', - node: config.getString('node'), - ...(sslConfig - ? { - ssl: { - rejectUnauthorized: - sslConfig?.getOptionalBoolean('rejectUnauthorized'), - }, - } - : {}), - ...AwsSigv4Signer({ - region: config.getOptionalString('region') ?? requestSigner.region, // for backwards compatibility - service: service, - getCredentials: () => { - // Any other method to acquire a new Credentials object can be used. - const credentialsProvider = defaultProvider(); - return credentialsProvider(); - }, - }), - }; - } - if (config.getOptionalString('provider') === 'opensearch') { - const authConfig = config.getConfig('auth'); - return { - provider: 'opensearch', - node: config.getString('node'), - auth: { - username: authConfig.getString('username'), - password: authConfig.getString('password'), - }, - ...(sslConfig - ? { - ssl: { - rejectUnauthorized: - sslConfig?.getOptionalBoolean('rejectUnauthorized'), - }, - } - : {}), - }; - } - const authConfig = config.getOptionalConfig('auth'); - const auth = - authConfig && - (authConfig.has('apiKey') - ? { - apiKey: authConfig.getString('apiKey'), - } - : { - username: authConfig.getString('username'), - password: authConfig.getString('password'), - }); - return { - node: config.getString('node'), - auth, - ...(sslConfig - ? { - ssl: { - rejectUnauthorized: - sslConfig?.getOptionalBoolean('rejectUnauthorized'), - }, - } - : {}), - }; -} diff --git a/yarn.lock b/yarn.lock index fbc069a7e0..d9de87d0d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8622,6 +8622,7 @@ __metadata: "@backstage/backend-plugin-api": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/integration-aws-node": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@elastic/elasticsearch": ^7.13.0 From 906dc9223c8897f5e5f877f87a528c2a327ef739 Mon Sep 17 00:00:00 2001 From: Zander Franks Date: Wed, 3 May 2023 17:35:16 -0500 Subject: [PATCH 06/32] feat(scaffolder): support text outputs from templates Signed-off-by: Zander Franks --- .../src/Template.v1beta3.schema.json | 27 +++++++++ plugins/scaffolder-react/src/api/types.ts | 8 +++ .../DefaultTemplateOutputs.tsx | 52 +++++++++++++---- .../TemplateOutputs/TextOutputs.tsx | 58 +++++++++++++++++++ 4 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx diff --git a/plugins/scaffolder-common/src/Template.v1beta3.schema.json b/plugins/scaffolder-common/src/Template.v1beta3.schema.json index beb0ca5e31..beb86f43cb 100644 --- a/plugins/scaffolder-common/src/Template.v1beta3.schema.json +++ b/plugins/scaffolder-common/src/Template.v1beta3.schema.json @@ -218,6 +218,33 @@ } } } + }, + "text": { + "type": "array", + "description": "A list of text blobs, like output data from the template.", + "items": { + "type": "object", + "required": [], + "properties": { + "title": { + "type": "string", + "description": "A user friendly display name for the text blob.", + "examples": ["Output Data"], + "minLength": 1 + }, + "icon": { + "type": "string", + "description": "A key representing a visual icon to be displayed in the UI.", + "examples": ["dashboard"], + "minLength": 1 + }, + "data": { + "type": "string", + "description": "The text blob to display in the UI.", + "examples": ["{}"] + } + } + } } }, "additionalProperties": { diff --git a/plugins/scaffolder-react/src/api/types.ts b/plugins/scaffolder-react/src/api/types.ts index b39555c14f..64bb0d05ac 100644 --- a/plugins/scaffolder-react/src/api/types.ts +++ b/plugins/scaffolder-react/src/api/types.ts @@ -84,9 +84,17 @@ export type ScaffolderOutputLink = { entityRef?: string; }; +/** @public */ +export type ScaffolderOutputText = { + title?: string; + icon?: string; + data?: string; +}; + /** @public */ export type ScaffolderTaskOutput = { links?: ScaffolderOutputLink[]; + text?: ScaffolderOutputText[]; } & { [key: string]: unknown; }; diff --git a/plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.tsx b/plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.tsx index 4d908461b0..de52b62700 100644 --- a/plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.tsx +++ b/plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.tsx @@ -13,10 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React from 'react'; -import { Box, Paper } from '@material-ui/core'; -import { LinkOutputs } from './LinkOutputs'; +import { InfoCard, MarkdownContent } from '@backstage/core-components'; import { ScaffolderTaskOutput } from '@backstage/plugin-scaffolder-react'; +import { Box, Paper } from '@material-ui/core'; +import React, { useMemo, useState } from 'react'; +import { LinkOutputs } from './LinkOutputs'; +import { TextOutputs } from './TextOutputs'; /** * The DefaultOutputs renderer for the scaffolder task output @@ -26,17 +28,47 @@ import { ScaffolderTaskOutput } from '@backstage/plugin-scaffolder-react'; export const DefaultTemplateOutputs = (props: { output?: ScaffolderTaskOutput; }) => { - if (!props.output?.links) { + const [textOutputIndex, setTextOutputIndex] = useState(); + + const textOutput = useMemo( + () => + textOutputIndex !== undefined + ? props.output?.text?.[textOutputIndex] + : null, + [props.output, textOutputIndex], + ); + + if (!props.output) { return null; } return ( - - - - + <> + + + + + + + + + {textOutput ? ( + + + + + + - - + ) : null} + ); }; diff --git a/plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx b/plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx new file mode 100644 index 0000000000..8600a281a0 --- /dev/null +++ b/plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx @@ -0,0 +1,58 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { IconComponent, useApp } from '@backstage/core-plugin-api'; +import { Button } from '@material-ui/core'; +import WebIcon from '@material-ui/icons/Web'; +import React from 'react'; +import { ScaffolderTaskOutput } from '../../../api'; + +export const TextOutputs = (props: { + output: ScaffolderTaskOutput; + index?: number; + setIndex?: (index: number | undefined) => void; +}) => { + const { + output: { text = [] }, + index, + setIndex, + } = props; + + const app = useApp(); + + const iconResolver = (key?: string): IconComponent => + app.getSystemIcon(key!) ?? WebIcon; + + return ( + <> + {text + .filter(({ data }) => data) + .map(({ title, icon }, i) => { + const Icon = iconResolver(icon); + return ( + + ); + })} + + ); +}; From 82e10a6939c2a9fbf641393598972d30d634f57d Mon Sep 17 00:00:00 2001 From: Zander Franks Date: Wed, 3 May 2023 17:42:07 -0500 Subject: [PATCH 07/32] chore: add changesets Signed-off-by: Zander Franks --- .changeset/heavy-penguins-burn.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/heavy-penguins-burn.md diff --git a/.changeset/heavy-penguins-burn.md b/.changeset/heavy-penguins-burn.md new file mode 100644 index 0000000000..f9fba9bfb8 --- /dev/null +++ b/.changeset/heavy-penguins-burn.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-scaffolder-common': patch +'@backstage/plugin-scaffolder-react': patch +--- + +Add support for Markdown text blob outputs from templates From d2727a2eb0024e9e9c7d8bffa0edee1517b1fefc Mon Sep 17 00:00:00 2001 From: Zander Franks Date: Wed, 3 May 2023 17:57:32 -0500 Subject: [PATCH 08/32] chore: add docs Signed-off-by: Zander Franks --- docs/features/software-templates/writing-templates.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md index 15e6a833cc..da8e54875b 100644 --- a/docs/features/software-templates/writing-templates.md +++ b/docs/features/software-templates/writing-templates.md @@ -510,10 +510,8 @@ take a look at, or you can Each individual step can output some variables that can be used in the scaffolder frontend for after the job is finished. This is useful for things -like linking to the entity that has been created with the backend, and also -linking to the created repository. - -The main two that are used are the following: +like linking to the entity that has been created with the backend, linking +to the created repository, or showing Markdown text blobs. ```yaml output: @@ -523,6 +521,10 @@ output: - title: Open in catalog icon: catalog entityRef: ${{ steps['register'].output.entityRef }} # link to the entity that has been ingested to the catalog + text: + - title: More data + data: | + Access the [remote repository](${{ steps['publish'].output.remoteUrl }}). ``` ## The templating syntax From 8c444f37d8156350673c9a670416a5ca556469de Mon Sep 17 00:00:00 2001 From: Zander Franks Date: Wed, 3 May 2023 18:19:09 -0500 Subject: [PATCH 09/32] chore: scaffolder api report Signed-off-by: Zander Franks --- plugins/scaffolder-react/api-report.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/scaffolder-react/api-report.md b/plugins/scaffolder-react/api-report.md index baf5afb312..a2a7dca3b2 100644 --- a/plugins/scaffolder-react/api-report.md +++ b/plugins/scaffolder-react/api-report.md @@ -217,6 +217,13 @@ export type ScaffolderOutputLink = { entityRef?: string; }; +// @public (undocumented) +export type ScaffolderOutputText = { + title?: string; + icon?: string; + data?: string; +}; + // @public export interface ScaffolderScaffoldOptions { // (undocumented) @@ -261,6 +268,7 @@ export type ScaffolderTask = { // @public (undocumented) export type ScaffolderTaskOutput = { links?: ScaffolderOutputLink[]; + text?: ScaffolderOutputText[]; } & { [key: string]: unknown; }; From ebf3c0e7a1f7249063d2cd7f2924c0ed901885aa Mon Sep 17 00:00:00 2001 From: Zander Franks Date: Wed, 3 May 2023 18:39:09 -0500 Subject: [PATCH 10/32] chore: fix docs Markdown example link Signed-off-by: Zander Franks --- docs/features/software-templates/writing-templates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md index da8e54875b..2508f4a592 100644 --- a/docs/features/software-templates/writing-templates.md +++ b/docs/features/software-templates/writing-templates.md @@ -524,7 +524,7 @@ output: text: - title: More data data: | - Access the [remote repository](${{ steps['publish'].output.remoteUrl }}). + Access the \[remote repository\](${{ steps['publish'].output.remoteUrl }}). ``` ## The templating syntax From 877df261085b2047a5211cfab86bcc59756b6ffa Mon Sep 17 00:00:00 2001 From: Sebastian Paredero Date: Thu, 4 May 2023 10:59:17 -0300 Subject: [PATCH 11/32] feat(plugins/azure-devops): support for multiple builds-definitions Signed-off-by: Sebastian Paredero --- .changeset/quiet-stingrays-fly.md | 5 +++++ plugins/azure-devops/README.md | 2 +- .../azure-devops/src/api/AzureDevOpsClient.ts | 18 +++++++++++++++++- 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 .changeset/quiet-stingrays-fly.md diff --git a/.changeset/quiet-stingrays-fly.md b/.changeset/quiet-stingrays-fly.md new file mode 100644 index 0000000000..536ee18a6f --- /dev/null +++ b/.changeset/quiet-stingrays-fly.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-azure-devops': minor +--- + +The getBuildRuns function now checks contains multiple comma-separated builds and splits them to send multiple requests for each and concatenates the results. diff --git a/plugins/azure-devops/README.md b/plugins/azure-devops/README.md index 5079fa4f16..7eeae09813 100644 --- a/plugins/azure-devops/README.md +++ b/plugins/azure-devops/README.md @@ -70,7 +70,7 @@ dev.azure.com/project: dev.azure.com/build-definition: ``` -In this case `` will be the name of your Team Project and `` will be the name of the Build Definition you would like to see Builds for. If the Build Definition name has spaces in it make sure to put quotes around it +In this case `` will be the name of your Team Project and `` will be the name of the Build Definition you would like to see Builds for, and it's possible to add more Builds separated by a comma. If the Build Definition name has spaces in it make sure to put quotes around it. ### Azure Pipelines Component diff --git a/plugins/azure-devops/src/api/AzureDevOpsClient.ts b/plugins/azure-devops/src/api/AzureDevOpsClient.ts index 68d8896538..a202008798 100644 --- a/plugins/azure-devops/src/api/AzureDevOpsClient.ts +++ b/plugins/azure-devops/src/api/AzureDevOpsClient.ts @@ -120,7 +120,23 @@ export class AzureDevOpsClient implements AzureDevOpsApi { queryString.append('repoName', repoName); } if (definitionName) { - queryString.append('definitionName', definitionName); + const definitionNames = definitionName.split(','); + if (definitionNames.length > 1) { + const buildRuns: BuildRun[] = []; + for (const name of definitionNames) { + queryString.set('definitionName', name.trim()); + if (options?.top) { + queryString.set('top', options.top.toString()); + } + const urlSegment = `builds/${encodeURIComponent( + projectName, + )}?${queryString}`; + const items = await this.get(urlSegment); + buildRuns.push(...items); + } + return { items: buildRuns }; + } + queryString.append('definitionName', definitionName.trim()); } if (options?.top) { queryString.append('top', options.top.toString()); From 970678adbe2d7b40cf8321741d6a7262ddd8f1e9 Mon Sep 17 00:00:00 2001 From: Phil Kuang Date: Wed, 19 Apr 2023 14:21:56 -0400 Subject: [PATCH 12/32] feat(GithubMultiOrgEntityProvider): support events Signed-off-by: Phil Kuang --- .changeset/unlucky-deers-teach.md | 5 + docs/integrations/github/org.md | 72 + .../api-report.md | 10 +- .../src/lib/github.test.ts | 32 + .../src/lib/github.ts | 37 + .../GithubMultiOrgEntityProvider.test.ts | 1249 ++++++++++++++++- .../providers/GithubMultiOrgEntityProvider.ts | 550 +++++++- 7 files changed, 1928 insertions(+), 27 deletions(-) create mode 100644 .changeset/unlucky-deers-teach.md diff --git a/.changeset/unlucky-deers-teach.md b/.changeset/unlucky-deers-teach.md new file mode 100644 index 0000000000..4073c92ba4 --- /dev/null +++ b/.changeset/unlucky-deers-teach.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Implement events support for `GithubMultiOrgEntityProvider` diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index ccffe60bd1..39532e806f 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -65,6 +65,42 @@ export default async function createPlugin( } ``` +Alternatively, if you wish to ingest data from multiple GitHub organizations you can use +the `GithubMultiOrgEntityProvider` instead. Note that by default, this provider will namespace +groups according to the org they originate from to avoid potential name duplicates: + +```ts title="packages/backend/src/plugins/catalog.ts" +/* highlight-add-next-line */ +import { GithubMultiOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + + /* highlight-add-start */ + // The GitHub URL below needs to match a configured integrations.github entry + // specified in your app-config. + builder.addEntityProvider( + GithubMultiOrgEntityProvider.fromConfig(env.config, { + id: 'production', + githubUrl: 'https://github.com', + // Set the following to list the GitHub orgs you wish to ingest from. You can + // also omit this option to ingest all orgs accessible by your GitHub integration + orgs: ['org-a', 'org-b'], + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 60 }, + timeout: { minutes: 15 }, + }), + }), + ); + /* highlight-add-end */ + + // .. +} +``` + ## Installation with Events Support Please follow the installation instructions at @@ -110,6 +146,42 @@ export default async function createPlugin( } ``` +Or, alternatively, if using the `GithubMultiOrgEntityProvider`: + +```ts title="packages/backend/src/plugins/catalog.ts" +/* highlight-add-next-line */ +import { GithubMultiOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + + /* highlight-add-start */ + // The GitHub URL below needs to match a configured integrations.github entry + // specified in your app-config. + builder.addEntityProvider( + GithubMultiOrgEntityProvider.fromConfig(env.config, { + id: 'production', + githubUrl: 'https://github.com', + // Set the following to list the GitHub orgs you wish to ingest from. You can + // also omit this option to ingest all orgs accessible by your GitHub integration + orgs: ['org-a', 'org-b'], + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 60 }, + timeout: { minutes: 15 }, + }), + // Pass in the eventBroker to allow this provider to subscribe to GitHub events + eventBroker: env.eventBroker, + }), + ); + /* highlight-add-end */ + + // .. +} +``` + You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). The webhook will need to be configured to forward `organization`,`team` and `membership` events. diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 0698fae8a0..6366a2d571 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -10,6 +10,7 @@ import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { EventBroker } from '@backstage/plugin-events-node'; import { EventParams } from '@backstage/plugin-events-node'; import { EventSubscriber } from '@backstage/plugin-events-node'; import { GithubCredentialsProvider } from '@backstage/integration'; @@ -133,7 +134,9 @@ export type GithubMultiOrgConfig = Array<{ }>; // @public -export class GithubMultiOrgEntityProvider implements EntityProvider { +export class GithubMultiOrgEntityProvider + implements EntityProvider, EventSubscriber +{ constructor(options: { id: string; gitHubConfig: GithubIntegrationConfig; @@ -153,11 +156,16 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { ): GithubMultiOrgEntityProvider; // (undocumented) getProviderName(): string; + // (undocumented) + onEvent(params: EventParams): Promise; read(options?: { logger?: Logger }): Promise; + // (undocumented) + supportsEventTopics(): string[]; } // @public export interface GithubMultiOrgEntityProviderOptions { + eventBroker?: EventBroker; githubCredentialsProvider?: GithubCredentialsProvider; githubUrl: string; id: string; diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index df4565a174..7a0a29e098 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -21,6 +21,7 @@ import { graphql as graphqlMsw } from 'msw'; import { setupServer } from 'msw/node'; import { TeamTransformer, UserTransformer } from './defaultTransformers'; import { + getOrganizationsFromUser, getOrganizationTeams, getOrganizationUsers, getTeamMembers, @@ -448,6 +449,37 @@ describe('github', () => { }); }); + describe('getOrganizationsFromUser', () => { + it('reads orgs from user', async () => { + const input: QueryResponse = { + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + }, + { + login: 'b', + }, + { + login: 'c', + }, + ], + }, + }, + }; + + server.use( + graphqlMsw.query('orgs', (_req, res, ctx) => res(ctx.data(input))), + ); + + await expect(getOrganizationsFromUser(graphql, 'foo')).resolves.toEqual({ + orgs: ['a', 'b', 'c'], + }); + }); + }); + describe('getTeamMembers', () => { it('reads team members', async () => { const input: QueryResponse = { diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index d0b629092a..412b751c38 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -33,6 +33,7 @@ import { DeferredEntity } from '@backstage/plugin-catalog-node'; export type QueryResponse = { organization?: OrganizationResponse; repositoryOwner?: RepositoryOwnerResponse; + user?: UserResponse; }; type RepositoryOwnerResponse = { @@ -46,11 +47,19 @@ export type OrganizationResponse = { repositories?: Connection; }; +export type UserResponse = { + organizations?: Connection; +}; + export type PageInfo = { hasNextPage: boolean; endCursor?: string; }; +export type GithubOrg = { + login: string; +}; + /** * Github User * @@ -334,6 +343,34 @@ export async function getOrganizationTeamsFromUsers( return { groups }; } +export async function getOrganizationsFromUser( + client: typeof graphql, + user: string, +): Promise<{ + orgs: string[]; +}> { + const query = ` + query orgs($user: String!) { + user(login: $user) { + organizations(first: 100) { + nodes { login } + pageInfo { hasNextPage, endCursor } + } + } + }`; + + const orgs = await queryWithPaging( + client, + query, + '', + r => r.user?.organizations, + async o => o.login, + { user }, + ); + + return { orgs }; +} + export async function getOrganizationTeam( client: typeof graphql, org: string, diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts index 76ab55552d..69076bf737 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts @@ -160,7 +160,7 @@ describe('GithubMultiOrgEntityProvider', () => { const logger = getVoidLogger(); const gitHubConfig: GithubIntegrationConfig = { - host: 'https://github.com', + host: 'github.com', }; const mockGetCredentials = jest.fn().mockReturnValue({ @@ -201,9 +201,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/a', + 'url:https://github.com/a', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/a', + 'url:https://github.com/a', 'github.com/user-login': 'a', }, description: 'c', @@ -227,9 +227,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/x', + 'url:https://github.com/x', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/x', + 'url:https://github.com/x', 'github.com/user-login': 'x', }, description: 'z', @@ -253,9 +253,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/q', + 'url:https://github.com/q', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/q', + 'url:https://github.com/q', 'github.com/user-login': 'q', }, description: 's', @@ -279,9 +279,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/orgs/orgA/teams/team', + 'url:https://github.com/orgs/orgA/teams/team', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/orgs/orgA/teams/team', + 'url:https://github.com/orgs/orgA/teams/team', 'github.com/team-slug': 'orgA/team', }, namespace: 'orga', @@ -308,9 +308,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/orgs/orgB/teams/team', + 'url:https://github.com/orgs/orgB/teams/team', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/orgs/orgB/teams/team', + 'url:https://github.com/orgs/orgB/teams/team', 'github.com/team-slug': 'orgB/team', }, namespace: 'orgb', @@ -467,7 +467,7 @@ describe('GithubMultiOrgEntityProvider', () => { const logger = getVoidLogger(); const gitHubConfig: GithubIntegrationConfig = { - host: 'https://github.com', + host: 'github.com', }; const mockGetCredentials = jest.fn().mockReturnValue({ @@ -507,9 +507,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/a', + 'url:https://github.com/a', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/a', + 'url:https://github.com/a', 'github.com/user-login': 'a', }, description: 'c', @@ -533,9 +533,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/x', + 'url:https://github.com/x', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/x', + 'url:https://github.com/x', 'github.com/user-login': 'x', }, description: 'z', @@ -559,9 +559,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/q', + 'url:https://github.com/q', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/q', + 'url:https://github.com/q', 'github.com/user-login': 'q', }, description: 's', @@ -585,9 +585,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/orgs/orgC/teams/team', + 'url:https://github.com/orgs/orgC/teams/team', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/orgs/orgC/teams/team', + 'url:https://github.com/orgs/orgC/teams/team', 'github.com/team-slug': 'orgC/team', }, namespace: 'orgc', @@ -614,9 +614,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/orgs/orgD/teams/team', + 'url:https://github.com/orgs/orgD/teams/team', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/orgs/orgD/teams/team', + 'url:https://github.com/orgs/orgD/teams/team', 'github.com/team-slug': 'orgD/team', }, namespace: 'orgd', @@ -713,4 +713,1209 @@ describe('GithubMultiOrgEntityProvider', () => { }); }); }); + + describe('events', () => { + let entityProvider: GithubMultiOrgEntityProvider; + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + beforeEach(() => { + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + entityProvider = new GithubMultiOrgEntityProvider({ + id: 'my-id', + gitHubConfig, + githubCredentialsProvider, + githubUrl: 'https://github.com', + logger, + orgs: ['orgA', 'orgB'], + }); + + entityProvider.connect(entityProviderConnection); + }); + + afterEach(() => jest.resetAllMocks()); + + it('should ignore events from non-applicable orgs', async () => { + await entityProvider.onEvent({ + topic: 'github.organization', + eventPayload: { + action: 'member_added', + membership: { + user: { + name: 'githubuser', + login: 'githubuser', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + organization: { + login: 'test-org', + }, + }, + }); + + expect(entityProviderConnection.applyMutation).not.toHaveBeenCalled(); + + await entityProvider.onEvent({ + topic: 'github.installation', + eventPayload: { + action: 'created', + installation: { + account: { + login: 'test-org', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).not.toHaveBeenCalled(); + }); + + describe('installation', () => { + it('adds users and groups from new org', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgB/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgB/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.installation', + eventPayload: { + action: 'created', + installation: { + account: { + login: 'orgB', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['orga/team', 'orgb/team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgB/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgB/teams/team', + 'github.com/team-slug': 'orgB/team', + }, + namespace: 'orgb', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'TeamB', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + }); + + describe('organization', () => { + it('should add a new user', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'orgA', + }, + { + login: 'orgB', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.organization', + eventPayload: { + action: 'member_added', + organization: { + login: 'orgB', + }, + membership: { + user: { + name: 'a', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + login: 'a', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + }, + spec: { + memberOf: ['orga/team'], + profile: { + displayName: 'a', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + + it('should remove a user', async () => { + const mockClient = jest.fn(); + + mockClient.mockResolvedValueOnce({ + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'orgC', + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.organization', + eventPayload: { + action: 'member_removed', + organization: { + login: 'orgB', + }, + membership: { + user: { + name: 'a', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + login: 'a', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + }, + spec: { + memberOf: [], + profile: { + displayName: 'a', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + }); + }); + + it('should only update group memberships of a user that still belongs to an applicable org', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'orgA', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.organization', + eventPayload: { + action: 'member_removed', + organization: { + login: 'orgB', + }, + membership: { + user: { + name: 'a', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + login: 'a', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + }, + spec: { + memberOf: ['orga/team'], + profile: { + displayName: 'a', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + }); + + describe('team', () => { + it('should create a new group from a new team', async () => { + await entityProvider.onEvent({ + topic: 'github.team', + eventPayload: { + action: 'created', + organization: { + login: 'orgB', + }, + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/orgB/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + namespace: 'orgb', + description: 'description from the new team', + annotations: { + 'backstage.io/edit-url': + 'https://github.com/orgs/orgB/teams/new-team/edit', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgB/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgB/teams/new-team', + 'github.com/team-slug': 'orgB/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: [], + parent: 'father-team', + profile: { + displayName: 'New Team', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + + it('should remove a group from a deleted team', async () => { + await entityProvider.onEvent({ + topic: 'github.team', + eventPayload: { + action: 'deleted', + organization: { + login: 'orgB', + }, + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/orgB/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + namespace: 'orgb', + description: 'description from the new team', + annotations: { + 'backstage.io/edit-url': + 'https://github.com/orgs/orgB/teams/new-team/edit', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgB/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgB/teams/new-team', + 'github.com/team-slug': 'orgB/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: [], + parent: 'father-team', + profile: { + displayName: 'New Team', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + }); + }); + + it('should update group and user entities when a team is edited', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'w', + name: 'x', + bio: 'y', + email: 'z', + avatarUrl: 'q', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgB/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.team', + eventPayload: { + action: 'edited', + changes: { + name: { + from: 'oldName', + }, + description: { + from: 'oldDescription', + }, + }, + team: { + slug: 'team', + parent: { + slug: 'parent', + }, + }, + organization: { + login: 'orgA', + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + description: 'c', + }, + spec: { + memberOf: ['orga/team', 'orgb/team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/edit-url': 'https://example.com', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'github.com/team-slug': 'orgA/team', + }, + namespace: 'orga', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'TeamA', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgA/teams/oldname', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgA/teams/oldname', + 'github.com/team-slug': 'orgA/oldname', + }, + namespace: 'orga', + name: 'oldname', + description: 'oldDescription', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'oldName', + }, + type: 'team', + members: [], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + }); + }); + }); + + describe('membership', () => { + it('should update group and user entities when a member is added', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + }, + }) + .mockResolvedValueOnce({ + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'orgA', + }, + { + login: 'orgB', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgB/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.membership', + eventPayload: { + action: 'added', + organization: { + login: 'orgA', + }, + team: { + slug: 'teama', + }, + member: { + name: 'a', + login: 'a', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + }); + await new Promise(process.nextTick); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + }, + spec: { + memberOf: ['orga/team', 'orgb/team'], + profile: { + displayName: 'a', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/edit-url': 'https://example.com', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'github.com/team-slug': 'orgA/team', + }, + namespace: 'orga', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'TeamA', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + + it('should update group and user entities when a member is removed', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [], + }, + }, + }, + }) + .mockResolvedValueOnce({ + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'orgA', + }, + { + login: 'orgB', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgB/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.membership', + eventPayload: { + action: 'removed', + organization: { + login: 'orgA', + }, + team: { + slug: 'teama', + }, + member: { + name: 'a', + login: 'a', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + }); + await new Promise(process.nextTick); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + }, + spec: { + memberOf: ['orgb/team'], + profile: { + displayName: 'a', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/edit-url': 'https://example.com', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'github.com/team-slug': 'orgA/team', + }, + namespace: 'orga', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'TeamA', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: [], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + }); + }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts index ce45f30d96..d3943bb66a 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts @@ -21,7 +21,9 @@ import { DEFAULT_NAMESPACE, Entity, GroupEntity, + parseEntityRef, stringifyEntityRef, + UserEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { @@ -32,10 +34,28 @@ import { ScmIntegrations, } from '@backstage/integration'; import { + DeferredEntity, EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; +import { + EventBroker, + EventParams, + EventSubscriber, +} from '@backstage/plugin-events-node'; import { graphql } from '@octokit/graphql'; +import { + InstallationCreatedEvent, + InstallationEvent, + OrganizationEvent, + OrganizationMemberAddedEvent, + OrganizationMemberRemovedEvent, + MembershipEvent, + TeamCreatedEvent, + TeamDeletedEvent, + TeamEditedEvent, + TeamEvent, +} from '@octokit/webhooks-types'; import { merge } from 'lodash'; import * as uuid from 'uuid'; import { Logger } from 'winston'; @@ -44,6 +64,7 @@ import { assignGroupsToUsers, buildOrgHierarchy, defaultOrganizationTeamTransformer, + defaultUserTransformer, getOrganizationTeams, getOrganizationUsers, GithubTeam, @@ -55,6 +76,11 @@ import { ANNOTATION_GITHUB_TEAM_SLUG, ANNOTATION_GITHUB_USER_LOGIN, } from '../lib/annotation'; +import { + getOrganizationsFromUser, + getOrganizationTeam, + getOrganizationTeamsFromUsers, +} from '../lib/github'; import { splitTeamSlug } from '../lib/util'; /** @@ -118,14 +144,26 @@ export interface GithubMultiOrgEntityProviderOptions { * By default groups will be namespaced according to their GitHub org. */ teamTransformer?: TeamTransformer; + + /** + * An EventBroker to subscribe this provider to GitHub events to trigger delta mutations + */ + eventBroker?: EventBroker; } +type CreateDeltaOperation = (entities: Entity[]) => { + added: DeferredEntity[]; + removed: DeferredEntity[]; +}; + /** * Ingests org data (users and groups) from GitHub. * * @public */ -export class GithubMultiOrgEntityProvider implements EntityProvider { +export class GithubMultiOrgEntityProvider + implements EntityProvider, EventSubscriber +{ private connection?: EntityProviderConnection; private scheduleFn?: () => Promise; @@ -161,6 +199,10 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { provider.schedule(options.schedule); + if (options.eventBroker) { + options.eventBroker.subscribe(provider); + } + return provider; } @@ -271,6 +313,478 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { markCommitComplete(); } + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ + supportsEventTopics(): string[] { + return [ + 'github.installation', + 'github.organization', + 'github.team', + 'github.membership', + ]; + } + + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */ + async onEvent(params: EventParams): Promise { + const { logger } = this.options; + logger.debug(`Received event from ${params.topic}`); + + const orgs = this.options.orgs?.length + ? this.options.orgs + : await this.getAllOrgs(this.options.gitHubConfig); + + const eventPayload = params.eventPayload as + | InstallationEvent + | OrganizationEvent + | MembershipEvent + | TeamEvent; + + if ( + !orgs.includes( + (eventPayload as InstallationEvent).installation?.account?.login, + ) && + !orgs.includes( + (eventPayload as OrganizationEvent | MembershipEvent | TeamEvent) + .organization?.login, + ) + ) { + return; + } + + // https://docs.github.com/webhooks-and-events/webhooks/webhook-events-and-payloads#installation + if ( + params.topic.includes('installation') && + eventPayload.action === 'created' + ) { + // We can only respond to installation.created events to add new users/groups since a + // installation.deleted event won't provide us info on what user/groups we should remove and + // we can't query the uninstalled org since we will no longer have access. This will need to be + // eventually resolved via occasional full mutation runs by calling read() + await this.onInstallationChange( + eventPayload as InstallationCreatedEvent, + orgs, + ); + } + + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization + if ( + params.topic.includes('organization') && + (eventPayload.action === 'member_added' || + eventPayload.action === 'member_removed') + ) { + await this.onMemberChangeInOrganization(eventPayload, orgs); + } + + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team + if (params.topic.includes('team')) { + if ( + eventPayload.action === 'created' || + eventPayload.action === 'deleted' + ) { + await this.onTeamChangeInOrganization( + eventPayload as TeamCreatedEvent | TeamDeletedEvent, + ); + } else if (eventPayload.action === 'edited') { + await this.onTeamEditedInOrganization( + eventPayload as TeamEditedEvent, + orgs, + ); + } + } + + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership + if (params.topic.includes('membership')) { + this.onMembershipChangedInTeam(eventPayload as MembershipEvent, orgs); + } + + return; + } + + private async onInstallationChange( + event: InstallationCreatedEvent, + applicableOrgs: string[], + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const org = event.installation.account.login; + const { headers, type: tokenType } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${org}`, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { users } = await getOrganizationUsers( + client, + org, + tokenType, + this.options.userTransformer, + ); + + const { groups } = await getOrganizationTeams( + client, + org, + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + // Fetch group memberships of users in case they already exist and + // have memberships in groups from other applicable orgs + for (const userOrg of applicableOrgs) { + const { headers: orgHeaders } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${userOrg}`, + }); + const orgClient = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers: orgHeaders, + }); + + const { groups: userGroups } = await getOrganizationTeamsFromUsers( + orgClient, + userOrg, + users.map( + u => + u.metadata.annotations?.[ANNOTATION_GITHUB_USER_LOGIN] || + u.metadata.name, + ), + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + assignGroupsToUsers(users, userGroups); + } + + const { added, removed } = this.createAddEntitiesOperation([ + ...users, + ...groups, + ]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onMemberChangeInOrganization( + event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent, + applicableOrgs: string[], + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const userTransformer = + this.options.userTransformer || defaultUserTransformer; + const { name, avatar_url: avatarUrl, email, login } = event.membership.user; + const org = event.organization.login; + const { headers } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${org}`, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { orgs } = await getOrganizationsFromUser(client, login); + const userApplicableOrgs = orgs.filter(o => applicableOrgs.includes(o)); + + let updateMemberships: boolean; + let createDeltaOperation: CreateDeltaOperation; + if (event.action === 'member_removed') { + if (userApplicableOrgs.length) { + // If the user is still associated with another applicable org then we don't want to remove + // them, just update the entity to remove any potential group memberships from the old org + createDeltaOperation = this.createAddEntitiesOperation.bind(this); + updateMemberships = true; + } else { + // User is no longer part of any applicable orgs so we can remove it, + // no need to take memberships into account + createDeltaOperation = this.createRemoveEntitiesOperation.bind(this); + updateMemberships = false; + } + } else { + // We're not sure if this user was already added as part of another applicable org + // so grab the latest memberships (potentially from teams of other orgs) to ensure + // we're not accidentally omitting them + createDeltaOperation = this.createAddEntitiesOperation.bind(this); + updateMemberships = true; + } + + const user = (await userTransformer( + { + name, + avatarUrl, + login, + email: email ?? undefined, + }, + { + org, + client, + query: '', + }, + )) as UserEntity; + + if (updateMemberships) { + for (const userOrg of userApplicableOrgs) { + const { headers: orgHeaders } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${userOrg}`, + }); + const orgClient = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers: orgHeaders, + }); + + const { groups } = await getOrganizationTeamsFromUsers( + orgClient, + userOrg, + [login], + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + assignGroupsToUsers([user], groups); + } + } + + const { added, removed } = createDeltaOperation([user]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onTeamChangeInOrganization( + event: TeamCreatedEvent | TeamDeletedEvent, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const org = event.organization.login; + const { headers } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${org}`, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { name, html_url: url, description, slug } = event.team; + const group = (await this.defaultMultiOrgTeamTransformer( + { + name, + slug, + editTeamUrl: `${url}/edit`, + combinedSlug: `${org}/${slug}`, + description: description ?? undefined, + parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam, + // entity will be removed or is new + members: [], + }, + { + org, + client, + query: '', + }, + )) as Entity; + + const createDeltaOperation = + event.action === 'created' + ? this.createAddEntitiesOperation.bind(this) + : this.createRemoveEntitiesOperation.bind(this); + const { added, removed } = createDeltaOperation([group]); + + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onTeamEditedInOrganization( + event: TeamEditedEvent, + applicableOrgs: string[], + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const org = event.organization.login; + const { headers, type: tokenType } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${org}`, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const teamSlug = event.team.slug; + const { group } = await getOrganizationTeam( + client, + org, + teamSlug, + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + const { users } = await getOrganizationUsers( + client, + org, + tokenType, + this.options.userTransformer, + ); + + const usersFromChangedGroup = + group.spec.members?.map(m => + stringifyEntityRef(parseEntityRef(m, { defaultKind: 'user' })), + ) || []; + const usersToRebuild = users.filter(u => + usersFromChangedGroup.includes(stringifyEntityRef(u)), + ); + + // Update memberships of associated members of this group in case the group entity ref changed + for (const userOrg of applicableOrgs) { + const { headers: orgHeaders } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${userOrg}`, + }); + const orgClient = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers: orgHeaders, + }); + + const { groups } = await getOrganizationTeamsFromUsers( + orgClient, + userOrg, + usersToRebuild.map( + u => + u.metadata.annotations?.[ANNOTATION_GITHUB_USER_LOGIN] || + u.metadata.name, + ), + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + assignGroupsToUsers(usersToRebuild, groups); + } + + const oldName = event.changes.name?.from || ''; + const oldSlug = oldName.toLowerCase().replaceAll(/\s/gi, '-'); + const oldGroup = (await this.defaultMultiOrgTeamTransformer( + { + name: event.changes.name?.from, + slug: oldSlug, + combinedSlug: `${org}/${oldSlug}`, + description: event.changes.description?.from, + parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam, + // entity will be removed + members: [], + }, + { + org, + client, + query: '', + }, + )) as Entity; + + // Remove the old group entity in case the entity ref is now different + const { removed } = this.createRemoveEntitiesOperation([oldGroup]); + const { added } = this.createAddEntitiesOperation([ + ...usersToRebuild, + group, + ]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onMembershipChangedInTeam( + event: MembershipEvent, + applicableOrgs: string[], + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + // The docs are saying I will receive the slug for the removed event, + // but the types don't reflect that, + // so I will just check to be sure the slug is there + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership + if (!('slug' in event.team)) { + return; + } + + const org = event.organization.login; + const { headers } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${org}`, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const teamSlug = event.team.slug; + const { group } = await getOrganizationTeam( + client, + org, + teamSlug, + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + const userTransformer = + this.options.userTransformer || defaultUserTransformer; + const { name, avatar_url: avatarUrl, email, login } = event.member; + const user = (await userTransformer( + { + name, + avatarUrl, + login, + email: email ?? undefined, + }, + { + org, + client, + query: '', + }, + )) as UserEntity; + + const { orgs } = await getOrganizationsFromUser(client, login); + const userApplicableOrgs = orgs.filter(o => applicableOrgs.includes(o)); + for (const userOrg of userApplicableOrgs) { + const { headers: orgHeaders } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${userOrg}`, + }); + const orgClient = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers: orgHeaders, + }); + + const { groups } = await getOrganizationTeamsFromUsers( + orgClient, + userOrg, + [login], + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + assignGroupsToUsers([user], groups); + } + + const { added, removed } = this.createAddEntitiesOperation([user, group]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + private schedule(schedule: GithubMultiOrgEntityProviderOptions['schedule']) { if (!schedule || schedule === 'manual') { return; @@ -301,9 +815,11 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { team: GithubTeam, ctx: TransformerContext, ): Promise { - const result = this.options.teamTransformer - ? await this.options.teamTransformer(team, ctx) - : await defaultOrganizationTeamTransformer(team, ctx); + if (this.options.teamTransformer) { + return await this.options.teamTransformer(team, ctx); + } + + const result = await defaultOrganizationTeamTransformer(team, ctx); if (result) { result.metadata.namespace = ctx.org.toLocaleLowerCase('en-US'); @@ -333,6 +849,32 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { ) .filter(Boolean) as string[]; } + + private createAddEntitiesOperation(entities: Entity[]) { + return { + removed: [], + added: entities.map(entity => ({ + locationKey: `github-multi-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + entity, + ), + })), + }; + } + + private createRemoveEntitiesOperation(entities: Entity[]) { + return { + added: [], + removed: entities.map(entity => ({ + locationKey: `github-multi-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + entity, + ), + })), + }; + } } // Helps wrap the timing and logging behaviors From 7c89eb4b36a04cb434a0d130271a05954980a34f Mon Sep 17 00:00:00 2001 From: Phil Kuang Date: Thu, 4 May 2023 11:35:03 -0400 Subject: [PATCH 13/32] refactor(GithubMultiOrgEntityProvider): remove EventSubscriber interface declaration Signed-off-by: Phil Kuang --- .changeset/unlucky-deers-teach.md | 4 +- .../api-report.md | 8 +-- .../GithubMultiOrgEntityProvider.test.ts | 50 ++++++++++++------- .../providers/GithubMultiOrgEntityProvider.ts | 21 +++----- 4 files changed, 45 insertions(+), 38 deletions(-) diff --git a/.changeset/unlucky-deers-teach.md b/.changeset/unlucky-deers-teach.md index 4073c92ba4..b773e9090f 100644 --- a/.changeset/unlucky-deers-teach.md +++ b/.changeset/unlucky-deers-teach.md @@ -1,5 +1,7 @@ --- -'@backstage/plugin-catalog-backend-module-github': patch +'@backstage/plugin-catalog-backend-module-github': minor --- Implement events support for `GithubMultiOrgEntityProvider` + +**BREAKING:** Passing in a custom `teamTransformer` will now correctly completely override the default transformer behavior diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 6366a2d571..7b944b33a9 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -134,9 +134,7 @@ export type GithubMultiOrgConfig = Array<{ }>; // @public -export class GithubMultiOrgEntityProvider - implements EntityProvider, EventSubscriber -{ +export class GithubMultiOrgEntityProvider implements EntityProvider { constructor(options: { id: string; gitHubConfig: GithubIntegrationConfig; @@ -156,11 +154,7 @@ export class GithubMultiOrgEntityProvider ): GithubMultiOrgEntityProvider; // (undocumented) getProviderName(): string; - // (undocumented) - onEvent(params: EventParams): Promise; read(options?: { logger?: Logger }): Promise; - // (undocumented) - supportsEventTopics(): string[]; } // @public diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts index 69076bf737..99bef6e074 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts @@ -16,11 +16,13 @@ import { getVoidLogger } from '@backstage/backend-common'; import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; import { GithubCredentialsProvider, GithubIntegrationConfig, } from '@backstage/integration'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { EventSubscriber } from '@backstage/plugin-events-node'; import { graphql } from '@octokit/graphql'; import { GithubMultiOrgEntityProvider, @@ -715,7 +717,8 @@ describe('GithubMultiOrgEntityProvider', () => { }); describe('events', () => { - let entityProvider: GithubMultiOrgEntityProvider; + let onEvent: Function; + const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), refresh: jest.fn(), @@ -723,9 +726,15 @@ describe('GithubMultiOrgEntityProvider', () => { beforeEach(() => { const logger = getVoidLogger(); - const gitHubConfig: GithubIntegrationConfig = { - host: 'github.com', - }; + const config = new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + }, + ], + }, + }); const mockGetCredentials = jest.fn().mockReturnValue({ headers: { token: 'blah' }, @@ -736,13 +745,20 @@ describe('GithubMultiOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - entityProvider = new GithubMultiOrgEntityProvider({ + const mockEventBroker = { + publish: async () => {}, + subscribe: (subscriber: EventSubscriber) => { + onEvent = subscriber.onEvent; + }, + }; + + const entityProvider = GithubMultiOrgEntityProvider.fromConfig(config, { id: 'my-id', - gitHubConfig, githubCredentialsProvider, githubUrl: 'https://github.com', logger, orgs: ['orgA', 'orgB'], + eventBroker: mockEventBroker, }); entityProvider.connect(entityProviderConnection); @@ -751,7 +767,7 @@ describe('GithubMultiOrgEntityProvider', () => { afterEach(() => jest.resetAllMocks()); it('should ignore events from non-applicable orgs', async () => { - await entityProvider.onEvent({ + await onEvent({ topic: 'github.organization', eventPayload: { action: 'member_added', @@ -771,7 +787,7 @@ describe('GithubMultiOrgEntityProvider', () => { expect(entityProviderConnection.applyMutation).not.toHaveBeenCalled(); - await entityProvider.onEvent({ + await onEvent({ topic: 'github.installation', eventPayload: { action: 'created', @@ -885,7 +901,7 @@ describe('GithubMultiOrgEntityProvider', () => { (graphql.defaults as jest.Mock).mockReturnValue(mockClient); - await entityProvider.onEvent({ + await onEvent({ topic: 'github.installation', eventPayload: { action: 'created', @@ -1017,7 +1033,7 @@ describe('GithubMultiOrgEntityProvider', () => { (graphql.defaults as jest.Mock).mockReturnValue(mockClient); - await entityProvider.onEvent({ + await onEvent({ topic: 'github.organization', eventPayload: { action: 'member_added', @@ -1087,7 +1103,7 @@ describe('GithubMultiOrgEntityProvider', () => { (graphql.defaults as jest.Mock).mockReturnValue(mockClient); - await entityProvider.onEvent({ + await onEvent({ topic: 'github.organization', eventPayload: { action: 'member_removed', @@ -1182,7 +1198,7 @@ describe('GithubMultiOrgEntityProvider', () => { (graphql.defaults as jest.Mock).mockReturnValue(mockClient); - await entityProvider.onEvent({ + await onEvent({ topic: 'github.organization', eventPayload: { action: 'member_removed', @@ -1237,7 +1253,7 @@ describe('GithubMultiOrgEntityProvider', () => { describe('team', () => { it('should create a new group from a new team', async () => { - await entityProvider.onEvent({ + await onEvent({ topic: 'github.team', eventPayload: { action: 'created', @@ -1296,7 +1312,7 @@ describe('GithubMultiOrgEntityProvider', () => { }); it('should remove a group from a deleted team', async () => { - await entityProvider.onEvent({ + await onEvent({ topic: 'github.team', eventPayload: { action: 'deleted', @@ -1451,7 +1467,7 @@ describe('GithubMultiOrgEntityProvider', () => { (graphql.defaults as jest.Mock).mockReturnValue(mockClient); - await entityProvider.onEvent({ + await onEvent({ topic: 'github.team', eventPayload: { action: 'edited', @@ -1660,7 +1676,7 @@ describe('GithubMultiOrgEntityProvider', () => { (graphql.defaults as jest.Mock).mockReturnValue(mockClient); - await entityProvider.onEvent({ + await onEvent({ topic: 'github.membership', eventPayload: { action: 'added', @@ -1833,7 +1849,7 @@ describe('GithubMultiOrgEntityProvider', () => { (graphql.defaults as jest.Mock).mockReturnValue(mockClient); - await entityProvider.onEvent({ + await onEvent({ topic: 'github.membership', eventPayload: { action: 'removed', diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts index d3943bb66a..51796c2116 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts @@ -38,11 +38,7 @@ import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; -import { - EventBroker, - EventParams, - EventSubscriber, -} from '@backstage/plugin-events-node'; +import { EventBroker, EventParams } from '@backstage/plugin-events-node'; import { graphql } from '@octokit/graphql'; import { InstallationCreatedEvent, @@ -161,9 +157,7 @@ type CreateDeltaOperation = (entities: Entity[]) => { * * @public */ -export class GithubMultiOrgEntityProvider - implements EntityProvider, EventSubscriber -{ +export class GithubMultiOrgEntityProvider implements EntityProvider { private connection?: EntityProviderConnection; private scheduleFn?: () => Promise; @@ -200,7 +194,10 @@ export class GithubMultiOrgEntityProvider provider.schedule(options.schedule); if (options.eventBroker) { - options.eventBroker.subscribe(provider); + options.eventBroker.subscribe({ + supportsEventTopics: provider.supportsEventTopics.bind(provider), + onEvent: provider.onEvent.bind(provider), + }); } return provider; @@ -313,8 +310,7 @@ export class GithubMultiOrgEntityProvider markCommitComplete(); } - /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ - supportsEventTopics(): string[] { + private supportsEventTopics(): string[] { return [ 'github.installation', 'github.organization', @@ -323,8 +319,7 @@ export class GithubMultiOrgEntityProvider ]; } - /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */ - async onEvent(params: EventParams): Promise { + private async onEvent(params: EventParams): Promise { const { logger } = this.options; logger.debug(`Received event from ${params.topic}`); From b779e9875664ee2e966b13d51d91cb615564a8c4 Mon Sep 17 00:00:00 2001 From: Zander Franks Date: Thu, 4 May 2023 12:37:35 -0500 Subject: [PATCH 14/32] fix(scaffolder): requested changes Signed-off-by: Zander Franks --- .changeset/heavy-penguins-burn.md | 4 ++-- .../features/software-templates/writing-templates.md | 4 ++-- .../src/Template.v1beta3.schema.json | 12 ++++++------ plugins/scaffolder-react/api-report.md | 2 +- plugins/scaffolder-react/src/api/types.ts | 2 +- .../TemplateOutputs/DefaultTemplateOutputs.tsx | 2 +- .../next/components/TemplateOutputs/TextOutputs.tsx | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.changeset/heavy-penguins-burn.md b/.changeset/heavy-penguins-burn.md index f9fba9bfb8..b8c243a99b 100644 --- a/.changeset/heavy-penguins-burn.md +++ b/.changeset/heavy-penguins-burn.md @@ -1,6 +1,6 @@ --- -'@backstage/plugin-scaffolder-common': patch -'@backstage/plugin-scaffolder-react': patch +'@backstage/plugin-scaffolder-common': minor +'@backstage/plugin-scaffolder-react': minor --- Add support for Markdown text blob outputs from templates diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md index 2508f4a592..44bd5e5928 100644 --- a/docs/features/software-templates/writing-templates.md +++ b/docs/features/software-templates/writing-templates.md @@ -522,8 +522,8 @@ output: icon: catalog entityRef: ${{ steps['register'].output.entityRef }} # link to the entity that has been ingested to the catalog text: - - title: More data - data: | + - title: More information + content: | Access the \[remote repository\](${{ steps['publish'].output.remoteUrl }}). ``` diff --git a/plugins/scaffolder-common/src/Template.v1beta3.schema.json b/plugins/scaffolder-common/src/Template.v1beta3.schema.json index beb86f43cb..1de68205b2 100644 --- a/plugins/scaffolder-common/src/Template.v1beta3.schema.json +++ b/plugins/scaffolder-common/src/Template.v1beta3.schema.json @@ -221,15 +221,15 @@ }, "text": { "type": "array", - "description": "A list of text blobs, like output data from the template.", + "description": "A list of Markdown text blobs, like output data from the template.", "items": { "type": "object", "required": [], "properties": { "title": { "type": "string", - "description": "A user friendly display name for the text blob.", - "examples": ["Output Data"], + "description": "A user friendly display name for the text.", + "examples": ["Output Content"], "minLength": 1 }, "icon": { @@ -238,10 +238,10 @@ "examples": ["dashboard"], "minLength": 1 }, - "data": { + "content": { "type": "string", - "description": "The text blob to display in the UI.", - "examples": ["{}"] + "description": "The text blob to display in the UI, rendered as Markdown.", + "examples": ["**hey** _I'm_ Markdown"] } } } diff --git a/plugins/scaffolder-react/api-report.md b/plugins/scaffolder-react/api-report.md index a2a7dca3b2..1761c88edf 100644 --- a/plugins/scaffolder-react/api-report.md +++ b/plugins/scaffolder-react/api-report.md @@ -221,7 +221,7 @@ export type ScaffolderOutputLink = { export type ScaffolderOutputText = { title?: string; icon?: string; - data?: string; + content?: string; }; // @public diff --git a/plugins/scaffolder-react/src/api/types.ts b/plugins/scaffolder-react/src/api/types.ts index 64bb0d05ac..2adecdd0d2 100644 --- a/plugins/scaffolder-react/src/api/types.ts +++ b/plugins/scaffolder-react/src/api/types.ts @@ -88,7 +88,7 @@ export type ScaffolderOutputLink = { export type ScaffolderOutputText = { title?: string; icon?: string; - data?: string; + content?: string; }; /** @public */ diff --git a/plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.tsx b/plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.tsx index de52b62700..a495e4ca9e 100644 --- a/plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.tsx +++ b/plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.tsx @@ -64,7 +64,7 @@ export const DefaultTemplateOutputs = (props: { titleTypographyProps={{ component: 'h2' }} > - + diff --git a/plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx b/plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx index 8600a281a0..4a9110d79c 100644 --- a/plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx +++ b/plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx @@ -38,7 +38,7 @@ export const TextOutputs = (props: { return ( <> {text - .filter(({ data }) => data) + .filter(({ content }) => content !== undefined) .map(({ title, icon }, i) => { const Icon = iconResolver(icon); return ( From c540b48fc4a569cd71048368714f02c597303d13 Mon Sep 17 00:00:00 2001 From: Alex Crome Date: Thu, 4 May 2023 20:17:34 +0100 Subject: [PATCH 15/32] Update CONTRIBUTING.md Correct code of conduct reference in CONTRIBUTING.md to refer to CNCF's code of conduct, not Spotify's FOSS code of conduct. Signed-off-by: Alex Crome --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7828791080..e324e8a7a9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,7 +92,7 @@ For those contributors who have earned write access to the repository, when a pu ## Code of Conduct -This project adheres to the [Spotify FOSS Code of Conduct][code-of-conduct]. By participating, you are expected to honor this code. +This project adheres to the [CNCF Code of Conduct][code-of-conduct]. By participating, you are expected to honor this code. [code-of-conduct]: https://github.com/backstage/backstage/blob/master/CODE_OF_CONDUCT.md From 3ffcdac7d075116468abcc09ff73960d9a9809ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 28 Apr 2023 13:55:17 +0200 Subject: [PATCH 16/32] Added a persistent session store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/thick-pugs-rush.md | 5 + .../migrations/20230428155633_sessions.js | 52 +++++ plugins/auth-backend/package.json | 1 + .../auth-backend/src/database/AuthDatabase.ts | 76 +++++++ .../src/identity/DatabaseKeyStore.test.ts | 187 +++++++++--------- .../src/identity/DatabaseKeyStore.ts | 32 +-- .../src/identity/KeyStores.test.ts | 18 +- .../auth-backend/src/identity/KeyStores.ts | 11 +- plugins/auth-backend/src/migrations.test.ts | 78 ++++++++ plugins/auth-backend/src/service/router.ts | 14 +- yarn.lock | 13 +- 11 files changed, 334 insertions(+), 153 deletions(-) create mode 100644 .changeset/thick-pugs-rush.md create mode 100644 plugins/auth-backend/migrations/20230428155633_sessions.js create mode 100644 plugins/auth-backend/src/database/AuthDatabase.ts create mode 100644 plugins/auth-backend/src/migrations.test.ts diff --git a/.changeset/thick-pugs-rush.md b/.changeset/thick-pugs-rush.md new file mode 100644 index 0000000000..7f0171eb83 --- /dev/null +++ b/.changeset/thick-pugs-rush.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Added a persistent session store through the database diff --git a/plugins/auth-backend/migrations/20230428155633_sessions.js b/plugins/auth-backend/migrations/20230428155633_sessions.js new file mode 100644 index 0000000000..49462ac503 --- /dev/null +++ b/plugins/auth-backend/migrations/20230428155633_sessions.js @@ -0,0 +1,52 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // See https://github.com/gx0r/connect-session-knex + // Modeled loosely after https://github.com/gx0r/connect-session-knex/blob/4e0e36a9afbb13c3000a89f5e341f2d2d4339a02/lib/index.js#L114 + // For simplicity we always make the session a string + // Do NOT change around this table or column names; the connect-session-knex library makes assumptions about them + await knex.schema.createTable('sessions', table => { + table.comment('Session data'); + table.string('sid').primary().notNullable().comment('ID of the session'); + table + .text('sess', 'longtext') + .notNullable() + .comment('Session data, JSON serialized'); + table + .timestamp('expired') + .notNullable() + .comment('The point in time when the session expires'); + table.index('sid', 'sessions_sid_idx'); + table.index('expired', 'sessions_expired_idx'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('sessions', table => { + table.dropIndex([], 'sessions_sid_idx'); + table.dropIndex([], 'sessions_expired_idx'); + }); + await knex.schema.dropTable('sessions'); +}; diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 61c3fe9a8d..aa57e06708 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -44,6 +44,7 @@ "@types/express": "^4.17.6", "@types/passport": "^1.0.3", "compression": "^1.7.4", + "connect-session-knex": "^3.0.1", "cookie-parser": "^1.4.5", "cors": "^2.8.5", "express": "^4.17.1", diff --git a/plugins/auth-backend/src/database/AuthDatabase.ts b/plugins/auth-backend/src/database/AuthDatabase.ts new file mode 100644 index 0000000000..9c112e0aac --- /dev/null +++ b/plugins/auth-backend/src/database/AuthDatabase.ts @@ -0,0 +1,76 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + DatabaseManager, + PluginDatabaseManager, + resolvePackagePath, +} from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { Knex } from 'knex'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-auth-backend', + 'migrations', +); + +/** + * Ensures that a database connection is established exactly once and only when + * asked for, and runs migrations. + */ +export class AuthDatabase { + readonly #database: PluginDatabaseManager; + #promise: Promise | undefined; + + static create(database: PluginDatabaseManager): AuthDatabase { + return new AuthDatabase(database); + } + + static forTesting(): AuthDatabase { + const config = new ConfigReader({ + backend: { + database: { + client: 'better-sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }, + }, + }); + const database = DatabaseManager.fromConfig(config).forPlugin('auth'); + return new AuthDatabase(database); + } + + static async runMigrations(knex: Knex): Promise { + await knex.migrate.latest({ + directory: migrationsDir, + }); + } + + private constructor(database: PluginDatabaseManager) { + this.#database = database; + } + + get(): Promise { + this.#promise ??= this.#database.getClient().then(async client => { + if (!this.#database.migrations?.skip) { + await AuthDatabase.runMigrations(client); + } + return client; + }); + + return this.#promise; + } +} diff --git a/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts b/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts index da67a2f88f..913dfcef22 100644 --- a/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts +++ b/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts @@ -14,33 +14,10 @@ * limitations under the License. */ -import Knex, { Knex as KnexType } from 'knex'; -import { DatabaseKeyStore } from './DatabaseKeyStore'; import { DateTime } from 'luxon'; - -function createDatabaseManager( - client: KnexType, - skipMigrations: boolean = false, -) { - return { - getClient: async () => client, - migrations: { - skip: skipMigrations, - }, - }; -} - -function createDB() { - const knex = Knex({ - client: 'better-sqlite3', - connection: ':memory:', - useNullAsDefault: true, - }); - knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { - resource.run('PRAGMA foreign_keys = ON', () => {}); - }); - return knex; -} +import { AuthDatabase } from '../database/AuthDatabase'; +import { DatabaseKeyStore } from './DatabaseKeyStore'; +import { TestDatabases } from '@backstage/backend-test-utils'; const keyBase = { use: 'sig', @@ -48,95 +25,107 @@ const keyBase = { alg: 'Base64', } as const; +jest.setTimeout(60_000); + describe('DatabaseKeyStore', () => { - it('should store a key', async () => { - const client = createDB(); - const store = await DatabaseKeyStore.create({ - database: createDatabaseManager(client), - }); - - const key = { - kid: '123', - ...keyBase, - }; - - await expect(store.listKeys()).resolves.toEqual({ items: [] }); - await store.addKey(key); - - const { items } = await store.listKeys(); - expect(items).toEqual([{ createdAt: expect.anything(), key }]); - expect( - Math.abs( - DateTime.fromJSDate(items[0].createdAt).diffNow('seconds').seconds, - ), - ).toBeLessThan(10); + const databases = TestDatabases.create({ + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); - it('should remove stored keys', async () => { - const client = createDB(); - const store = await DatabaseKeyStore.create({ - database: createDatabaseManager(client), - }); + it.each(databases.eachSupportedId())( + 'should store a key, %p', + async databaseId => { + const knex = await databases.init(databaseId); + await AuthDatabase.runMigrations(knex); - const key1 = { kid: '1', ...keyBase }; - const key2 = { kid: '2', ...keyBase }; - const key3 = { kid: '3', ...keyBase }; + const store = new DatabaseKeyStore(knex); - await store.addKey(key1); - await store.addKey(key2); - await store.addKey(key3); + const key = { + kid: '123', + ...keyBase, + }; - await expect(store.listKeys()).resolves.toEqual({ - items: [ - { key: key1, createdAt: expect.anything() }, - { key: key2, createdAt: expect.anything() }, - { key: key3, createdAt: expect.anything() }, - ], - }); + await expect(store.listKeys()).resolves.toEqual({ items: [] }); + await store.addKey(key); - store.removeKeys(['1']); + const { items } = await store.listKeys(); + expect(items).toEqual([{ createdAt: expect.anything(), key }]); + expect( + Math.abs( + DateTime.fromJSDate(items[0].createdAt).diffNow('seconds').seconds, + ), + ).toBeLessThan(10); + }, + ); - await expect(store.listKeys()).resolves.toEqual({ - items: [ - { key: key2, createdAt: expect.anything() }, - { key: key3, createdAt: expect.anything() }, - ], - }); + it.each(databases.eachSupportedId())( + 'should remove stored keys, %p', + async databaseId => { + const knex = await databases.init(databaseId); + await AuthDatabase.runMigrations(knex); - store.removeKeys(['1', '2']); + const store = new DatabaseKeyStore(knex); - await expect(store.listKeys()).resolves.toEqual({ - items: [{ key: key3, createdAt: expect.anything() }], - }); + const key1 = { kid: '1', ...keyBase }; + const key2 = { kid: '2', ...keyBase }; + const key3 = { kid: '3', ...keyBase }; - store.removeKeys([]); + await store.addKey(key1); + await store.addKey(key2); + await store.addKey(key3); - await expect(store.listKeys()).resolves.toEqual({ - items: [{ key: key3, createdAt: expect.anything() }], - }); + await expect(store.listKeys()).resolves.toEqual({ + items: [ + { key: key1, createdAt: expect.anything() }, + { key: key2, createdAt: expect.anything() }, + { key: key3, createdAt: expect.anything() }, + ], + }); - store.removeKeys(['3', '4']); + await store.removeKeys(['1']); - await expect(store.listKeys()).resolves.toEqual({ - items: [], - }); + await expect(store.listKeys()).resolves.toEqual({ + items: [ + { key: key2, createdAt: expect.anything() }, + { key: key3, createdAt: expect.anything() }, + ], + }); - await store.addKey(key1); - await store.addKey(key2); - await store.addKey(key3); + await store.removeKeys(['1', '2']); - await expect(store.listKeys()).resolves.toEqual({ - items: [ - { key: key1, createdAt: expect.anything() }, - { key: key2, createdAt: expect.anything() }, - { key: key3, createdAt: expect.anything() }, - ], - }); + await expect(store.listKeys()).resolves.toEqual({ + items: [{ key: key3, createdAt: expect.anything() }], + }); - store.removeKeys(['1', '2', '3']); + await store.removeKeys([]); - await expect(store.listKeys()).resolves.toEqual({ - items: [], - }); - }); + await expect(store.listKeys()).resolves.toEqual({ + items: [{ key: key3, createdAt: expect.anything() }], + }); + + await store.removeKeys(['3', '4']); + + await expect(store.listKeys()).resolves.toEqual({ + items: [], + }); + + await store.addKey(key1); + await store.addKey(key2); + await store.addKey(key3); + + await expect(store.listKeys()).resolves.toEqual({ + items: [ + { key: key1, createdAt: expect.anything() }, + { key: key2, createdAt: expect.anything() }, + { key: key3, createdAt: expect.anything() }, + ], + }); + + await store.removeKeys(['1', '2', '3']); + + await expect(store.listKeys()).resolves.toEqual({ + items: [], + }); + }, + ); }); diff --git a/plugins/auth-backend/src/identity/DatabaseKeyStore.ts b/plugins/auth-backend/src/identity/DatabaseKeyStore.ts index 8039837469..0559b698e9 100644 --- a/plugins/auth-backend/src/identity/DatabaseKeyStore.ts +++ b/plugins/auth-backend/src/identity/DatabaseKeyStore.ts @@ -14,19 +14,10 @@ * limitations under the License. */ -import { - PluginDatabaseManager, - resolvePackagePath, -} from '@backstage/backend-common'; import { Knex } from 'knex'; import { DateTime } from 'luxon'; import { AnyJWK, KeyStore, StoredKey } from './types'; -const migrationsDir = resolvePackagePath( - '@backstage/plugin-auth-backend', - 'migrations', -); - const TABLE = 'signing_keys'; type Row = { @@ -35,10 +26,6 @@ type Row = { key: string; }; -type Options = { - database: PluginDatabaseManager; -}; - const parseDate = (date: string | Date) => { const parsedDate = typeof date === 'string' @@ -55,24 +42,7 @@ const parseDate = (date: string | Date) => { }; export class DatabaseKeyStore implements KeyStore { - static async create(options: Options): Promise { - const { database } = options; - const client = await database.getClient(); - - if (!database.migrations?.skip) { - await client.migrate.latest({ - directory: migrationsDir, - }); - } - - return new DatabaseKeyStore(client); - } - - private readonly client: Knex; - - private constructor(client: Knex) { - this.client = client; - } + constructor(private readonly client: Knex) {} async addKey(key: AnyJWK): Promise { await this.client(TABLE).insert({ diff --git a/plugins/auth-backend/src/identity/KeyStores.test.ts b/plugins/auth-backend/src/identity/KeyStores.test.ts index 375db4698b..d186db6c46 100644 --- a/plugins/auth-backend/src/identity/KeyStores.test.ts +++ b/plugins/auth-backend/src/identity/KeyStores.test.ts @@ -14,13 +14,12 @@ * limitations under the License. */ -import { DatabaseManager } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; - -import { MemoryKeyStore } from './MemoryKeyStore'; +import { AuthDatabase } from '../database/AuthDatabase'; import { DatabaseKeyStore } from './DatabaseKeyStore'; import { FirestoreKeyStore } from './FirestoreKeyStore'; import { KeyStores } from './KeyStores'; +import { MemoryKeyStore } from './MemoryKeyStore'; describe('KeyStores', () => { const defaultConfigOptions = { @@ -46,18 +45,9 @@ describe('KeyStores', () => { }); it('can handle without auth config', async () => { - const config = new ConfigReader({ - backend: { - database: { - client: 'better-sqlite3', - connection: ':memory:', - }, - }, + const keyStore = await KeyStores.fromConfig(new ConfigReader({}), { + database: AuthDatabase.forTesting(), }); - const database = - DatabaseManager.fromConfig(config).forPlugin('auth-backend'); - const keyStore = await KeyStores.fromConfig(config, { database }); - expect(keyStore).toBeInstanceOf(DatabaseKeyStore); }); diff --git a/plugins/auth-backend/src/identity/KeyStores.ts b/plugins/auth-backend/src/identity/KeyStores.ts index e29b4b6fc4..d000a96879 100644 --- a/plugins/auth-backend/src/identity/KeyStores.ts +++ b/plugins/auth-backend/src/identity/KeyStores.ts @@ -14,20 +14,20 @@ * limitations under the License. */ -import { Logger } from 'winston'; import { pickBy } from 'lodash'; +import { Logger } from 'winston'; -import { PluginDatabaseManager } from '@backstage/backend-common'; import { Config } from '@backstage/config'; +import { AuthDatabase } from '../database/AuthDatabase'; import { DatabaseKeyStore } from './DatabaseKeyStore'; -import { MemoryKeyStore } from './MemoryKeyStore'; import { FirestoreKeyStore } from './FirestoreKeyStore'; +import { MemoryKeyStore } from './MemoryKeyStore'; import { KeyStore } from './types'; type Options = { logger?: Logger; - database?: PluginDatabaseManager; + database: AuthDatabase; }; export class KeyStores { @@ -52,8 +52,7 @@ export class KeyStores { if (!database) { throw new Error('This KeyStore provider requires a database'); } - - return await DatabaseKeyStore.create({ database }); + return new DatabaseKeyStore(await database.get()); } if (provider === 'memory') { diff --git a/plugins/auth-backend/src/migrations.test.ts b/plugins/auth-backend/src/migrations.test.ts new file mode 100644 index 0000000000..75df0de3e2 --- /dev/null +++ b/plugins/auth-backend/src/migrations.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import fs from 'fs'; + +const migrationsDir = `${__dirname}/../migrations`; +const migrationsFiles = fs.readdirSync(migrationsDir).sort(); + +async function migrateUpOnce(knex: Knex): Promise { + await knex.migrate.up({ directory: migrationsDir }); +} + +async function migrateDownOnce(knex: Knex): Promise { + await knex.migrate.down({ directory: migrationsDir }); +} + +async function migrateUntilBefore(knex: Knex, target: string): Promise { + const index = migrationsFiles.indexOf(target); + if (index === -1) { + throw new Error(`Migration ${target} not found`); + } + for (let i = 0; i < index; i++) { + await migrateUpOnce(knex); + } +} + +jest.setTimeout(60_000); + +describe('migrations', () => { + const databases = TestDatabases.create({ + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + }); + + it.each(databases.eachSupportedId())( + '20230428155633_sessions.js, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore(knex, '20230428155633_sessions.js'); + await migrateUpOnce(knex); + + // Ensure that large cookies are supported + const data = `{"cookie":"${'a'.repeat(100_000)}"}`; + await knex + .insert({ sid: 'abc', expired: knex.fn.now(), sess: data }) + .into('sessions'); + await knex + .insert({ sid: 'def', expired: knex.fn.now(), sess: data }) + .into('sessions'); + + await expect(knex('sessions').orderBy('sid', 'asc')).resolves.toEqual([ + { sid: 'abc', expired: expect.anything(), sess: data }, + { sid: 'def', expired: expect.anything(), sess: data }, + ]); + + await migrateDownOnce(knex); + + await expect(knex('sessions')).rejects.toThrow(); + + await knex.destroy(); + }, + ); +}); diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index 8013777415..2996ff68be 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -32,9 +32,11 @@ import { CatalogApi, CatalogClient } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; import { createOidcRouter, TokenFactory, KeyStores } from '../identity'; import session from 'express-session'; +import connectSessionKnex from 'connect-session-knex'; import passport from 'passport'; import { Minimatch } from 'minimatch'; import { CatalogAuthResolverContext } from '../lib/resolvers'; +import { AuthDatabase } from '../database/AuthDatabase'; /** @public */ export type ProviderFactories = { [s: string]: AuthProviderFactory }; @@ -70,7 +72,11 @@ export async function createRouter( const appUrl = config.getString('app.baseUrl'); const authUrl = await discovery.getExternalBaseUrl('auth'); - const keyStore = await KeyStores.fromConfig(config, { logger, database }); + const authDb = AuthDatabase.create(database); + const keyStore = await KeyStores.fromConfig(config, { + logger, + database: authDb, + }); const keyDurationSeconds = 3600; const tokenIssuer = new TokenFactory({ @@ -84,14 +90,18 @@ export async function createRouter( const secret = config.getOptionalString('auth.session.secret'); if (secret) { router.use(cookieParser(secret)); - // TODO: Configure the server-side session storage. The default MemoryStore is not designed for production const enforceCookieSSL = authUrl.startsWith('https'); + const KnexSessionStore = connectSessionKnex(session); router.use( session({ secret, saveUninitialized: false, resave: false, cookie: { secure: enforceCookieSSL ? 'auto' : false }, + store: new KnexSessionStore({ + createtable: false, + knex: await authDb.get(), + }), }), ); router.use(passport.initialize()); diff --git a/yarn.lock b/yarn.lock index d9be5d6064..f0648b5a13 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4796,6 +4796,7 @@ __metadata: "@types/passport-strategy": ^0.2.35 "@types/xml2js": ^0.4.7 compression: ^1.7.4 + connect-session-knex: ^3.0.1 cookie-parser: ^1.4.5 cors: ^2.8.5 express: ^4.17.1 @@ -20544,6 +20545,16 @@ __metadata: languageName: node linkType: hard +"connect-session-knex@npm:^3.0.1": + version: 3.0.1 + resolution: "connect-session-knex@npm:3.0.1" + dependencies: + bluebird: ^3.7.2 + knex: ^2.3.0 + checksum: f5a80c3c34d30e7cd4e79a6aae09d112825986d51ddc3a4563ef95ade425178239d4e3e05420fb3b6204b2353b0dfd754d318b288989b539583f6c0e52758b7a + languageName: node + linkType: hard + "consola@npm:^2.15.0": version: 2.15.3 resolution: "consola@npm:2.15.3" @@ -28982,7 +28993,7 @@ __metadata: languageName: node linkType: hard -"knex@npm:^2.0.0, knex@npm:^2.4.2": +"knex@npm:^2.0.0, knex@npm:^2.3.0, knex@npm:^2.4.2": version: 2.4.2 resolution: "knex@npm:2.4.2" dependencies: From af321b324c36910d08f3b1bf1172c6c49f65a00c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 4 May 2023 22:10:14 +0200 Subject: [PATCH 17/32] address comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- plugins/auth-backend/src/database/AuthDatabase.ts | 1 + .../auth-backend/src/identity/KeyStores.test.ts | 12 ++++++++++-- plugins/auth-backend/src/identity/KeyStores.ts | 14 ++++---------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/plugins/auth-backend/src/database/AuthDatabase.ts b/plugins/auth-backend/src/database/AuthDatabase.ts index 9c112e0aac..97e4236754 100644 --- a/plugins/auth-backend/src/database/AuthDatabase.ts +++ b/plugins/auth-backend/src/database/AuthDatabase.ts @@ -39,6 +39,7 @@ export class AuthDatabase { return new AuthDatabase(database); } + /** @internal */ static forTesting(): AuthDatabase { const config = new ConfigReader({ backend: { diff --git a/plugins/auth-backend/src/identity/KeyStores.test.ts b/plugins/auth-backend/src/identity/KeyStores.test.ts index d186db6c46..6c034eb64b 100644 --- a/plugins/auth-backend/src/identity/KeyStores.test.ts +++ b/plugins/auth-backend/src/identity/KeyStores.test.ts @@ -20,6 +20,7 @@ import { DatabaseKeyStore } from './DatabaseKeyStore'; import { FirestoreKeyStore } from './FirestoreKeyStore'; import { KeyStores } from './KeyStores'; import { MemoryKeyStore } from './MemoryKeyStore'; +import { getVoidLogger } from '@backstage/backend-common'; describe('KeyStores', () => { const defaultConfigOptions = { @@ -33,7 +34,10 @@ describe('KeyStores', () => { it('reads auth section from config', async () => { const configSpy = jest.spyOn(defaultConfig, 'getOptionalConfig'); - const keyStore = await KeyStores.fromConfig(defaultConfig); + const keyStore = await KeyStores.fromConfig(defaultConfig, { + logger: getVoidLogger(), + database: AuthDatabase.forTesting(), + }); expect(keyStore).toBeInstanceOf(MemoryKeyStore); expect(configSpy).toHaveBeenCalledWith('auth.keyStore'); @@ -46,6 +50,7 @@ describe('KeyStores', () => { it('can handle without auth config', async () => { const keyStore = await KeyStores.fromConfig(new ConfigReader({}), { + logger: getVoidLogger(), database: AuthDatabase.forTesting(), }); expect(keyStore).toBeInstanceOf(DatabaseKeyStore); @@ -72,7 +77,10 @@ describe('KeyStores', () => { }, }; const config = new ConfigReader(configOptions); - const keyStore = await KeyStores.fromConfig(config); + const keyStore = await KeyStores.fromConfig(config, { + logger: getVoidLogger(), + database: AuthDatabase.forTesting(), + }); expect(keyStore).toBeInstanceOf(FirestoreKeyStore); expect(createSpy).toHaveBeenCalledWith( diff --git a/plugins/auth-backend/src/identity/KeyStores.ts b/plugins/auth-backend/src/identity/KeyStores.ts index d000a96879..4cd8f91842 100644 --- a/plugins/auth-backend/src/identity/KeyStores.ts +++ b/plugins/auth-backend/src/identity/KeyStores.ts @@ -26,7 +26,7 @@ import { MemoryKeyStore } from './MemoryKeyStore'; import { KeyStore } from './types'; type Options = { - logger?: Logger; + logger: Logger; database: AuthDatabase; }; @@ -37,21 +37,15 @@ export class KeyStores { * * @returns a KeyStore store */ - static async fromConfig( - config: Config, - options?: Options, - ): Promise { - const { logger, database } = options ?? {}; + static async fromConfig(config: Config, options: Options): Promise { + const { logger, database } = options; const ks = config.getOptionalConfig('auth.keyStore'); const provider = ks?.getOptionalString('provider') ?? 'database'; - logger?.info(`Configuring "${provider}" as KeyStore provider`); + logger.info(`Configuring "${provider}" as KeyStore provider`); if (provider === 'database') { - if (!database) { - throw new Error('This KeyStore provider requires a database'); - } return new DatabaseKeyStore(await database.get()); } From ae397f8ef8c009294bda9effda95b2bd9ce4ddf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ladislav=20Vit=C3=A1sek?= Date: Fri, 5 May 2023 11:15:12 +0200 Subject: [PATCH 18/32] New `xkcd comic` plugin to the market place microsite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ladislav Vitásek --- microsite/data/plugins/xkcd.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 microsite/data/plugins/xkcd.yaml diff --git a/microsite/data/plugins/xkcd.yaml b/microsite/data/plugins/xkcd.yaml new file mode 100644 index 0000000000..370718a08f --- /dev/null +++ b/microsite/data/plugins/xkcd.yaml @@ -0,0 +1,10 @@ +--- +title: xkcd comic +author: Vity +authorUrl: https://github.com/Vity01 +category: Humor +description: xkcd comic within Backstage +documentation: https://github.com/Vity01/backstage-xkcd/ +iconUrl: https://raw.githubusercontent.com/Vity01/backstage-xkcd/main/docs/xkcd.svg +npmPackageName: 'backstage-plugin-xkcd' +addedDate: '2023-05-05' From 977aa9130023c003b29286950bde0e02d7c4c055 Mon Sep 17 00:00:00 2001 From: Zander Date: Fri, 5 May 2023 11:55:25 -0500 Subject: [PATCH 19/32] feat(scaffolder): DefaultTemplateOutputs test Signed-off-by: Zander --- .../DefaultTemplateOutputs.test.tsx | 56 +++++++++++++++++++ .../TemplateOutputs/TextOutputs.tsx | 1 + 2 files changed, 57 insertions(+) create mode 100644 plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.test.tsx diff --git a/plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.test.tsx b/plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.test.tsx new file mode 100644 index 0000000000..8366dcadf8 --- /dev/null +++ b/plugins/scaffolder-react/src/next/components/TemplateOutputs/DefaultTemplateOutputs.test.tsx @@ -0,0 +1,56 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { entityRouteRef } from '@backstage/plugin-catalog-react'; +import { renderInTestApp } from '@backstage/test-utils'; +import { fireEvent } from '@testing-library/react'; +import React from 'react'; +import { act } from 'react-dom/test-utils'; +import { DefaultTemplateOutputs } from '.'; + +describe('', () => { + it('should render template output', async () => { + const output = { + links: [{ title: 'Link 1', url: 'https://backstage.io/' }], + text: [{ title: 'Text 1', content: 'Hello, **world**!' }], + }; + + const { getByRole } = await renderInTestApp( + , + { + mountedRoutes: { + '/catalog/:namespace/:kind/:name': entityRouteRef, + }, + }, + ); + + // test link outputs + for (const link of output.links ?? []) { + expect( + getByRole('button', { name: link.title }).closest('a'), + ).toHaveAttribute('href', link.url); + } + + // test text outputs + for (const text of output.text ?? []) { + await act(async () => { + fireEvent.click(getByRole('button', { name: text.title })); + }); + + expect(getByRole('heading', { level: 2 }).innerHTML).toBe(text.title); + } + }); +}); diff --git a/plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx b/plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx index 4a9110d79c..7e64603e92 100644 --- a/plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx +++ b/plugins/scaffolder-react/src/next/components/TemplateOutputs/TextOutputs.tsx @@ -43,6 +43,7 @@ export const TextOutputs = (props: { const Icon = iconResolver(icon); return (