From f9a82d452e53ed8488019a09b765e2e1c30278a1 Mon Sep 17 00:00:00 2001 From: Steven Lougheed Date: Mon, 14 Feb 2022 09:44:12 -0500 Subject: [PATCH 001/150] Fixed path for loading cookiecutter.json Signed-off-by: slougheed --- .../src/actions/fetch/cookiecutter.ts | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts index c4d7f1e3e8..efc8b6215e 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts @@ -62,25 +62,34 @@ export class CookiecutterRunner { values: JsonObject; logStream: Writable; }): Promise { - const templateDir = path.join(workspacePath, 'template'); const intermediateDir = path.join(workspacePath, 'intermediate'); await fs.ensureDir(intermediateDir); const resultDir = path.join(workspacePath, 'result'); + const { + templateContentsDir, + templateDir, + imageName, + ...valuesForCookieCutterJson + } = values; // First lets grab the default cookiecutter.json file - const cookieCutterJson = await this.fetchTemplateCookieCutter(templateDir); + const cookieCutterJson = await this.fetchTemplateCookieCutter( + templateContentsDir as string, + ); - const { imageName, ...valuesForCookieCutterJson } = values; const cookieInfo = { ...cookieCutterJson, ...valuesForCookieCutterJson, }; - await fs.writeJSON(path.join(templateDir, 'cookiecutter.json'), cookieInfo); + await fs.writeJSON( + path.join(templateDir as string, 'cookiecutter.json'), + cookieInfo, + ); // Directories to bind on container const mountDirs = { - [templateDir]: '/input', + [templateDir as string]: '/input', [intermediateDir]: '/output', }; @@ -91,7 +100,13 @@ export class CookiecutterRunner { if (cookieCutterInstalled) { await runCommand({ command: 'cookiecutter', - args: ['--no-input', '-o', intermediateDir, templateDir, '--verbose'], + args: [ + '--no-input', + '-o', + intermediateDir, + templateDir as string, + '--verbose', + ], logStream, }); } else { @@ -233,6 +248,8 @@ export function createFetchCookiecutterAction(options: { _copy_without_render: ctx.input.copyWithoutRender, _extensions: ctx.input.extensions, imageName: ctx.input.imageName, + templateDir: templateDir, + templateContentsDir: templateContentsDir, }; // Will execute the template in ./template and put the result in ./result From 2a865343c2bcaa6c70fe123b0eff32debde42a0d Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Wed, 16 Feb 2022 16:42:33 +1100 Subject: [PATCH 002/150] Add a new interface: DocsBuildStrategy This adds a new interface called DocsBuildStrategy. This strategy allows for different answers to the question: should the TechDocs backend perform a build of the given entity? The default implementation replicates the existing functionality, in that if the techdocs.builder config value is set to local, then we do trigger builds, and if the value is set to anything else, we don't trigger builds. However the strategy has access to the entity, and so more complex strategies are possible. This strategy is added as an optional parameter on the RouterOptions types, and defaults to the DefaultDocsBuildStrategy if unspecified, allowing for backwards compatibility. TODO: rename some of the config options, and reword errors to reflect the new interface Signed-off-by: Nikolas Skoufis --- .../src/service/DocsBuildStrategy.test.ts | 63 +++++++++++++++++++ .../src/service/DocsBuildStrategy.ts | 33 ++++++++++ .../src/service/router.test.ts | 30 +++++---- .../techdocs-backend/src/service/router.ts | 11 +++- 4 files changed, 124 insertions(+), 13 deletions(-) create mode 100644 plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts create mode 100644 plugins/techdocs-backend/src/service/DocsBuildStrategy.ts diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts new file mode 100644 index 0000000000..71e5377a03 --- /dev/null +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DefaultDocsBuildStrategy } from './DocsBuildStrategy'; +import { ConfigReader } from '@backstage/config'; + +const MockedConfigReader = ConfigReader as jest.MockedClass< + typeof ConfigReader +>; + +jest.mock('@backstage/config'); + +describe('DefaultDocsBuildStrategy', () => { + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + uid: '0', + name: 'test', + }, + }; + + const config = new ConfigReader({}); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('shouldBuild', () => { + it('should return true when techdocs.build is set to local', async () => { + const defaultDocsBuildStrategy = new DefaultDocsBuildStrategy(config); + + MockedConfigReader.prototype.getString.mockReturnValue('local'); + + const result = await defaultDocsBuildStrategy.shouldBuild(entity); + + expect(result).toBe(true); + }); + + it('should return false when techdocs.build is set to external', async () => { + const defaultDocsBuildStrategy = new DefaultDocsBuildStrategy(config); + + MockedConfigReader.prototype.getString.mockReturnValue('external'); + + const result = await defaultDocsBuildStrategy.shouldBuild(entity); + + expect(result).toBe(false); + }); + }); +}); diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts new file mode 100644 index 0000000000..69b0474a85 --- /dev/null +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Entity } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; + +export interface DocsBuildStrategy { + shouldBuild(entity: Entity): Promise; +} + +export class DefaultDocsBuildStrategy { + private readonly config: Config; + + constructor(config: Config) { + this.config = config; + } + + async shouldBuild(_: Entity): Promise { + return this.config.getString('techdocs.builder') === 'local'; + } +} diff --git a/plugins/techdocs-backend/src/service/router.test.ts b/plugins/techdocs-backend/src/service/router.test.ts index 43e9641efb..521024252c 100644 --- a/plugins/techdocs-backend/src/service/router.test.ts +++ b/plugins/techdocs-backend/src/service/router.test.ts @@ -38,6 +38,7 @@ import { RouterOptions, } from './router'; import { TechDocsCache } from '../cache'; +import { DocsBuildStrategy } from './DocsBuildStrategy'; jest.mock('@backstage/catalog-client'); jest.mock('@backstage/config'); @@ -120,6 +121,9 @@ describe('createRouter', () => { const cache: jest.Mocked = { getClient: jest.fn(), }; + const docsBuildStrategy: jest.Mocked = { + shouldBuild: jest.fn(), + }; const outOfTheBoxOptions = { preparers, generators, @@ -128,6 +132,7 @@ describe('createRouter', () => { logger: getVoidLogger(), discovery, cache, + docsBuildStrategy, }; const recommendedOptions = { publisher, @@ -135,6 +140,7 @@ describe('createRouter', () => { logger: getVoidLogger(), discovery, cache, + docsBuildStrategy, }; beforeEach(() => { @@ -181,10 +187,10 @@ describe('createRouter', () => { expect(response.status).toBe(404); }); - it('should not check for an update without local builder', async () => { + it('should not check for an update when shouldBuild returns false', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('external'); + docsBuildStrategy.shouldBuild.mockResolvedValue(false); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doCacheSync.mockImplementation( async ({ responseHandler }) => @@ -198,10 +204,10 @@ describe('createRouter', () => { expect(response.status).toBe(304); }); - it('should error if missing builder', async () => { + it('should error if build is required and is missing preparer', async () => { const app = await createApp(recommendedOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); const response = await request(app) @@ -219,7 +225,7 @@ describe('createRouter', () => { it('should execute synchronization', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => @@ -244,7 +250,7 @@ describe('createRouter', () => { it('should return on updated', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => { @@ -297,10 +303,10 @@ describe('createRouter', () => { expect(response.status).toBe(404); }); - it('should not check for an update without local builder', async () => { + it('should not check for an update when shouldBuild returns false', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('external'); + docsBuildStrategy.shouldBuild.mockResolvedValue(false); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doCacheSync.mockImplementation( async ({ responseHandler }) => @@ -322,10 +328,10 @@ data: {"updated":false} ); }); - it('should error if missing builder', async () => { + it('should error if build is required and is missing preparer', async () => { const app = await createApp(recommendedOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); const response = await request(app) @@ -348,7 +354,7 @@ data: "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'prep it('should execute synchronization', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => @@ -376,7 +382,7 @@ data: "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'prep it('should return an event-stream', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => { diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 141ce93f79..96916bdc9b 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -35,6 +35,10 @@ import { ScmIntegrations } from '@backstage/integration'; import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer'; import { createCacheMiddleware, TechDocsCache } from '../cache'; import { CachedEntityLoader } from './CachedEntityLoader'; +import { + DefaultDocsBuildStrategy, + DocsBuildStrategy, +} from './DocsBuildStrategy'; /** * All of the required dependencies for running TechDocs in the "out-of-the-box" @@ -49,6 +53,7 @@ export type OutOfTheBoxDeploymentOptions = { database?: Knex; // TODO: Make database required when we're implementing database stuff. config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; /** @@ -61,6 +66,7 @@ export type RecommendedDeploymentOptions = { discovery: PluginEndpointDiscovery; config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; /** @@ -86,6 +92,8 @@ export async function createRouter( const router = Router(); const { publisher, config, logger, discovery } = options; const catalogClient = new CatalogClient({ discoveryApi: discovery }); + const docsBuildStrategy = + options.docsBuildStrategy ?? new DefaultDocsBuildStrategy(config); // Entities are cached to optimize the /static/docs request path, which can be called many times // when loading a single techdocs page. @@ -200,7 +208,8 @@ export async function createRouter( // techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local' // If set to 'external', it will assume that an external process (e.g. CI/CD pipeline // of the repository) is responsible for building and publishing documentation to the storage provider - if (config.getString('techdocs.builder') !== 'local') { + const shouldBuild = await docsBuildStrategy.shouldBuild(entity); + if (!shouldBuild) { // However, if caching is enabled, take the opportunity to check and // invalidate stale cache entries. if (cache) { From 0d1c461780fad011bf48d46420f12b2be473e65f Mon Sep 17 00:00:00 2001 From: su-gupta Date: Tue, 22 Feb 2022 18:11:14 -0500 Subject: [PATCH 003/150] adding Confluence search functionality upstream v1 Signed-off-by: su-gupta --- contrib/search/README.md | 3 + contrib/search/confluence/ConfluenceCollator | 87 +++++++++++++++++++ .../confluence/ConfluenceResultListItem | 50 +++++++++++ contrib/search/confluence/README.md | 6 ++ 4 files changed, 146 insertions(+) create mode 100644 contrib/search/README.md create mode 100644 contrib/search/confluence/ConfluenceCollator create mode 100644 contrib/search/confluence/ConfluenceResultListItem create mode 100644 contrib/search/confluence/README.md diff --git a/contrib/search/README.md b/contrib/search/README.md new file mode 100644 index 0000000000..9a43a05c4c --- /dev/null +++ b/contrib/search/README.md @@ -0,0 +1,3 @@ +# Search + +Contributions/extensions to the Search plugin diff --git a/contrib/search/confluence/ConfluenceCollator b/contrib/search/confluence/ConfluenceCollator new file mode 100644 index 0000000000..5d6668b953 --- /dev/null +++ b/contrib/search/confluence/ConfluenceCollator @@ -0,0 +1,87 @@ +import { DocumentCollator } from '@backstage/search-common'; +import fetch from 'cross-fetch' + +export class ConfluenceCollator implements DocumentCollator { + public readonly type: string = 'confluence'; + + async execute() { + + const ConfluenceUrlBase = 'https://{CONFLUENCE-ORG-NAME}.atlassian.net/wiki/rest/api' + + async function getConfluenceData(requestUrl: string) { + var emptyJson = {} + try { + const res = await fetch(requestUrl, { + method: 'get', + headers: { + 'Authorization': `Basic ${process.env.CONFLUENCE_TOKEN}` + }, + }); + if (res.ok) { + return await res.json(); + } + } catch (err) { + console.error(err); + } + return emptyJson + } + + async function getSpaces(): Promise { + const data = await getConfluenceData(`${ConfluenceUrlBase}/space?&limit=1000&type=global&status=current`); + let spacesList = [] + if (data["results"]) { + const results = data["results"]; + for (const result of results) { + spacesList.push(result["key"]) + } + } + return spacesList + } + + async function getDocumentsFromSpaces(spaces: string[]): Promise { + let documentsList = [] + for (var space of spaces) { + let next = true + let requestUrl = `${ConfluenceUrlBase}/content?limit=1000&status=current&spaceKey=${space}` + while (next) { + const data = await getConfluenceData(requestUrl) + if (data["results"]) { + const results = data["results"] + for (const result of results) { + documentsList.push(result["_links"]["self"]) + } + if (data["_links"]["next"]) { + requestUrl = data["_links"]["base"] + data["_links"]["next"] + } else { + next = false + } + } else { + break + } + } + } + return documentsList + } + + async function getDocumentInfo(documents: string[]) { + let documentInfo = [] + for (var documentUrl of documents) { + const data = await getConfluenceData(documentUrl + '?expand=body.storage') + if (data["status"] && data["status"]=="current") { + const documentMetaData = { + title: data["title"], + text: data["body"]["storage"]["value"], + location: data["_links"]["base"] + data["_links"]["webui"], + } + documentInfo.push(documentMetaData) + } + } + return documentInfo + } + + const spacesList = await getSpaces(); + const documentsList = await getDocumentsFromSpaces(spacesList); + const documentMetaDataList = await getDocumentInfo(documentsList); + return documentMetaDataList + } +} diff --git a/contrib/search/confluence/ConfluenceResultListItem b/contrib/search/confluence/ConfluenceResultListItem new file mode 100644 index 0000000000..cd6d13bdea --- /dev/null +++ b/contrib/search/confluence/ConfluenceResultListItem @@ -0,0 +1,50 @@ +import React from 'react'; +import { Link } from '@backstage/core-components'; +import { IndexableDocument } from '@backstage/search-common'; +import { + Divider, + ListItem, + ListItemIcon, + ListItemText, +} from '@material-ui/core'; + +type Props = { + result: IndexableDocument; +}; + +export const ConfluenceResultListItem = ({ result }: Props) => { + // Remove html tags from document text before displaying + const chars = []; + let isTag = false; + for (const c of result.text.substring(0, 500)) { + if (c === "<") { + isTag = true; + continue; + } + if (c === ">") { + isTag = false; + chars.push(" ") + continue; + } + if (!isTag) { + chars.push(c); + } + } + const excerpt = chars.join("").substring(0, 80) + (result.text.length > 80 ? "..." : ""); + + return ( + + + + + + + + + + ); +}; diff --git a/contrib/search/confluence/README.md b/contrib/search/confluence/README.md new file mode 100644 index 0000000000..5ebd275560 --- /dev/null +++ b/contrib/search/confluence/README.md @@ -0,0 +1,6 @@ +# Confluence + +These files help you add Confluence as a source to the Backstage Search plugin. +To do so, add both files in this directory under the packages/backend/src/plugins/search/ pathway in your Backstage app as TypeScript files. +Then, update your packages/app/src/components/search/SearchPage.tsx and packages/backend/src/plugins/search.ts +to include the new Search source. From 0ce1d75871954ef8d59e5e489cc1e478d9ace0b7 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 25 Feb 2022 09:26:56 +0100 Subject: [PATCH 004/150] core-plugin-api: Remove generic auth APIs Signed-off-by: Johan Haals --- packages/app-defaults/src/defaults/apis.ts | 66 ------------- packages/app/src/identityProviders.ts | 21 ---- packages/core-app-api/api-report.md | 7 -- .../implementations/auth/auth0/Auth0Auth.ts | 67 ------------- .../apis/implementations/auth/auth0/index.ts | 17 ---- .../src/apis/implementations/auth/index.ts | 1 - .../src/layout/SignInPage/auth0Provider.tsx | 95 ------------------- packages/core-plugin-api/api-report.md | 28 ------ .../src/apis/definitions/auth.ts | 61 ------------ .../test-utils/src/testUtils/defaultApis.ts | 66 ------------- .../AuthProviders/DefaultProviderSettings.tsx | 18 ---- 11 files changed, 447 deletions(-) delete mode 100644 packages/core-app-api/src/apis/implementations/auth/auth0/Auth0Auth.ts delete mode 100644 packages/core-app-api/src/apis/implementations/auth/auth0/index.ts delete mode 100644 packages/core-components/src/layout/SignInPage/auth0Provider.tsx diff --git a/packages/app-defaults/src/defaults/apis.ts b/packages/app-defaults/src/defaults/apis.ts index f26c298fb1..3f5cfc1c58 100644 --- a/packages/app-defaults/src/defaults/apis.ts +++ b/packages/app-defaults/src/defaults/apis.ts @@ -21,16 +21,13 @@ import { ErrorAlerter, GoogleAuth, GithubAuth, - OAuth2, OktaAuth, GitlabAuth, - Auth0Auth, MicrosoftAuth, BitbucketAuth, OAuthRequestManager, WebStorage, UrlPatternDiscovery, - SamlAuth, OneLoginAuth, UnhandledErrorForwarder, AtlassianAuth, @@ -49,16 +46,12 @@ import { oauthRequestApiRef, googleAuthApiRef, githubAuthApiRef, - oauth2ApiRef, oktaAuthApiRef, gitlabAuthApiRef, - auth0AuthApiRef, microsoftAuthApiRef, storageApiRef, configApiRef, - samlAuthApiRef, oneloginAuthApiRef, - oidcAuthApiRef, bitbucketAuthApiRef, atlassianAuthApiRef, } from '@backstage/core-plugin-api'; @@ -197,46 +190,6 @@ export const apis = [ environment: configApi.getOptionalString('auth.environment'), }), }), - createApiFactory({ - api: auth0AuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - Auth0Auth.create({ - discoveryApi, - oauthRequestApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), - createApiFactory({ - api: oauth2ApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - OAuth2.create({ - discoveryApi, - oauthRequestApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), - createApiFactory({ - api: samlAuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, configApi }) => - SamlAuth.create({ - discoveryApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), createApiFactory({ api: oneloginAuthApiRef, deps: { @@ -251,25 +204,6 @@ export const apis = [ environment: configApi.getOptionalString('auth.environment'), }), }), - createApiFactory({ - api: oidcAuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - OAuth2.create({ - discoveryApi, - oauthRequestApi, - provider: { - id: 'oidc', - title: 'Your Identity Provider', - icon: () => null, - }, - environment: configApi.getOptionalString('auth.environment'), - }), - }), createApiFactory({ api: bitbucketAuthApiRef, deps: { diff --git a/packages/app/src/identityProviders.ts b/packages/app/src/identityProviders.ts index 49204d2b5e..5f8c0faf47 100644 --- a/packages/app/src/identityProviders.ts +++ b/packages/app/src/identityProviders.ts @@ -19,27 +19,12 @@ import { gitlabAuthApiRef, oktaAuthApiRef, githubAuthApiRef, - samlAuthApiRef, microsoftAuthApiRef, oneloginAuthApiRef, - oauth2ApiRef, - oidcAuthApiRef, bitbucketAuthApiRef, } from '@backstage/core-plugin-api'; export const providers = [ - { - id: 'oidc-auth-provider', - title: 'Oidc', - message: 'Sign In using OpenId Connect', - apiRef: oidcAuthApiRef, - }, - { - id: 'oauth2-auth-provider', - title: 'OAuth 2.0', - message: 'Sign In using OAuth 2.0', - apiRef: oauth2ApiRef, - }, { id: 'google-auth-provider', title: 'Google', @@ -70,12 +55,6 @@ export const providers = [ message: 'Sign In using Okta', apiRef: oktaAuthApiRef, }, - { - id: 'saml-auth-provider', - title: 'SAML', - message: 'Sign In using SAML', - apiRef: samlAuthApiRef, - }, { id: 'onelogin-auth-provider', title: 'OneLogin', diff --git a/packages/core-app-api/api-report.md b/packages/core-app-api/api-report.md index 3b53813615..17a9592523 100644 --- a/packages/core-app-api/api-report.md +++ b/packages/core-app-api/api-report.md @@ -16,7 +16,6 @@ import { AppConfig } from '@backstage/config'; import { AppTheme } from '@backstage/core-plugin-api'; import { AppThemeApi } from '@backstage/core-plugin-api'; import { atlassianAuthApiRef } from '@backstage/core-plugin-api'; -import { auth0AuthApiRef } from '@backstage/core-plugin-api'; import { AuthProviderInfo } from '@backstage/core-plugin-api'; import { AuthRequestOptions } from '@backstage/core-plugin-api'; import { BackstageIdentityApi } from '@backstage/core-plugin-api'; @@ -246,12 +245,6 @@ export class AtlassianAuth { static create(options: OAuthApiCreateOptions): typeof atlassianAuthApiRef.T; } -// @public @deprecated -export class Auth0Auth { - // (undocumented) - static create(options: OAuthApiCreateOptions): typeof auth0AuthApiRef.T; -} - // @public export type AuthApiCreateOptions = { discoveryApi: DiscoveryApi; diff --git a/packages/core-app-api/src/apis/implementations/auth/auth0/Auth0Auth.ts b/packages/core-app-api/src/apis/implementations/auth/auth0/Auth0Auth.ts deleted file mode 100644 index d8f942b94b..0000000000 --- a/packages/core-app-api/src/apis/implementations/auth/auth0/Auth0Auth.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { auth0AuthApiRef } from '@backstage/core-plugin-api'; -import { OAuth2 } from '../oauth2'; -import { OAuthApiCreateOptions } from '../types'; - -const DEFAULT_PROVIDER = { - id: 'auth0', - title: 'Auth0', - icon: () => null, -}; - -/** - * Implements the OAuth flow to Auth0 products. - * - * @public - * @deprecated Use {@link OAuth2} instead - * - * @example - * - * ```ts - * OAuth2.create({ - * discoveryApi, - * oauthRequestApi, - * provider: { - * id: 'auth0', - * title: 'Auth0', - * icon: () => null, - * }, - * defaultScopes: ['openid', 'email', 'profile'], - * environment: configApi.getOptionalString('auth.environment'), - * }) - * ``` - */ -export default class Auth0Auth { - static create(options: OAuthApiCreateOptions): typeof auth0AuthApiRef.T { - const { - discoveryApi, - environment = 'development', - provider = DEFAULT_PROVIDER, - oauthRequestApi, - defaultScopes = ['openid', `email`, `profile`], - } = options; - - return OAuth2.create({ - discoveryApi, - oauthRequestApi, - provider, - environment, - defaultScopes, - }); - } -} diff --git a/packages/core-app-api/src/apis/implementations/auth/auth0/index.ts b/packages/core-app-api/src/apis/implementations/auth/auth0/index.ts deleted file mode 100644 index 9daed7d13e..0000000000 --- a/packages/core-app-api/src/apis/implementations/auth/auth0/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { default as Auth0Auth } from './Auth0Auth'; diff --git a/packages/core-app-api/src/apis/implementations/auth/index.ts b/packages/core-app-api/src/apis/implementations/auth/index.ts index 50333f07a0..c4cf520db5 100644 --- a/packages/core-app-api/src/apis/implementations/auth/index.ts +++ b/packages/core-app-api/src/apis/implementations/auth/index.ts @@ -20,7 +20,6 @@ export * from './google'; export * from './oauth2'; export * from './okta'; export * from './saml'; -export * from './auth0'; export * from './microsoft'; export * from './onelogin'; export * from './bitbucket'; diff --git a/packages/core-components/src/layout/SignInPage/auth0Provider.tsx b/packages/core-components/src/layout/SignInPage/auth0Provider.tsx deleted file mode 100644 index 739c3709a9..0000000000 --- a/packages/core-components/src/layout/SignInPage/auth0Provider.tsx +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React from 'react'; -import Grid from '@material-ui/core/Grid'; -import Typography from '@material-ui/core/Typography'; -import Button from '@material-ui/core/Button'; -import { InfoCard } from '../InfoCard/InfoCard'; -import { ProviderComponent, ProviderLoader, SignInProvider } from './types'; -import { - useApi, - auth0AuthApiRef, - errorApiRef, -} from '@backstage/core-plugin-api'; -import { ForwardedError } from '@backstage/errors'; -import { UserIdentity } from './UserIdentity'; - -const Component: ProviderComponent = ({ onSignInSuccess }) => { - const auth0AuthApi = useApi(auth0AuthApiRef); - const errorApi = useApi(errorApiRef); - - const handleLogin = async () => { - try { - const identityResponse = await auth0AuthApi.getBackstageIdentity({ - instantPopup: true, - }); - if (!identityResponse) { - throw new Error( - 'The Auth0 provider is not configured to support sign-in', - ); - } - - const profile = await auth0AuthApi.getProfile(); - - onSignInSuccess( - UserIdentity.create({ - identity: identityResponse.identity, - authApi: auth0AuthApi, - profile, - }), - ); - } catch (error) { - errorApi.post(new ForwardedError('Auth0 login failed', error)); - } - }; - - return ( - - - Sign In - - } - > - Sign In using Auth0 - - - ); -}; - -const loader: ProviderLoader = async apis => { - const auth0AuthApi = apis.get(auth0AuthApiRef)!; - - const identityResponse = await auth0AuthApi.getBackstageIdentity({ - optional: true, - }); - - if (!identityResponse) { - return undefined; - } - - const profile = await auth0AuthApi.getProfile(); - return UserIdentity.create({ - identity: identityResponse.identity, - authApi: auth0AuthApi, - profile, - }); -}; - -export const auth0Provider: SignInProvider = { Component, loader }; diff --git a/packages/core-plugin-api/api-report.md b/packages/core-plugin-api/api-report.md index 1647d1c2cb..2b0e061cce 100644 --- a/packages/core-plugin-api/api-report.md +++ b/packages/core-plugin-api/api-report.md @@ -184,11 +184,6 @@ export function attachComponentData

( data: unknown, ): void; -// @public @deprecated -export const auth0AuthApiRef: ApiRef< - OpenIdConnectApi & ProfileInfoApi & BackstageIdentityApi & SessionApi ->; - // @public export type AuthProviderInfo = { id: string; @@ -531,15 +526,6 @@ export const microsoftAuthApiRef: ApiRef< SessionApi >; -// @public @deprecated -export const oauth2ApiRef: ApiRef< - OAuthApi & - OpenIdConnectApi & - ProfileInfoApi & - BackstageIdentityApi & - SessionApi ->; - // @public export type OAuthApi = { getAccessToken( @@ -575,15 +561,6 @@ export type OAuthRequesterOptions = { // @public export type OAuthScope = string | string[]; -// @public @deprecated -export const oidcAuthApiRef: ApiRef< - OAuthApi & - OpenIdConnectApi & - ProfileInfoApi & - BackstageIdentityApi & - SessionApi ->; - // @alpha export const oktaAuthApiRef: ApiRef< OAuthApi & @@ -684,11 +661,6 @@ export type RouteRef = { params: ParamKeys; }; -// @public @deprecated -export const samlAuthApiRef: ApiRef< - ProfileInfoApi & BackstageIdentityApi & SessionApi ->; - // @public export type SessionApi = { signIn(): Promise; diff --git a/packages/core-plugin-api/src/apis/definitions/auth.ts b/packages/core-plugin-api/src/apis/definitions/auth.ts index ed5a7a3264..497ce8233b 100644 --- a/packages/core-plugin-api/src/apis/definitions/auth.ts +++ b/packages/core-plugin-api/src/apis/definitions/auth.ts @@ -366,23 +366,6 @@ export const gitlabAuthApiRef: ApiRef< id: 'core.auth.gitlab', }); -/** - * Provides authentication towards Auth0 APIs. - * - * @remarks - * - * See {@link https://auth0.com/docs/scopes/current/oidc-scopes} - * for a full list of supported scopes. - * - * @public - * @deprecated See https://backstage.io/docs/api/deprecations#generic-auth-api-refs - */ -export const auth0AuthApiRef: ApiRef< - OpenIdConnectApi & ProfileInfoApi & BackstageIdentityApi & SessionApi -> = createApiRef({ - id: 'core.auth.auth0', -}); - /** * Provides authentication towards Microsoft APIs and identities. * @@ -404,50 +387,6 @@ export const microsoftAuthApiRef: ApiRef< id: 'core.auth.microsoft', }); -/** - * Provides authentication for custom identity providers. - * - * @public - * @deprecated See https://backstage.io/docs/api/deprecations#generic-auth-api-refs - */ -export const oauth2ApiRef: ApiRef< - OAuthApi & - OpenIdConnectApi & - ProfileInfoApi & - BackstageIdentityApi & - SessionApi -> = createApiRef({ - id: 'core.auth.oauth2', -}); - -/** - * Provides authentication for custom OpenID Connect identity providers. - * - * @public - * @deprecated See https://backstage.io/docs/api/deprecations#generic-auth-api-refs - */ -export const oidcAuthApiRef: ApiRef< - OAuthApi & - OpenIdConnectApi & - ProfileInfoApi & - BackstageIdentityApi & - SessionApi -> = createApiRef({ - id: 'core.auth.oidc', -}); - -/** - * Provides authentication for SAML-based identity providers. - * - * @public - * @deprecated See https://backstage.io/docs/api/deprecations#generic-auth-api-refs - */ -export const samlAuthApiRef: ApiRef< - ProfileInfoApi & BackstageIdentityApi & SessionApi -> = createApiRef({ - id: 'core.auth.saml', -}); - /** * Provides authentication towards OneLogin APIs. * diff --git a/packages/test-utils/src/testUtils/defaultApis.ts b/packages/test-utils/src/testUtils/defaultApis.ts index f06e1ba6b7..1ff10bfe6f 100644 --- a/packages/test-utils/src/testUtils/defaultApis.ts +++ b/packages/test-utils/src/testUtils/defaultApis.ts @@ -21,16 +21,13 @@ import { ErrorAlerter, GoogleAuth, GithubAuth, - OAuth2, OktaAuth, GitlabAuth, - Auth0Auth, MicrosoftAuth, BitbucketAuth, OAuthRequestManager, WebStorage, UrlPatternDiscovery, - SamlAuth, OneLoginAuth, UnhandledErrorForwarder, AtlassianAuth, @@ -45,16 +42,12 @@ import { oauthRequestApiRef, googleAuthApiRef, githubAuthApiRef, - oauth2ApiRef, oktaAuthApiRef, gitlabAuthApiRef, - auth0AuthApiRef, microsoftAuthApiRef, storageApiRef, configApiRef, - samlAuthApiRef, oneloginAuthApiRef, - oidcAuthApiRef, bitbucketAuthApiRef, atlassianAuthApiRef, } from '@backstage/core-plugin-api'; @@ -158,46 +151,6 @@ export const defaultApis = [ environment: configApi.getOptionalString('auth.environment'), }), }), - createApiFactory({ - api: auth0AuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - Auth0Auth.create({ - discoveryApi, - oauthRequestApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), - createApiFactory({ - api: oauth2ApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - OAuth2.create({ - discoveryApi, - oauthRequestApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), - createApiFactory({ - api: samlAuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, configApi }) => - SamlAuth.create({ - discoveryApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), createApiFactory({ api: oneloginAuthApiRef, deps: { @@ -212,25 +165,6 @@ export const defaultApis = [ environment: configApi.getOptionalString('auth.environment'), }), }), - createApiFactory({ - api: oidcAuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - OAuth2.create({ - discoveryApi, - oauthRequestApi, - provider: { - id: 'oidc', - title: 'Your Identity Provider', - icon: () => null, - }, - environment: configApi.getOptionalString('auth.environment'), - }), - }), createApiFactory({ api: bitbucketAuthApiRef, deps: { diff --git a/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx b/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx index 8cfb3a7257..7025167658 100644 --- a/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx +++ b/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx @@ -17,11 +17,9 @@ import Star from '@material-ui/icons/Star'; import React from 'react'; import { ProviderSettingsItem } from './ProviderSettingsItem'; import { - auth0AuthApiRef, githubAuthApiRef, gitlabAuthApiRef, googleAuthApiRef, - oauth2ApiRef, oktaAuthApiRef, microsoftAuthApiRef, bitbucketAuthApiRef, @@ -66,14 +64,6 @@ export const DefaultProviderSettings = ({ configuredProviders }: Props) => ( icon={Star} /> )} - {configuredProviders.includes('auth0') && ( - - )} {configuredProviders.includes('okta') && ( ( icon={Star} /> )} - {configuredProviders.includes('oauth2') && ( - - )} ); From af5eaa87f43fa63c503fb64fe07b9ceacce47048 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 25 Feb 2022 09:43:51 +0100 Subject: [PATCH 005/150] add changeset Signed-off-by: Johan Haals --- .changeset/rare-insects-punch.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/rare-insects-punch.md diff --git a/.changeset/rare-insects-punch.md b/.changeset/rare-insects-punch.md new file mode 100644 index 0000000000..d22b851abd --- /dev/null +++ b/.changeset/rare-insects-punch.md @@ -0,0 +1,10 @@ +--- +'@backstage/app-defaults': minor +'@backstage/core-app-api': minor +'@backstage/core-components': minor +'@backstage/core-plugin-api': minor +'@backstage/test-utils': minor +'@backstage/plugin-user-settings': minor +--- + +**BREAKING**: Removed deprecated `auth0AuthApiRef`, `oauth2ApiRef`, `samlAuthApiRef` and `oidcAuthApiRef` as these APIs are too generic to be useful. Instructions for how to migrate can be found at [https://backstage.io/docs/api/deprecations#generic-auth-api-refs](https://backstage.io/docs/api/deprecations#generic-auth-api-refs). From 34af86517c8cc8caf0cae77692021ae867417e11 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Thu, 24 Feb 2022 12:45:04 +0100 Subject: [PATCH 006/150] feat(bitbucket): ensure apiBaseUrl, replace hardcoded cases Ensure presence of apiBaseUrl for bitbucket integrations for both cases Bitbucket Cloud and Bitbucket Server by setting the default for Bitbucket Server at the integration config, too. Replace hardcoded uses of the default apiBaseUrl with the use of the integration config's value. Signed-off-by: Patrick Jungermann --- .changeset/honest-students-clean.md | 7 +++++++ .../src/reading/BitbucketUrlReader.test.ts | 16 ---------------- .../src/reading/BitbucketUrlReader.ts | 9 ++------- packages/integration/api-report.md | 2 +- packages/integration/src/bitbucket/config.ts | 10 +++++----- packages/integration/src/helpers.test.ts | 10 ++++++++-- .../actions/builtin/publish/bitbucket.ts | 18 ++++++++++-------- 7 files changed, 33 insertions(+), 39 deletions(-) create mode 100644 .changeset/honest-students-clean.md diff --git a/.changeset/honest-students-clean.md b/.changeset/honest-students-clean.md new file mode 100644 index 0000000000..ca8db28343 --- /dev/null +++ b/.changeset/honest-students-clean.md @@ -0,0 +1,7 @@ +--- +'@backstage/integration': minor +'@backstage/backend-common': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +ensure `apiBaseUrl` being set for Bitbucket integrations, replace hardcoded defaults diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index ea8e4626be..371e11b364 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -270,22 +270,6 @@ describe('BitbucketUrlReader', () => { expect(response.etag).toBe('12ab34cd56ef'); }); - - it('should throw error when apiBaseUrl is missing', () => { - expect(() => { - /* eslint-disable no-new */ - new BitbucketUrlReader( - new BitbucketIntegration( - readBitbucketIntegrationConfig( - new ConfigReader({ - host: 'bitbucket.mycompany.net', - }), - ), - ), - { treeResponseFactory }, - ); - }).toThrowError('must configure an explicit apiBaseUrl'); - }); }); describe('search hosted', () => { diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 2006637545..e22137ea69 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -62,14 +62,9 @@ export class BitbucketUrlReader implements UrlReader { private readonly integration: BitbucketIntegration, private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, ) { - const { host, apiBaseUrl, token, username, appPassword } = - integration.config; + const { host, token, username, appPassword } = integration.config; - if (!apiBaseUrl) { - throw new Error( - `Bitbucket integration for '${host}' must configure an explicit apiBaseUrl`, - ); - } else if (!token && username && !appPassword) { + if (!token && username && !appPassword) { throw new Error( `Bitbucket integration for '${host}' has configured a username but is missing a required appPassword.`, ); diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 10da05784c..9a6ba03f8e 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -88,7 +88,7 @@ export class BitbucketIntegration implements ScmIntegration { // @public export type BitbucketIntegrationConfig = { host: string; - apiBaseUrl?: string; + apiBaseUrl: string; token?: string; username?: string; appPassword?: string; diff --git a/packages/integration/src/bitbucket/config.ts b/packages/integration/src/bitbucket/config.ts index 1cd911aad3..44a2f0cc1f 100644 --- a/packages/integration/src/bitbucket/config.ts +++ b/packages/integration/src/bitbucket/config.ts @@ -36,12 +36,10 @@ export type BitbucketIntegrationConfig = { * The base URL of the API of this provider, e.g. "https://api.bitbucket.org/2.0", * with no trailing slash. * - * May be omitted specifically for Bitbucket Cloud; then it will be deduced. - * - * The API will always be preferred if both its base URL and a token are - * present. + * Values omitted at the optional property at the app-config will be deduced + * from the "host" value. */ - apiBaseUrl?: string; + apiBaseUrl: string; /** * The authorization token to use for requests to a Bitbucket Server provider. @@ -90,6 +88,8 @@ export function readBitbucketIntegrationConfig( apiBaseUrl = trimEnd(apiBaseUrl, '/'); } else if (host === BITBUCKET_HOST) { apiBaseUrl = BITBUCKET_API_BASE_URL; + } else { + apiBaseUrl = `https://${host}/rest/api/1.0`; } return { diff --git a/packages/integration/src/helpers.test.ts b/packages/integration/src/helpers.test.ts index a69c7faec9..60a2789f05 100644 --- a/packages/integration/src/helpers.test.ts +++ b/packages/integration/src/helpers.test.ts @@ -24,7 +24,10 @@ import { describe('basicIntegrations', () => { describe('byUrl', () => { it('handles hosts without a port', () => { - const integration = new BitbucketIntegration({ host: 'host.com' }); + const integration = new BitbucketIntegration({ + host: 'host.com', + apiBaseUrl: 'a', + }); const integrations = basicIntegrations( [integration], i => i.config.host, @@ -33,7 +36,10 @@ describe('basicIntegrations', () => { expect(integrations.byUrl('https://host.com:8080/a')).toBeUndefined(); }); it('handles hosts with a port', () => { - const integration = new BitbucketIntegration({ host: 'host.com:8080' }); + const integration = new BitbucketIntegration({ + host: 'host.com:8080', + apiBaseUrl: 'a', + }); const integrations = basicIntegrations( [integration], i => i.config.host, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts index 47ebce0550..494fca1af9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -32,6 +32,7 @@ const createBitbucketCloudRepository = async (opts: { description?: string; repoVisibility: 'private' | 'public'; authorization: string; + apiBaseUrl: string; }) => { const { workspace, @@ -40,6 +41,7 @@ const createBitbucketCloudRepository = async (opts: { description, repoVisibility, authorization, + apiBaseUrl, } = opts; const options: RequestInit = { @@ -59,7 +61,7 @@ const createBitbucketCloudRepository = async (opts: { let response: Response; try { response = await fetch( - `https://api.bitbucket.org/2.0/repositories/${workspace}/${repo}`, + `${apiBaseUrl}/repositories/${workspace}/${repo}`, options, ); } catch (e) { @@ -88,16 +90,14 @@ const createBitbucketCloudRepository = async (opts: { }; const createBitbucketServerRepository = async (opts: { - host: string; project: string; repo: string; description?: string; repoVisibility: 'private' | 'public'; authorization: string; - apiBaseUrl?: string; + apiBaseUrl: string; }) => { const { - host, project, repo, description, @@ -121,8 +121,7 @@ const createBitbucketServerRepository = async (opts: { }; try { - const baseUrl = apiBaseUrl ? apiBaseUrl : `https://${host}/rest/api/1.0`; - response = await fetch(`${baseUrl}/projects/${project}/repos`, options); + response = await fetch(`${apiBaseUrl}/projects/${project}/repos`, options); } catch (e) { throw new Error(`Unable to create repository, ${e}`); } @@ -306,7 +305,11 @@ export function createPublishBitbucketAction(options: { const authorization = getAuthorizationHeader( ctx.input.token - ? { host: integrationConfig.config.host, token: ctx.input.token } + ? { + host: integrationConfig.config.host, + apiBaseUrl: integrationConfig.config.apiBaseUrl, + token: ctx.input.token, + } : integrationConfig.config, ); @@ -319,7 +322,6 @@ export function createPublishBitbucketAction(options: { const { remoteUrl, repoContentsUrl } = await createMethod({ authorization, - host, workspace: workspace || '', project, repo, From e26fd1c7abb95a86dafd4b3957478d51c70bd2df Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 25 Feb 2022 15:41:53 +0100 Subject: [PATCH 007/150] catalog-react: mark useEntityPermission as alpha Signed-off-by: Johan Haals --- .changeset/polite-poems-nail.md | 5 +++++ plugins/catalog-react/api-report.md | 2 +- plugins/catalog-react/src/hooks/useEntityPermission.ts | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changeset/polite-poems-nail.md diff --git a/.changeset/polite-poems-nail.md b/.changeset/polite-poems-nail.md new file mode 100644 index 0000000000..c3eb97fce0 --- /dev/null +++ b/.changeset/polite-poems-nail.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': minor +--- + +Marked `useEntityPermission` as alpha since the underlying permission framework is under active development. diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index f117e8cd5c..dfd5f7d02b 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -564,7 +564,7 @@ export function useEntityOwnership(): { isOwnedEntity: (entity: Entity | EntityName) => boolean; }; -// @public +// @alpha export function useEntityPermission(permission: Permission): { loading: boolean; allowed: boolean; diff --git a/plugins/catalog-react/src/hooks/useEntityPermission.ts b/plugins/catalog-react/src/hooks/useEntityPermission.ts index c1f62cfc39..092114936a 100644 --- a/plugins/catalog-react/src/hooks/useEntityPermission.ts +++ b/plugins/catalog-react/src/hooks/useEntityPermission.ts @@ -28,7 +28,7 @@ import { useEntity } from './useEntity'; * Note: this hook blocks the permission request until the entity has loaded in * context. If you have the entityRef and need concurrent requests, use the * `usePermission` hook directly. - * @public + * @alpha */ export function useEntityPermission(permission: Permission): { loading: boolean; From 4ed5ce50610812779c33f233eab63e7b19f5c5c3 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 25 Feb 2022 15:54:28 +0100 Subject: [PATCH 008/150] publish alpha types Signed-off-by: Johan Haals --- plugins/catalog-react/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-react/package.json b/plugins/catalog-react/package.json index c7e255f973..b80d1f5789 100644 --- a/plugins/catalog-react/package.json +++ b/plugins/catalog-react/package.json @@ -9,7 +9,8 @@ "publishConfig": { "access": "public", "main": "dist/index.esm.js", - "types": "dist/index.d.ts" + "types": "dist/index.d.ts", + "alphaTypes": "dist/index.alpha.d.ts" }, "backstage": { "role": "web-library" From 4bc61a64e279b4438092124d3f733c563d29e841 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 25 Feb 2022 17:12:57 +0100 Subject: [PATCH 009/150] fix(msgraph): add missing docs for config options Adds documentation for the recently introduced config options - userGroupMemberSearch - groupSearch Closes: #9819 Signed-off-by: Patrick Jungermann --- .changeset/quick-mugs-pay.md | 5 +++++ plugins/catalog-backend-module-msgraph/README.md | 9 +++++++++ 2 files changed, 14 insertions(+) create mode 100644 .changeset/quick-mugs-pay.md diff --git a/.changeset/quick-mugs-pay.md b/.changeset/quick-mugs-pay.md new file mode 100644 index 0000000000..f2d3a00eb8 --- /dev/null +++ b/.changeset/quick-mugs-pay.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +add documentation for config options `userGroupMemberSearch` and `groupSearch` diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index 0e088d3a4e..9f879895d9 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -41,11 +41,20 @@ catalog: # This and userGroupMemberFilter are mutually exclusive, only one can be specified userFilter: accountEnabled eq true and userType eq 'member' # Optional filter for users, use group membership to get users. + # (Filtered groups and fetch their members.) # This and userFilter are mutually exclusive, only one can be specified + # See https://docs.microsoft.com/en-us/graph/search-query-parameter userGroupMemberFilter: "displayName eq 'Backstage Users'" + # Optional search for users, use group membership to get users. + # (Search for groups and fetch their members.) + # This and userFilter are mutually exclusive, only one can be specified + userGroupMemberSearch: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' # Optional filter for group, see Microsoft Graph API for the syntax # See https://docs.microsoft.com/en-us/graph/api/resources/group?view=graph-rest-1.0#properties groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') + # Optional search for groups, see Microsoft Graph API for the syntax + # See https://docs.microsoft.com/en-us/graph/search-query-parameter + groupSearch: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' ``` `userFilter` and `userGroupMemberFilter` are mutually exclusive, only one can be provided. If both are provided, an error will be thrown. From f9bb6aa0aa59730001913fcf38059086938138b7 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 25 Feb 2022 20:33:07 +0100 Subject: [PATCH 010/150] feat(msgraph): add `userExpand` config option Previously, the `userExpand` option was added to the config schema, but not read or passed. This change will fully implement `userExpand` and additionally. making it work with `userGroupMember[...]` options. Relates-to: issue #9819 Relates-to: PR #9824 Relates-to: PR #9721 Signed-off-by: Patrick Jungermann --- .changeset/tidy-jokes-dream.md | 5 +++ .../catalog-backend-module-msgraph/README.md | 6 ++++ .../api-report.md | 11 ++++--- .../src/microsoftGraph/client.test.ts | 2 +- .../src/microsoftGraph/client.ts | 12 ++++--- .../src/microsoftGraph/config.test.ts | 2 ++ .../src/microsoftGraph/config.ts | 5 ++- .../src/microsoftGraph/read.test.ts | 32 +++++++++++++++---- .../src/microsoftGraph/read.ts | 9 ++++-- 9 files changed, 65 insertions(+), 19 deletions(-) create mode 100644 .changeset/tidy-jokes-dream.md diff --git a/.changeset/tidy-jokes-dream.md b/.changeset/tidy-jokes-dream.md new file mode 100644 index 0000000000..aa9c84fc4b --- /dev/null +++ b/.changeset/tidy-jokes-dream.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +add `userExpand` config option to allow expanding a single relationship diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index 0e088d3a4e..5608231405 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -35,6 +35,12 @@ catalog: # the App registration in the Microsoft Azure Portal. clientId: ${MICROSOFT_GRAPH_CLIENT_ID} clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN} + # Optional parameter to include the expanded resource or collection referenced + # by a single relationship (navigation property) in your results. + # Only one relationship can be expanded in a single request. + # See https://docs.microsoft.com/en-us/graph/query-parameters#expand-parameter + # Can be combined with userGroupMember[...] instead of userFilter. + userExpand: manager # Optional filter for user, see Microsoft Graph API for the syntax # See https://docs.microsoft.com/en-us/graph/api/resources/user?view=graph-rest-1.0#properties # and for the syntax https://docs.microsoft.com/en-us/graph/query-parameters#filter-parameter diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index 7ad603e6ca..574b948dea 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -78,7 +78,10 @@ export class MicrosoftGraphClient { userId: string, maxSize: number, ): Promise; - getUserProfile(userId: string): Promise; + getUserProfile( + userId: string, + query?: ODataQuery, + ): Promise; getUsers(query?: ODataQuery): AsyncIterable; requestApi( path: string, @@ -158,7 +161,7 @@ export type MicrosoftGraphProviderConfig = { clientId: string; clientSecret: string; userFilter?: string; - userExpand?: string[]; + userExpand?: string; userGroupMemberFilter?: string; userGroupMemberSearch?: string; groupFilter?: string; @@ -172,7 +175,7 @@ export function normalizeEntityName(name: string): string; export type ODataQuery = { search?: string; filter?: string; - expand?: string[]; + expand?: string; select?: string[]; }; @@ -191,7 +194,7 @@ export function readMicrosoftGraphOrg( client: MicrosoftGraphClient, tenantId: string, options: { - userExpand?: string[]; + userExpand?: string; userFilter?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts index 6413a97c15..94a126e1a7 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts @@ -85,7 +85,7 @@ describe('MicrosoftGraphClient', () => { const response = await client.requestApi('users', { filter: 'test eq true', - expand: ['children'], + expand: 'children', select: ['id', 'children'], }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts index db749a536e..cba5d9f640 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts @@ -38,7 +38,7 @@ export type ODataQuery = { /** * specifies the related resources or media streams to be included in line with retrieved resources */ - expand?: string[]; + expand?: string; /** * request a specific set of properties for each entity or complex type */ @@ -155,7 +155,7 @@ export class MicrosoftGraphClient { $search: query?.search, $filter: query?.filter, $select: query?.select?.join(','), - $expand: query?.expand?.join(','), + $expand: query?.expand, }, { addQueryPrefix: true, @@ -203,10 +203,14 @@ export class MicrosoftGraphClient { * * @public * @param userId - The unique identifier for the `User` resource + * @param query - OData Query {@link ODataQuery} * */ - async getUserProfile(userId: string): Promise { - const response = await this.requestApi(`users/${userId}`); + async getUserProfile( + userId: string, + query?: ODataQuery, + ): Promise { + const response = await this.requestApi(`users/${userId}`, query); if (response.status !== 200) { await this.handleError('user profile', response); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts index efbbca1f5c..700e5cd8fe 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts @@ -53,6 +53,7 @@ describe('readMicrosoftGraphConfig', () => { clientId: 'clientId', clientSecret: 'clientSecret', authority: 'https://login.example.com/', + userExpand: 'manager', userFilter: 'accountEnabled eq true', groupFilter: 'securityEnabled eq false', }, @@ -66,6 +67,7 @@ describe('readMicrosoftGraphConfig', () => { clientId: 'clientId', clientSecret: 'clientSecret', authority: 'https://login.example.com', + userExpand: 'manager', userFilter: 'accountEnabled eq true', groupFilter: 'securityEnabled eq false', }, diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index 6809144f80..c2789d767a 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -57,7 +57,7 @@ export type MicrosoftGraphProviderConfig = { * * E.g. "manager" */ - userExpand?: string[]; + userExpand?: string; /** * The filter to apply to extract users by groups memberships. * @@ -106,6 +106,8 @@ export function readMicrosoftGraphConfig( const tenantId = providerConfig.getString('tenantId'); const clientId = providerConfig.getString('clientId'); const clientSecret = providerConfig.getString('clientSecret'); + + const userExpand = providerConfig.getOptionalString('userExpand'); const userFilter = providerConfig.getOptionalString('userFilter'); const userGroupMemberFilter = providerConfig.getOptionalString( 'userGroupMemberFilter', @@ -133,6 +135,7 @@ export function readMicrosoftGraphConfig( tenantId, clientId, clientSecret, + userExpand, userFilter, userGroupMemberFilter, userGroupMemberSearch, diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts index b0273e1f58..dba3741c73 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts @@ -118,7 +118,7 @@ describe('read microsoft graph', () => { expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); - it('should read users with custom transformer', async () => { + it('should read users with userExpand and custom transformer', async () => { async function* getExampleUsers() { yield { id: 'userid', @@ -133,6 +133,7 @@ describe('read microsoft graph', () => { ); const { users } = await readMicrosoftGraphUsers(client, { + userExpand: 'manager', userFilter: 'accountEnabled eq true', transformer: async () => ({ apiVersion: 'backstage.io/v1alpha1', @@ -154,6 +155,7 @@ describe('read microsoft graph', () => { expect(client.getUsers).toBeCalledTimes(1); expect(client.getUsers).toBeCalledWith({ + expand: 'manager', filter: 'accountEnabled eq true', }); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); @@ -227,12 +229,14 @@ describe('read microsoft graph', () => { expect(client.getGroupMembers).toBeCalledWith('groupid'); expect(client.getUserProfile).toBeCalledTimes(1); - expect(client.getUserProfile).toBeCalledWith('userid'); + expect(client.getUserProfile).toBeCalledWith('userid', { + expand: undefined, + }); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); - it('should read users with custom transformer', async () => { + it('should read users with userExpand and custom transformer', async () => { async function* getExampleGroups() { yield { id: 'groupid', @@ -266,6 +270,7 @@ describe('read microsoft graph', () => { ); const { users } = await readMicrosoftGraphUsersInGroups(client, { + userExpand: 'manager', userGroupMemberFilter: 'securityEnabled eq true', transformer: async () => ({ apiVersion: 'backstage.io/v1alpha1', @@ -293,7 +298,9 @@ describe('read microsoft graph', () => { expect(client.getGroupMembers).toBeCalledWith('groupid'); expect(client.getUserProfile).toBeCalledTimes(1); - expect(client.getUserProfile).toBeCalledWith('userid'); + expect(client.getUserProfile).toBeCalledWith('userid', { + expand: 'manager', + }); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); @@ -634,6 +641,14 @@ describe('read microsoft graph', () => { }; } + async function getExampleUserProfile(userId: string) { + return { + id: userId, + displayName: 'User Name', + mail: 'user.name@example.com', + }; + } + async function* getExampleGroups() { yield { id: 'groupid', @@ -686,7 +701,7 @@ describe('read microsoft graph', () => { }); }); - it('should read users using userFilter', async () => { + it('should read users using userExpand and userFilter', async () => { client.getOrganization.mockResolvedValue({ id: 'tenantid', displayName: 'Organization Name', @@ -705,12 +720,14 @@ describe('read microsoft graph', () => { await readMicrosoftGraphOrg(client, 'tenantid', { logger: getVoidLogger(), + userExpand: 'manager', userFilter: 'accountEnabled eq true', groupFilter: 'securityEnabled eq false', }); expect(client.getUsers).toBeCalledTimes(1); expect(client.getUsers).toBeCalledWith({ + expand: 'manager', filter: 'accountEnabled eq true', }); expect(client.getGroups).toBeCalledTimes(1); @@ -719,13 +736,14 @@ describe('read microsoft graph', () => { }); }); - it('should read users using userGroupMemberFilter', async () => { + it('should read users using userExpand and userGroupMemberFilter', async () => { client.getOrganization.mockResolvedValue({ id: 'tenantid', displayName: 'Organization Name', }); client.getUsers.mockImplementation(getExampleUsers); + client.getUserProfile.mockImplementation(getExampleUserProfile); client.getUserPhotoWithSizeLimit.mockResolvedValue( 'data:image/jpeg;base64,...', ); @@ -750,6 +768,8 @@ describe('read microsoft graph', () => { expect(client.getGroups).toBeCalledWith({ filter: 'securityEnabled eq false', }); + expect(client.getUserProfile).toBeCalledTimes(1); + expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); }); }); }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index f862fd5ace..333eaa6b0a 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -85,7 +85,7 @@ export async function readMicrosoftGraphUsers( client: MicrosoftGraphClient, options: { userFilter?: string; - userExpand?: string[]; + userExpand?: string; transformer?: UserTransformer; logger: Logger; }, @@ -137,6 +137,7 @@ export async function readMicrosoftGraphUsers( export async function readMicrosoftGraphUsersInGroups( client: MicrosoftGraphClient, options: { + userExpand?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; transformer?: UserTransformer; @@ -186,7 +187,9 @@ export async function readMicrosoftGraphUsersInGroups( let user; let userPhoto; try { - user = await client.getUserProfile(userId); + user = await client.getUserProfile(userId, { + expand: options.userExpand, + }); } catch (e) { options.logger.warn(`Unable to load user for ${userId}`); } @@ -506,7 +509,7 @@ export async function readMicrosoftGraphOrg( client: MicrosoftGraphClient, tenantId: string, options: { - userExpand?: string[]; + userExpand?: string; userFilter?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; From 766f969de973a8dbba12e62a964234c0ff596c07 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 26 Feb 2022 15:40:34 +0100 Subject: [PATCH 011/150] scripts/prepare-release: update to detect patch versions from patch branches Signed-off-by: Patrik Oldsberg --- docs/publishing.md | 9 -- scripts/prepare-release.js | 174 +++++++++++++++++++++++++++++-------- 2 files changed, 137 insertions(+), 46 deletions(-) diff --git a/docs/publishing.md b/docs/publishing.md index fbbef93493..a0d67b7d24 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -67,12 +67,3 @@ process is used to release an emergency fix as version `6.5.1` in the patch rele - [ ] The fix, which you can likely cherry-pick from your patch branch: `git cherry-pick origin/patch/v1.18.0^` - [ ] An updated `CHANGELOG.md` of all patched packages from the tip of the patch branch, `git checkout origin/patch/v1.18.0 -- {packages,plugins}/*/CHANGELOG.md`. - [ ] A changeset with the message "Applied the fix from version `6.5.1` of this package, which is part of the `v1.18.1` release of Backstage." - - [ ] An entry in `.changeset/patched.json` that sets the current release version to `6.5.1`: - - ```json - { - "currentReleaseVersion": { - "@backstage/plugin-foo": "6.5.1" - } - } - ``` diff --git a/scripts/prepare-release.js b/scripts/prepare-release.js index be82e19344..5cc888fd66 100755 --- a/scripts/prepare-release.js +++ b/scripts/prepare-release.js @@ -28,6 +28,10 @@ const execFile = promisify(execFileCb); // All of these are considered to be main-line release branches const MAIN_BRANCHES = ['master', 'origin/master', 'changeset-release/master']; +// This prefix is used for patch branches, followed by the release version WITH a 'v' prefix +// For example, `patch/v1.2.0` +const PATCH_BRANCH_PREFIX = 'patch/'; + const DEPENDENCY_TYPES = [ 'dependencies', 'devDependencies', @@ -35,25 +39,84 @@ const DEPENDENCY_TYPES = [ 'peerDependencies', ]; +/** + * Returns the most recent release version on the main branch that is not a pre-release. + */ +async function getPreviousReleaseVersion(repo) { + // TODO(Rugvip): Figure out which field to sort by to avoid manual sort after + const { stdout: tagsStr } = await execFile( + 'git', + ['tag', '--list', 'v*', '--merged=HEAD'], + { shell: true, cwd: repo.root.dir }, + ); + const tags = tagsStr.trim().split(/\r\n|\n/); + const [latestTag] = semver.rsort(tags).filter(t => !semver.prerelease(t)); + return latestTag; +} + +/** + * Finds the tip of the patch branch of a given release version. + * Returns undefined if no patch branch exists. + */ +async function findTipOfPatchBranch(repo, release) { + try { + await execFile('git', ['fetch', 'origin', PATCH_BRANCH_PREFIX + release], { + shell: true, + cwd: repo.root.dir, + }); + } catch (error) { + if (error.stderr?.match(/fatal: couldn't find remote ref/i)) { + return undefined; + } + throw error; + } + const { stdout: refStr } = await execFile('git', ['rev-parse', 'FETCH_HEAD']); + return refStr.trim(); +} + +/** + * Returns a map of packages to their versions for any package version + * in that does not match the current version in the working directory. + */ +async function detectPatchVersionsForRef(repo, ref) { + const patchVersions = new Map(); + + for (const pkg of repo.packages) { + const pkgJsonPath = path.join( + path.relative(repo.root.dir, pkg.dir), + 'package.json', + ); + const { stdout: pkgJsonStr } = await execFile('git', [ + 'show', + `${ref}:${pkgJsonPath}`, + ]); + if (pkgJsonStr) { + const releasePkgJson = JSON.parse(pkgJsonStr); + const pkgJson = pkg.packageJson; + if (releasePkgJson.name !== pkgJson.name) { + throw new Error( + `Mismatched package name at ${pkg.dir}, ${releasePkgJson.name} !== ${pkgJson.name}`, + ); + } + if (releasePkgJson.version !== pkgJson.version) { + patchVersions.set(pkgJson.name, releasePkgJson.version); + } + } + } + + return patchVersions; +} + /** * Bumps up the versions of packages to account for * the base versions that are set in .changeset/patched.json. * This may be needed when we have made emergency releases. */ -async function updatePatchVersions() { - const patchedJsonPath = path.resolve('.changeset', 'patched.json'); - const { currentReleaseVersion } = await fs.readJson(patchedJsonPath); - if (Object.keys(currentReleaseVersion).length === 0) { - console.log('No currentReleaseVersion overrides found, skipping.'); - return; - } - - const { packages } = await getPackages(path.resolve('.')); - +async function applyPatchVersions(repo, patchVersions) { const pendingVersionBumps = new Map(); - for (const [name, version] of Object.entries(currentReleaseVersion)) { - const pkg = packages.find(p => p.packageJson.name === name); + for (const [name, version] of patchVersions) { + const pkg = repo.packages.find(p => p.packageJson.name === name); if (!pkg) { throw new Error(`Package ${name} not found`); } @@ -81,7 +144,7 @@ async function updatePatchVersions() { }); } - for (const { dir, packageJson } of packages) { + for (const { dir, packageJson } of [repo.root, ...repo.packages]) { let hasChanges = false; if (pendingVersionBumps.has(packageJson.name)) { @@ -117,20 +180,44 @@ async function updatePatchVersions() { }); } } +} - await fs.writeJSON( - patchedJsonPath, - { currentReleaseVersion: {} }, - { spaces: 2, encoding: 'utf8' }, - ); +/** + * Detects any patched packages version since the most recent release on + * the main branch, and then bumps all packages in the repo accordingly. + */ +async function updatePackageVersions(repo) { + const previousRelease = await getPreviousReleaseVersion(repo); + console.log(`Found release version: ${previousRelease}`); + + const patchRef = await findTipOfPatchBranch(repo, previousRelease); + if (patchRef) { + console.log(`Tip of the patch branch: ${patchRef}`); + + const patchVersions = await detectPatchVersionsForRef(repo, patchRef); + if (patchVersions.size > 0) { + console.log( + `Found ${patchVersions.size} packages that were patched since the last release`, + ); + for (const [name, version] of patchVersions) { + console.log(` ${name}: ${version}`); + } + + await applyPatchVersions(repo, patchVersions); + } else { + console.log('No packages were patched since the last release'); + } + } else { + console.log('No patch branch found'); + } } /** * Returns the mode and tag that is currently set * in the .changeset/pre.json file */ -async function getPreInfo(rootPath) { - const pre = path.join(rootPath, '.changeset', 'pre.json'); +async function getPreInfo(repo) { + const pre = path.join(repo.root.dir, '.changeset', 'pre.json'); if (!(await fs.pathExists(pre))) { return { mode: undefined, tag: undefined }; } @@ -139,26 +226,30 @@ async function getPreInfo(rootPath) { return { mode, tag }; } +/** + * Returns the name of the current git branch + */ +async function getCurrentBranch(repo) { + const { stdout } = await execFile( + 'git', + ['rev-parse', '--abbrev-ref', 'HEAD'], + { cwd: repo.root.dir, shell: true }, + ); + return stdout.trim(); +} + /** * Bumps the release version in the root package.json. * * This takes into account whether we're in pre-release mode or on a patch branch. */ -async function updateBackstageReleaseVersion() { - const rootPath = path.resolve(__dirname, '..'); - const branchName = await execFile( - 'git', - ['rev-parse', '--abbrev-ref', 'HEAD'], - { shell: true }, - ).then(({ stdout }) => stdout.trim()); - const { mode: preMode, tag: preTag } = await getPreInfo(rootPath); +async function updateBackstageReleaseVersion(repo, type) { + const { mode: preMode, tag: preTag } = await getPreInfo(repo); - const packagePath = path.join(rootPath, 'package.json'); - const package = await fs.readJson(packagePath); - const { version: currentVersion } = package; + const { version: currentVersion } = repo.root.packageJson; let nextVersion; - if (MAIN_BRANCHES.includes(branchName)) { + if (type === 'minor') { if (preMode === 'pre') { if (semver.prerelease(currentVersion)) { nextVersion = semver.inc(currentVersion, 'pre', preTag); @@ -170,7 +261,7 @@ async function updateBackstageReleaseVersion() { } else { nextVersion = semver.inc(currentVersion, 'minor'); } - } else { + } else if (type === 'patch') { if (preMode) { throw new Error(`Unexpected pre mode ${preMode} on branch ${branchName}`); } @@ -178,9 +269,9 @@ async function updateBackstageReleaseVersion() { } await fs.writeJson( - packagePath, + path.join(repo.root.dir, 'package.json'), { - ...package, + ...repo.root.packageJson, version: nextVersion, }, { spaces: 2, encoding: 'utf8' }, @@ -188,8 +279,17 @@ async function updateBackstageReleaseVersion() { } async function main() { - await updatePatchVersions(); - await updateBackstageReleaseVersion(); + const repo = await getPackages(__dirname); + const branchName = await getCurrentBranch(repo); + const isMainBranch = MAIN_BRANCHES.includes(branchName); + + console.log(`Current branch: ${branchName}`); + if (isMainBranch) { + console.log('Main release, updating package versions'); + await updatePackageVersions(repo); + } + + await updateBackstageReleaseVersion(repo, isMainBranch ? 'minor' : 'patch'); } main().catch(error => { From 6537a601c7de369c8aa9a3601b8fe780db46aa2b Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Sun, 27 Feb 2022 18:36:35 +1100 Subject: [PATCH 012/150] Add changeset for these changes Signed-off-by: Nikolas Skoufis --- .changeset/dull-months-knock.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/dull-months-knock.md diff --git a/.changeset/dull-months-knock.md b/.changeset/dull-months-knock.md new file mode 100644 index 0000000000..3b957518c2 --- /dev/null +++ b/.changeset/dull-months-knock.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs-backend': patch +--- + +Added a new interface that allows for customization of when to build techdocs From f7e6a2fbc7cd1a7c24016b205a77117e77000b42 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Sun, 27 Feb 2022 18:49:59 +1100 Subject: [PATCH 013/150] Add updated api report Signed-off-by: Nikolas Skoufis --- plugins/techdocs-backend/api-report.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index 5d106cfafb..28700b245b 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -6,6 +6,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; import { DocumentCollator } from '@backstage/search-common'; +import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { GeneratorBuilder } from '@backstage/techdocs-common'; import { Knex } from 'knex'; @@ -51,6 +52,7 @@ export type OutOfTheBoxDeploymentOptions = { database?: Knex; config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; // @public @@ -60,6 +62,7 @@ export type RecommendedDeploymentOptions = { discovery: PluginEndpointDiscovery; config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; // @public @@ -81,4 +84,8 @@ export type TechDocsCollatorOptions = { export { TechDocsDocument }; export * from '@backstage/techdocs-common'; + +// Warnings were encountered during analysis: +// +// src/service/router.d.ts:24:5 - (ae-forgotten-export) The symbol "DocsBuildStrategy" needs to be exported by the entry point index.d.ts ``` From 0bedff8c9abecdc13730fe54b816e9cc6674306a Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Sun, 27 Feb 2022 19:11:09 +1100 Subject: [PATCH 014/150] Add docs for build strategy stuff Signed-off-by: Nikolas Skoufis --- docs/features/techdocs/concepts.md | 18 +++++++++ docs/features/techdocs/configuration.md | 15 +++++--- docs/features/techdocs/how-to-guides.md | 50 +++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/docs/features/techdocs/concepts.md b/docs/features/techdocs/concepts.md index 408f092ebd..f8e813d89e 100644 --- a/docs/features/techdocs/concepts.md +++ b/docs/features/techdocs/concepts.md @@ -46,6 +46,24 @@ between `techdocs-backend` and the storage) [TechDocs Backend](https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend) +## TechDocs Build Strategy + +To accommodate more complex logic surrounding whether or not to build TechDocs, the TechDocs backend +supports selecting a Build Strategy. +The Build Strategy is responsible for deciding whether the documentation requested should be built locally +by the TechDocs backend or not. +Customization of the Build Strategy allows for more complex behaviour regarding whether the TechDocs backend +is responsible for building TechDocs, whether an external process is responsible, or whether a combination +of local builds and an external process is responsible, on an entity-by-entity basis. + +The default Build Strategy results in the TechDocs backend building documentation locally if the +`techdocs.builder` configuration option is set to `'local'`, and skipping any building otherwise. +However any logic that satisfies the Build Strategy interface can be implemented, using the Backstage +config as well as the entity being processed to make a decision. + +For an example of how the Build Strategy can be used to implement a 'hybrid' build model, refer to +the [How to implement a hybrid build strategy](./how-to-guides#how-to-implement-a-hybrid-build-strategy) guide. + ## TechDocs Container The TechDocs container is a Docker container available at diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 6317ae365b..9509a9a545 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -38,11 +38,16 @@ techdocs: pullImage: true # techdocs.builder can be either 'local' or 'external. - # If builder is set to 'local' and you open a TechDocs page, techdocs-backend will try to generate the docs, publish to storage - # and show the generated docs afterwords. This is the "Basic" setup of the TechDocs Architecture. - # If builder is set to 'external', techdocs-backend will only fetch the docs and will NOT try to generate and publish. In this case of 'external', - # we assume that docs are being built by an external process (e.g. in the CI/CD pipeline of the repository). This is the "Recommended" setup of - # the architecture. Read more here https://backstage.io/docs/features/techdocs/architecture + # Using the default build strategy, if builder is set to 'local' and you open a TechDocs page, + # techdocs-backend will try to generate the docs, publish to storage and show the generated docs afterwords. + # This is the "Basic" setup of the TechDocs Architecture. + # Using the default build strategy, if builder is set to 'external' (or anything other than 'local'), techdocs-backend + # will only fetch the docs and will NOT try to generate and publish. + # In this case, we assume that docs are being built by an external process (e.g. in the CI/CD pipeline of the repository). + # This is the "Recommended" setup of the architecture. + # Note that custom build strategies may alter this behaviour. + # Read more about the "Basic" and "Recommended" setups here https://backstage.io/docs/features/techdocs/architecture + # Read more about build strategies here: https://backstage.io/docs/features/techdocs/concepts#techdocs-build-strategy builder: 'local' diff --git a/docs/features/techdocs/how-to-guides.md b/docs/features/techdocs/how-to-guides.md index 863b4b3ecb..5ec8aac307 100644 --- a/docs/features/techdocs/how-to-guides.md +++ b/docs/features/techdocs/how-to-guides.md @@ -538,3 +538,53 @@ Done! Now you have a support of the following diagrams along with mermaid: - `Vega` - `Vega-Lite` - `WaveDrom` + +## How to implement a hybrid build strategy + +One limitation of the [Recommended deployment](./architecture#recommended-deployment) is that +the experience for users requires modifying their CI/CD process to publish +their TechDocs. For some users, this may be unnecessary, and provides a barrier +to entry for onboarding users to Backstage. However, a purely local TechDocs +build restricts TechDocs creators to using the tooling provided in Backstage, +as well as the plugins and features provided in the Backstage-included `mkdocs` +installation. + +To accommodate both of these use-cases, users can implement a custom [Build Strategy](./concepts#techdocs-build-strategy) +with logic to encode which TechDocs should be built locally, and which will be +built externally. + +To achieve this hybrid build model: + +1. In your Backstage instance's `app-config.yaml`, set `techdocs.builder` to + `'local'`. This ensures that Backstage will build docs for users who want the + 'out-of-the-box' experience. +2. Configure external storage of TechDocs as normal for a production deployment. + This allows Backstage to publish documentation to your storage, as well as + allowing other users to publish documentation from their CI/CD pipelines. +3. Create a custom build strategy, that implements the `DocsBuildStrategy` interface, + and which implements your custom logic for determining whether to build docs for + a given entity. + For example, to only build docs when an entity has the `company.com/techdocs-builder` + annotation set to `'local'`: + ```typescript + export class AnnotationBasedBuildStrategy { + private readonly config: Config; + + constructor(config: Config) { + this.config = config; + } + + async shouldBuild(_: Entity): Promise { + return this.entity.metadata?.annotations?.["company.com/techdocs-builder"] === 'local' + } + } + ``` +4. Pass an instance of this Build Strategy as the `docsBuildStrategy` parameter of the + TechDocs backend `createRouter` method. + +Users should now be able to choose to have their documentation built and published by +the TechDocs backend by adding the `company.com/techdocs-builder` annotation to their +entity. If the value of this annotation is `'local'`, the TechDocs backend will build +and publish the documentation for them. If the value of the `company.com/techdocs-builder` +annotation is anything other than `'local'`, the user is responsible for publishing +documentation to the appropriate location in the TechDocs external storage. From 9731e500688c598dfb6c19b7d752775202cadd2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Feb 2022 04:11:23 +0000 Subject: [PATCH 015/150] chore(deps): bump eslint from 8.7.0 to 8.10.0 Bumps [eslint](https://github.com/eslint/eslint) from 8.7.0 to 8.10.0. - [Release notes](https://github.com/eslint/eslint/releases) - [Changelog](https://github.com/eslint/eslint/blob/main/CHANGELOG.md) - [Commits](https://github.com/eslint/eslint/compare/v8.7.0...v8.10.0) --- updated-dependencies: - dependency-name: eslint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- yarn.lock | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/yarn.lock b/yarn.lock index 997e55c9e9..2e89e7b0d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1808,14 +1808,14 @@ ts-node "^9" tslib "^2" -"@eslint/eslintrc@^1.0.5": - version "1.0.5" - resolved "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.0.5.tgz#33f1b838dbf1f923bfa517e008362b78ddbbf318" - integrity sha512-BLxsnmK3KyPunz5wmCCpqy0YelEoxxGmH73Is+Z74oOTMtExcjkr3dDR6quwrjh1YspA8DH9gnX1o069KiS9AQ== +"@eslint/eslintrc@^1.2.0": + version "1.2.0" + resolved "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.2.0.tgz#7ce1547a5c46dfe56e1e45c3c9ed18038c721c6a" + integrity sha512-igm9SjJHNEJRiUnecP/1R5T3wKLEJ7pL6e2P+GUSfCd0dGjPYYZve08uzw8L2J8foVHFz+NGu12JxRcU2gGo6w== dependencies: ajv "^6.12.4" debug "^4.3.2" - espree "^9.2.0" + espree "^9.3.1" globals "^13.9.0" ignore "^4.0.6" import-fresh "^3.2.1" @@ -11688,10 +11688,10 @@ eslint-scope@5.1.1, eslint-scope@^5.1.1: esrecurse "^4.3.0" estraverse "^4.1.1" -eslint-scope@^7.1.0: - version "7.1.0" - resolved "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.0.tgz#c1f6ea30ac583031f203d65c73e723b01298f153" - integrity sha512-aWwkhnS0qAXqNOgKOK0dJ2nvzEbhEvpy8OlJ9kZ0FeZnA6zpjv1/Vei+puGFFX7zkPCkHHXb7IDX3A+7yPrRWg== +eslint-scope@^7.1.1: + version "7.1.1" + resolved "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz#fff34894c2f65e5226d3041ac480b4513a163642" + integrity sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw== dependencies: esrecurse "^4.3.0" estraverse "^5.2.0" @@ -11708,10 +11708,10 @@ eslint-visitor-keys@^2.0.0: resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz#21fdc8fbcd9c795cc0321f0563702095751511a8" integrity sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ== -eslint-visitor-keys@^3.0.0, eslint-visitor-keys@^3.1.0, eslint-visitor-keys@^3.2.0: - version "3.2.0" - resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.2.0.tgz#6fbb166a6798ee5991358bc2daa1ba76cc1254a1" - integrity sha512-IOzT0X126zn7ALX0dwFiUQEdsfzrm4+ISsQS8nukaJXwEyYKRSnEIIDULYg1mCtGp7UUXgfGl7BIolXREQK+XQ== +eslint-visitor-keys@^3.0.0, eslint-visitor-keys@^3.3.0: + version "3.3.0" + resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" + integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== eslint-webpack-plugin@^2.6.0: version "2.6.0" @@ -11726,11 +11726,11 @@ eslint-webpack-plugin@^2.6.0: schema-utils "^3.1.1" eslint@^8.6.0: - version "8.7.0" - resolved "https://registry.npmjs.org/eslint/-/eslint-8.7.0.tgz#22e036842ee5b7cf87b03fe237731675b4d3633c" - integrity sha512-ifHYzkBGrzS2iDU7KjhCAVMGCvF6M3Xfs8X8b37cgrUlDt6bWRTpRh6T/gtSXv1HJ/BUGgmjvNvOEGu85Iif7w== + version "8.10.0" + resolved "https://registry.npmjs.org/eslint/-/eslint-8.10.0.tgz#931be395eb60f900c01658b278e05b6dae47199d" + integrity sha512-tcI1D9lfVec+R4LE1mNDnzoJ/f71Kl/9Cv4nG47jOueCMBrCCKYXr4AUVS7go6mWYGFD4+EoN6+eXSrEbRzXVw== dependencies: - "@eslint/eslintrc" "^1.0.5" + "@eslint/eslintrc" "^1.2.0" "@humanwhocodes/config-array" "^0.9.2" ajv "^6.10.0" chalk "^4.0.0" @@ -11738,10 +11738,10 @@ eslint@^8.6.0: debug "^4.3.2" doctrine "^3.0.0" escape-string-regexp "^4.0.0" - eslint-scope "^7.1.0" + eslint-scope "^7.1.1" eslint-utils "^3.0.0" - eslint-visitor-keys "^3.2.0" - espree "^9.3.0" + eslint-visitor-keys "^3.3.0" + espree "^9.3.1" esquery "^1.4.0" esutils "^2.0.2" fast-deep-equal "^3.1.3" @@ -11771,14 +11771,14 @@ esm@^3.2.25: resolved "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz#342c18c29d56157688ba5ce31f8431fbb795cc10" integrity sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA== -espree@^9.2.0, espree@^9.3.0: - version "9.3.0" - resolved "https://registry.npmjs.org/espree/-/espree-9.3.0.tgz#c1240d79183b72aaee6ccfa5a90bc9111df085a8" - integrity sha512-d/5nCsb0JcqsSEeQzFZ8DH1RmxPcglRWh24EFTlUEmCKoehXGdpsx0RkHDubqUI8LSAIKMQp4r9SzQ3n+sm4HQ== +espree@^9.3.1: + version "9.3.1" + resolved "https://registry.npmjs.org/espree/-/espree-9.3.1.tgz#8793b4bc27ea4c778c19908e0719e7b8f4115bcd" + integrity sha512-bvdyLmJMfwkV3NCRl5ZhJf22zBFo1y8bYh3VYb+bfzqNB4Je68P2sSuXyuFquzWLebHpNd2/d5uv7yoP9ISnGQ== dependencies: acorn "^8.7.0" acorn-jsx "^5.3.1" - eslint-visitor-keys "^3.1.0" + eslint-visitor-keys "^3.3.0" esprima@^4.0.0, esprima@^4.0.1, esprima@~4.0.0: version "4.0.1" From 56e401cce099e7616377f50f1fbb691c4eca058d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Feb 2022 04:11:49 +0000 Subject: [PATCH 016/150] chore(deps): bump @rollup/plugin-commonjs from 21.0.1 to 21.0.2 Bumps [@rollup/plugin-commonjs](https://github.com/rollup/plugins/tree/HEAD/packages/commonjs) from 21.0.1 to 21.0.2. - [Release notes](https://github.com/rollup/plugins/releases) - [Changelog](https://github.com/rollup/plugins/blob/master/packages/commonjs/CHANGELOG.md) - [Commits](https://github.com/rollup/plugins/commits/commonjs-v21.0.2/packages/commonjs) --- updated-dependencies: - dependency-name: "@rollup/plugin-commonjs" dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 997e55c9e9..16a11c090d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4848,9 +4848,9 @@ react-use "^17.2.4" "@rollup/plugin-commonjs@^21.0.1": - version "21.0.1" - resolved "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-21.0.1.tgz#1e57c81ae1518e4df0954d681c642e7d94588fee" - integrity sha512-EA+g22lbNJ8p5kuZJUYyhhDK7WgJckW5g4pNN7n4mAFUM96VuwUnNT3xr2Db2iCZPI1pJPbGyfT5mS9T1dHfMg== + version "21.0.2" + resolved "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-21.0.2.tgz#0b9c539aa1837c94abfaf87945838b0fc8564891" + integrity sha512-d/OmjaLVO4j/aQX69bwpWPpbvI3TJkQuxoAk7BH8ew1PyoMBLTOuvJTjzG8oEoW7drIIqB0KCJtfFLu/2GClWg== dependencies: "@rollup/pluginutils" "^3.1.0" commondir "^1.0.1" From f88f3cd80857225c32007172221fe39915404476 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Feb 2022 04:12:44 +0000 Subject: [PATCH 017/150] chore(deps): bump @microsoft/microsoft-graph-types from 2.13.0 to 2.15.0 Bumps [@microsoft/microsoft-graph-types](https://github.com/microsoftgraph/msgraph-typescript-typings) from 2.13.0 to 2.15.0. - [Release notes](https://github.com/microsoftgraph/msgraph-typescript-typings/releases) - [Commits](https://github.com/microsoftgraph/msgraph-typescript-typings/compare/2.13.0...2.15.0) --- updated-dependencies: - dependency-name: "@microsoft/microsoft-graph-types" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 997e55c9e9..2e43b60b36 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4050,9 +4050,9 @@ integrity sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA== "@microsoft/microsoft-graph-types@^2.6.0": - version "2.13.0" - resolved "https://registry.npmjs.org/@microsoft/microsoft-graph-types/-/microsoft-graph-types-2.13.0.tgz#aa584e4897665df5a9c8869a226264cd6ec5882b" - integrity sha512-63FfWBLcyNo8tMP4oPcdqHQvk4ehuWpiUMjVLD7zJXPENIowpdwudP969AALkKzlwsjWImamdivGKd2Zc8Z1Uw== + version "2.15.0" + resolved "https://registry.npmjs.org/@microsoft/microsoft-graph-types/-/microsoft-graph-types-2.15.0.tgz#1705ea1ce84c3de4705957392d7f0e3ae465c9f8" + integrity sha512-EyuOpZs55HUoC37Ujrp6IRgE5ghf/wtDrlWuJm7J/DKoB7B/Iek7eXdavTygx2uBeDZ5b4jXXvwl4PiDLlEcsw== "@microsoft/tsdoc-config@~0.15.2": version "0.15.2" From 2d858b5c2a2dc4f6e3e74ebb6e99f34ab40937e8 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 28 Feb 2022 10:48:09 +0100 Subject: [PATCH 018/150] use experimental type build, include alpha types Signed-off-by: Johan Haals --- plugins/catalog-react/package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-react/package.json b/plugins/catalog-react/package.json index b80d1f5789..119c3f19a4 100644 --- a/plugins/catalog-react/package.json +++ b/plugins/catalog-react/package.json @@ -25,7 +25,7 @@ "backstage" ], "scripts": { - "build": "backstage-cli package build", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", @@ -77,6 +77,7 @@ "react-test-renderer": "^16.13.1" }, "files": [ - "dist" + "dist", + "alpha" ] } From d88b32361fc54250694706fd45ab941fe00db43e Mon Sep 17 00:00:00 2001 From: Karan Shah Date: Mon, 28 Feb 2022 11:21:30 +0000 Subject: [PATCH 019/150] Update the Airbrake installation instructions Signed-off-by: Karan Shah --- plugins/airbrake/README.md | 64 ++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/plugins/airbrake/README.md b/plugins/airbrake/README.md index d5ea9bb006..28c230fb05 100644 --- a/plugins/airbrake/README.md +++ b/plugins/airbrake/README.md @@ -20,7 +20,7 @@ The Airbrake plugin provides connectivity between Backstage and Airbrake (https: yarn add @backstage/plugin-airbrake-backend ``` -3. Add the `EntityAirbrakeContent` to `packages/app/src/components/catalog/EntityPage.tsx`: +3. Add the `EntityAirbrakeContent` to `packages/app/src/components/catalog/EntityPage.tsx` for all entity pages you want Airbrake to be in: ```typescript jsx import { EntityAirbrakeContent } from '@backstage/plugin-airbrake'; @@ -32,40 +32,78 @@ The Airbrake plugin provides connectivity between Backstage and Airbrake (https: ); + + const websiteEntityPage = ( + + + + + + ); + + const defaultEntityPage = ( + + + + + + ); ``` -4. Setup the Backend code in `packages/backend/src/index.ts`: +4. Create `packages/backend/src/plugins/airbrake.ts` with these contents: ```typescript + import { Router } from 'express'; + import { PluginEnvironment } from '../types'; import { - createRouter as createAirbrakeRouter, + createRouter, extractAirbrakeConfig, } from '@backstage/plugin-airbrake-backend'; - async function main() { - //... After const config = await loadBackendConfig({ ... - - const airbrakeRouter = await createAirbrakeRouter({ + export default async function createPlugin({ + logger, + config, + }: PluginEnvironment): Promise { + return createRouter({ logger, airbrakeConfig: extractAirbrakeConfig(config), }); - - const service = createServiceBuilder(module) - // ... Add the airbrakeRouter here - .addRouter('/api/airbrake', airbrakeRouter); } ``` -5. Add this config as a top level section in your `app-config.yaml`: +5. Setup the Backend code in `packages/backend/src/index.ts`: + + ```typescript + import airbrake from './plugins/airbrake'; + + async function main() { + //... After const createEnv = makeCreateEnv(config) ... + + const airbrakeEnv = useHotMemoize(module, () => createEnv('airbrake')); + + //... After const apiRouter = Router() ... + apiRouter.use('/airbrake', await airbrake(airbrakeEnv)); + } + ``` + +6. Add this config as a top level section in your `app-config.yaml`: ```yaml airbrake: apiKey: ${AIRBRAKE_API_KEY} ``` -6. Set an environment variable `AIRBRAKE_API_KEY` with your [API key](https://airbrake.io/docs/api/#authentication) +7. Set an environment variable `AIRBRAKE_API_KEY` with your [API key](https://airbrake.io/docs/api/#authentication) before starting Backstage backend. +8. Add the following annotation to the `catalog-info.yaml` for a repo you want to link to an Airbrake project: + + ```yaml + metadata: + annotations: + airbrake.io/project-id: '123456' + ``` + ## Local Development Start this plugin in standalone mode by running `yarn start` inside the plugin directory. This method of serving the plugin provides quicker From 71353b0fe0cbe53dd1775c217d1fffb2ad58370f Mon Sep 17 00:00:00 2001 From: Karan Shah Date: Mon, 28 Feb 2022 11:25:16 +0000 Subject: [PATCH 020/150] Small grammar improvement Signed-off-by: Karan Shah --- plugins/airbrake/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/airbrake/README.md b/plugins/airbrake/README.md index 28c230fb05..bc830c326d 100644 --- a/plugins/airbrake/README.md +++ b/plugins/airbrake/README.md @@ -20,7 +20,7 @@ The Airbrake plugin provides connectivity between Backstage and Airbrake (https: yarn add @backstage/plugin-airbrake-backend ``` -3. Add the `EntityAirbrakeContent` to `packages/app/src/components/catalog/EntityPage.tsx` for all entity pages you want Airbrake to be in: +3. Add the `EntityAirbrakeContent` to `packages/app/src/components/catalog/EntityPage.tsx` for all the entity pages you want Airbrake to be in: ```typescript jsx import { EntityAirbrakeContent } from '@backstage/plugin-airbrake'; From ed1083a12fea50a9c59d4ba113e173fb01f19a89 Mon Sep 17 00:00:00 2001 From: Karan Shah Date: Mon, 28 Feb 2022 11:35:09 +0000 Subject: [PATCH 021/150] Improve the backend README as well. Signed-off-by: Karan Shah --- plugins/airbrake-backend/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/airbrake-backend/README.md b/plugins/airbrake-backend/README.md index 9c9b56ade4..21056229de 100644 --- a/plugins/airbrake-backend/README.md +++ b/plugins/airbrake-backend/README.md @@ -24,7 +24,7 @@ This method of serving the plugin provides quicker iteration speed and a faster 3. Go into the plugin's directory and run it in standalone mode by running `yarn start`. -Access it from http://localhost:7007/api/airbrake. Or use the Airbrake plugin which will talk to it automatically. +Access it from http://localhost:7007/api/airbrake. Or use the [Airbrake plugin in standalone mode](../airbrake/README.md#local-development) which will talk to it automatically. Here are some example endpoints: From 2a8d9ff6350362a0a2e9adc02cfc22c204c63d38 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 28 Feb 2022 13:16:42 +0100 Subject: [PATCH 022/150] storybook: remove usage of oauth2ApiRef & auth0AuthApiRef Signed-off-by: Johan Haals --- storybook/.storybook/apis.js | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/storybook/.storybook/apis.js b/storybook/.storybook/apis.js index 4c0813a8e9..37cc8eb605 100644 --- a/storybook/.storybook/apis.js +++ b/storybook/.storybook/apis.js @@ -5,10 +5,8 @@ import { GithubAuth, GitlabAuth, GoogleAuth, - OAuth2, OAuthRequestManager, OktaAuth, - Auth0Auth, ConfigReader, LocalStorageFeatureFlags, } from '@backstage/core-app-api'; @@ -20,10 +18,8 @@ import { gitlabAuthApiRef, googleAuthApiRef, identityApiRef, - oauth2ApiRef, oauthRequestApiRef, oktaAuthApiRef, - auth0AuthApiRef, configApiRef, featureFlagsApiRef, } from '@backstage/core-plugin-api'; @@ -59,16 +55,6 @@ const oktaAuthApi = OktaAuth.create({ basePath: '/auth/', oauthRequestApi, }); -const auth0AuthApi = Auth0Auth.create({ - apiOrigin: 'http://localhost:7007', - basePath: '/auth/', - oauthRequestApi, -}); -const oauth2Api = OAuth2.create({ - apiOrigin: 'http://localhost:7007', - basePath: '/auth/', - oauthRequestApi, -}); export const apis = [ [configApiRef, configApi], @@ -81,6 +67,4 @@ export const apis = [ [githubAuthApiRef, githubAuthApi], [gitlabAuthApiRef, gitlabAuthApi], [oktaAuthApiRef, oktaAuthApi], - [auth0AuthApiRef, auth0AuthApi], - [oauth2ApiRef, oauth2Api], ]; From 03ec06bf7f62daa648c34eea70080881d0b00eef Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 18 Feb 2022 17:05:28 +0100 Subject: [PATCH 023/150] catalog-react: refactor and fixes for DefaultStarredEntitiesApi Signed-off-by: Patrik Oldsberg --- .changeset/silver-boxes-flash.md | 7 + plugins/catalog-react/api-report.md | 2 - .../DefaultStarredEntitiesApi.test.ts | 120 ++++++++---------- .../DefaultStarredEntitiesApi.ts | 8 +- 4 files changed, 65 insertions(+), 72 deletions(-) create mode 100644 .changeset/silver-boxes-flash.md diff --git a/.changeset/silver-boxes-flash.md b/.changeset/silver-boxes-flash.md new file mode 100644 index 0000000000..4d3d70259a --- /dev/null +++ b/.changeset/silver-boxes-flash.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +Fixed a risky behavior where `DefaultStarredEntitiesApi` forwarded values to observers that were later mutated. + +Removed the `isStarred` method from `DefaultStarredEntitiesApi`, as it is not part of the `StarredEntitiesApi`. diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index e78ec75daa..add76a7c1c 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -142,8 +142,6 @@ export type DefaultEntityFilters = { export class DefaultStarredEntitiesApi implements StarredEntitiesApi { constructor(opts: { storageApi: StorageApi }); // (undocumented) - isStarred(entityRef: string): boolean; - // (undocumented) starredEntitie$(): Observable>; // (undocumented) toggleStarred(entityRef: string): Promise; diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts index 55b992fadf..2a3c4d83ed 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts @@ -14,33 +14,27 @@ * limitations under the License. */ -import { stringifyEntityRef } from '@backstage/catalog-model'; -import { StorageApi } from '@backstage/core-plugin-api'; import { MockStorageApi } from '@backstage/test-utils'; import { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; import { performMigrationToTheNewBucket } from './migration'; jest.mock('./migration'); -describe('DefaultStarredEntitiesApi', () => { - let mockStorage: StorageApi; - let starredEntitiesApi: DefaultStarredEntitiesApi; - - const mockEntityRef = stringifyEntityRef({ - apiVersion: '1', - kind: 'Component', - metadata: { - name: 'mock', - }, +function getStarred(api: DefaultStarredEntitiesApi) { + return new Promise((resolve, reject) => { + const subscription = api.starredEntitie$().subscribe({ + next(starred) { + resolve(starred); + subscription.unsubscribe(); + }, + error: reject, + }); }); +} +describe('DefaultStarredEntitiesApi', () => { beforeEach(() => { (performMigrationToTheNewBucket as jest.Mock).mockResolvedValue(undefined); - - mockStorage = MockStorageApi.create(); - starredEntitiesApi = new DefaultStarredEntitiesApi({ - storageApi: mockStorage, - }); }); afterEach(() => { @@ -49,55 +43,53 @@ describe('DefaultStarredEntitiesApi', () => { describe('constructor', () => { it('should call migration', () => { - expect(performMigrationToTheNewBucket).toBeCalledTimes(1); - }); - }); - - describe('toggleStarred', () => { - it('should star unstarred entity', async () => { - expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(false); - - await starredEntitiesApi.toggleStarred(mockEntityRef); - - expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(true); - }); - - it('should unstar starred entity', async () => { - const bucket = mockStorage.forBucket('starredEntities'); - await bucket.set('entityRefs', ['component:default/mock']); - - expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(true); - - await starredEntitiesApi.toggleStarred(mockEntityRef); - - expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(false); - }); - }); - - describe('starredEntities$', () => { - const handler = jest.fn(); - - beforeEach(async () => { - await new Promise(resolve => { - starredEntitiesApi.starredEntitie$().subscribe({ - next: (...args) => { - handler(...args); - - if (handler.mock.calls.length >= 2) { - resolve(); - } - }, - }); - - const bucket = mockStorage.forBucket('starredEntities'); - bucket.set('entityRefs', ['component:default/mock']).then(); + const api = new DefaultStarredEntitiesApi({ + storageApi: MockStorageApi.create(), }); + expect(performMigrationToTheNewBucket).toBeCalledTimes(1); + expect(api).toBeDefined(); + }); + }); + + it('should notify and toggle starred entities', async () => { + const entityRef = 'component:default/mock'; + + const storageApi = MockStorageApi.create(); + const storageBucket = storageApi.forBucket('starredEntities'); + const api = new DefaultStarredEntitiesApi({ storageApi }); + + const values = new Array>(); + api.starredEntitie$().subscribe({ + next: value => { + values.push(value); + }, }); - it('should receive updates', async () => { - expect(handler).toBeCalledTimes(2); - expect(handler).toBeCalledWith(new Set()); - expect(handler).toBeCalledWith(new Set(['component:default/mock'])); - }); + await expect(getStarred(api)).resolves.toEqual(new Set()); + + await api.toggleStarred(entityRef); + await expect(getStarred(api)).resolves.toEqual(new Set([entityRef])); + expect(storageBucket.snapshot('entityRefs')).toEqual( + expect.objectContaining({ presence: 'present', value: [entityRef] }), + ); + + await api.toggleStarred(entityRef); + await expect(getStarred(api)).resolves.toEqual(new Set()); + expect(storageBucket.snapshot('entityRefs')).toEqual( + expect.objectContaining({ presence: 'present', value: [] }), + ); + + expect(values).toEqual([new Set(), new Set([entityRef]), new Set()]); + }); + + it('should read starred entities from storage', async () => { + const entityRef = 'component:default/mock'; + + const storageApi = MockStorageApi.create(); + const storageBucket = storageApi.forBucket('starredEntities'); + storageBucket.set('entityRefs', [entityRef]); + const api = new DefaultStarredEntitiesApi({ storageApi }); + + await expect(getStarred(api)).resolves.toEqual(new Set([entityRef])); }); }); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts index 46a8e63084..c411c5a81f 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -64,17 +64,13 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { return this.observable; } - isStarred(entityRef: string): boolean { - return this.starredEntities.has(entityRef); - } - private readonly subscribers = new Set< ZenObservable.SubscriptionObserver> >(); private readonly observable = new ObservableImpl>(subscriber => { // forward the the latest value - subscriber.next(this.starredEntities); + subscriber.next(new Set(this.starredEntities)); this.subscribers.add(subscriber); return () => { @@ -84,7 +80,7 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { private notifyChanges() { for (const subscription of this.subscribers) { - subscription.next(this.starredEntities); + subscription.next(new Set(this.starredEntities)); } } } From c077b432b8cf1f76ea95881b84df9d2949cba2be Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 18 Feb 2022 17:42:18 +0100 Subject: [PATCH 024/150] catalog-react: moved DefaultStarredEntitiesApi implementation to catalog plugin Signed-off-by: Patrik Oldsberg --- .changeset/silver-boxes-flash.md | 2 ++ .../src/components/catalog/EntityPage.test.tsx | 4 +++- .../DefaultApiExplorerPage.test.tsx | 6 ++++-- plugins/catalog-react/api-report.md | 10 ---------- .../src/apis/StarredEntitiesApi/index.ts | 1 - .../src/hooks/useEntityListProvider.test.tsx | 3 ++- .../src/hooks/useStarredEntities.test.tsx | 3 ++- plugins/catalog/api-report.md | 12 ++++++++++++ plugins/catalog/package.json | 4 +++- .../DefaultStarredEntitiesApi.test.ts | 0 .../DefaultStarredEntitiesApi.ts | 2 +- .../src/apis/StarredEntitiesApi/index.ts | 17 +++++++++++++++++ .../apis/StarredEntitiesApi/migration.test.ts | 0 .../src/apis/StarredEntitiesApi/migration.ts | 0 plugins/catalog/src/apis/index.ts | 17 +++++++++++++++++ .../CatalogPage/DefaultCatalogPage.test.tsx | 2 +- .../CatalogTable/CatalogTable.test.tsx | 2 +- .../EntityLayout/EntityLayout.test.tsx | 2 +- plugins/catalog/src/index.ts | 2 ++ plugins/catalog/src/plugin.ts | 2 +- .../StarredEntities/Content.test.tsx | 2 +- .../StarredEntities/StarredEntities.stories.tsx | 2 +- .../src/templates/DefaultTemplate.stories.tsx | 3 +-- plugins/scaffolder/dev/index.tsx | 6 ++++-- .../components/DefaultTechDocsHome.test.tsx | 2 +- 25 files changed, 77 insertions(+), 29 deletions(-) rename plugins/{catalog-react => catalog}/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts (100%) rename plugins/{catalog-react => catalog}/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts (97%) create mode 100644 plugins/catalog/src/apis/StarredEntitiesApi/index.ts rename plugins/{catalog-react => catalog}/src/apis/StarredEntitiesApi/migration.test.ts (100%) rename plugins/{catalog-react => catalog}/src/apis/StarredEntitiesApi/migration.ts (100%) create mode 100644 plugins/catalog/src/apis/index.ts diff --git a/.changeset/silver-boxes-flash.md b/.changeset/silver-boxes-flash.md index 4d3d70259a..3b860a06d6 100644 --- a/.changeset/silver-boxes-flash.md +++ b/.changeset/silver-boxes-flash.md @@ -2,6 +2,8 @@ '@backstage/plugin-catalog-react': patch --- +**BREAKING**: Moved **DefaultStarredEntitiesApi** to `@backstage/plugin-catalog`. If you were using this in tests, you can add `@backstage/plugin-catalog` your packages `devDependencies` instead. + Fixed a risky behavior where `DefaultStarredEntitiesApi` forwarded values to observers that were later mutated. Removed the `isStarred` method from `DefaultStarredEntitiesApi`, as it is not part of the `StarredEntitiesApi`. diff --git a/packages/app/src/components/catalog/EntityPage.test.tsx b/packages/app/src/components/catalog/EntityPage.test.tsx index b68b9e9895..4608dc7533 100644 --- a/packages/app/src/components/catalog/EntityPage.test.tsx +++ b/packages/app/src/components/catalog/EntityPage.test.tsx @@ -14,9 +14,11 @@ * limitations under the License. */ -import { EntityLayout } from '@backstage/plugin-catalog'; import { + EntityLayout, DefaultStarredEntitiesApi, +} from '@backstage/plugin-catalog'; +import { EntityProvider, starredEntitiesApiRef, } from '@backstage/plugin-catalog-react'; diff --git a/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx b/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx index 16b335b0a7..5924af3ef3 100644 --- a/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx +++ b/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx @@ -22,11 +22,13 @@ import { configApiRef, storageApiRef, } from '@backstage/core-plugin-api'; -import { CatalogTableRow } from '@backstage/plugin-catalog'; +import { + CatalogTableRow, + DefaultStarredEntitiesApi, +} from '@backstage/plugin-catalog'; import { CatalogApi, catalogApiRef, - DefaultStarredEntitiesApi, entityRouteRef, starredEntitiesApiRef, } from '@backstage/plugin-catalog-react'; diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index add76a7c1c..b8f0fa5538 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -24,7 +24,6 @@ import { default as React_2 } from 'react'; import { ReactNode } from 'react'; import { RouteRef } from '@backstage/core-plugin-api'; import { ScmIntegrationRegistry } from '@backstage/integration'; -import { StorageApi } from '@backstage/core-plugin-api'; import { StyleRules } from '@material-ui/core/styles/withStyles'; import { SystemEntity } from '@backstage/catalog-model'; import { TableColumn } from '@backstage/core-components'; @@ -138,15 +137,6 @@ export type DefaultEntityFilters = { text?: EntityTextFilter; }; -// @public -export class DefaultStarredEntitiesApi implements StarredEntitiesApi { - constructor(opts: { storageApi: StorageApi }); - // (undocumented) - starredEntitie$(): Observable>; - // (undocumented) - toggleStarred(entityRef: string): Promise; -} - // @public (undocumented) export type EntityFilter = { getCatalogFilters?: () => Record< diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts index e9f9c8923a..94a221b36f 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts @@ -14,6 +14,5 @@ * limitations under the License. */ -export { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; export { starredEntitiesApiRef } from './StarredEntitiesApi'; export type { StarredEntitiesApi } from './StarredEntitiesApi'; diff --git a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx index c8ca29b07f..5db9e2be9c 100644 --- a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx +++ b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx @@ -23,13 +23,14 @@ import { identityApiRef, storageApiRef, } from '@backstage/core-plugin-api'; +import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; import { MockStorageApi, TestApiProvider } from '@backstage/test-utils'; import { act, renderHook } from '@testing-library/react-hooks'; import qs from 'qs'; import React, { PropsWithChildren } from 'react'; import { MemoryRouter } from 'react-router'; import { catalogApiRef } from '../api'; -import { DefaultStarredEntitiesApi, starredEntitiesApiRef } from '../apis'; +import { starredEntitiesApiRef } from '../apis'; import { EntityKindPicker, UserListPicker } from '../components'; import { EntityKindFilter, EntityTypeFilter, UserListFilter } from '../filters'; import { UserListFilterKind } from '../types'; diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx b/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx index aef876a1b2..949ac6515b 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx +++ b/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx @@ -16,10 +16,11 @@ import { Entity } from '@backstage/catalog-model'; import { StorageApi } from '@backstage/core-plugin-api'; +import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; import { MockStorageApi, TestApiProvider } from '@backstage/test-utils'; import { act, renderHook } from '@testing-library/react-hooks'; import React, { PropsWithChildren } from 'react'; -import { DefaultStarredEntitiesApi, starredEntitiesApiRef } from '../apis'; +import { starredEntitiesApiRef } from '../apis'; import { useStarredEntities } from './useStarredEntities'; describe('useStarredEntities', () => { diff --git a/plugins/catalog/api-report.md b/plugins/catalog/api-report.md index bd5edcbfa6..f314932b9a 100644 --- a/plugins/catalog/api-report.md +++ b/plugins/catalog/api-report.md @@ -13,10 +13,13 @@ import { ExternalRouteRef } from '@backstage/core-plugin-api'; import { IconComponent } from '@backstage/core-plugin-api'; import { IndexableDocument } from '@backstage/search-common'; import { InfoCardVariants } from '@backstage/core-components'; +import { Observable } from '@backstage/types'; import { Overrides } from '@material-ui/core/styles/overrides'; import { default as React_2 } from 'react'; import { ReactNode } from 'react'; import { RouteRef } from '@backstage/core-plugin-api'; +import { StarredEntitiesApi } from '@backstage/plugin-catalog-react'; +import { StorageApi } from '@backstage/core-plugin-api'; import { StyleRules } from '@material-ui/core/styles/withStyles'; import { TableColumn } from '@backstage/core-components'; import { TableProps } from '@backstage/core-components'; @@ -170,6 +173,15 @@ export interface DefaultCatalogPageProps { initiallySelectedFilter?: UserListFilterKind; } +// @public +export class DefaultStarredEntitiesApi implements StarredEntitiesApi { + constructor(opts: { storageApi: StorageApi }); + // (undocumented) + starredEntitie$(): Observable>; + // (undocumented) + toggleStarred(entityRef: string): Promise; +} + // @public (undocumented) export interface DependencyOfComponentsCardProps { // (undocumented) diff --git a/plugins/catalog/package.json b/plugins/catalog/package.json index f7800af139..6179cf6c44 100644 --- a/plugins/catalog/package.json +++ b/plugins/catalog/package.json @@ -44,6 +44,7 @@ "@backstage/plugin-catalog-react": "^0.7.0", "@backstage/search-common": "^0.2.4", "@backstage/theme": "^0.2.15", + "@backstage/types": "^0.1.2", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.57", @@ -51,7 +52,8 @@ "lodash": "^4.17.21", "react-helmet": "6.1.0", "react-router": "6.0.0-beta.0", - "react-use": "^17.2.4" + "react-use": "^17.2.4", + "zen-observable": "^0.8.15" }, "peerDependencies": { "@types/react": "^16.13.1 || ^17.0.0", diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts similarity index 100% rename from plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts rename to plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts similarity index 97% rename from plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts rename to plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts index c411c5a81f..e9cf84f11b 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts +++ b/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -15,10 +15,10 @@ */ import { StorageApi } from '@backstage/core-plugin-api'; +import { StarredEntitiesApi } from '@backstage/plugin-catalog-react'; import { Observable } from '@backstage/types'; import ObservableImpl from 'zen-observable'; import { performMigrationToTheNewBucket } from './migration'; -import { StarredEntitiesApi } from './StarredEntitiesApi'; /** * Default implementation of the StarredEntitiesApi that is backed by the StorageApi. diff --git a/plugins/catalog/src/apis/StarredEntitiesApi/index.ts b/plugins/catalog/src/apis/StarredEntitiesApi/index.ts new file mode 100644 index 0000000000..42dc977fb6 --- /dev/null +++ b/plugins/catalog/src/apis/StarredEntitiesApi/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.test.ts b/plugins/catalog/src/apis/StarredEntitiesApi/migration.test.ts similarity index 100% rename from plugins/catalog-react/src/apis/StarredEntitiesApi/migration.test.ts rename to plugins/catalog/src/apis/StarredEntitiesApi/migration.test.ts diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.ts b/plugins/catalog/src/apis/StarredEntitiesApi/migration.ts similarity index 100% rename from plugins/catalog-react/src/apis/StarredEntitiesApi/migration.ts rename to plugins/catalog/src/apis/StarredEntitiesApi/migration.ts diff --git a/plugins/catalog/src/apis/index.ts b/plugins/catalog/src/apis/index.ts new file mode 100644 index 0000000000..5c7e980890 --- /dev/null +++ b/plugins/catalog/src/apis/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './StarredEntitiesApi'; diff --git a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx index dcf819f976..e36c1e9338 100644 --- a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx +++ b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx @@ -29,7 +29,6 @@ import { } from '@backstage/core-plugin-api'; import { catalogApiRef, - DefaultStarredEntitiesApi, entityRouteRef, starredEntitiesApiRef, } from '@backstage/plugin-catalog-react'; @@ -43,6 +42,7 @@ import { import DashboardIcon from '@material-ui/icons/Dashboard'; import { fireEvent, screen } from '@testing-library/react'; import React from 'react'; +import { DefaultStarredEntitiesApi } from '../../apis'; import { createComponentRouteRef } from '../../routes'; import { CatalogTableRow } from '../CatalogTable'; import { DefaultCatalogPage } from './DefaultCatalogPage'; diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx index db9835e3a3..1d49638eed 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx @@ -22,7 +22,6 @@ import { import { ApiProvider } from '@backstage/core-app-api'; import { entityRouteRef, - DefaultStarredEntitiesApi, MockEntityListContextProvider, starredEntitiesApiRef, UserListFilter, @@ -34,6 +33,7 @@ import { } from '@backstage/test-utils'; import { act, fireEvent } from '@testing-library/react'; import * as React from 'react'; +import { DefaultStarredEntitiesApi } from '../../apis'; import { CatalogTable } from './CatalogTable'; const entities: Entity[] = [ diff --git a/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx b/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx index 78dcf855e8..b29261c28b 100644 --- a/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx +++ b/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx @@ -21,7 +21,6 @@ import { AlertApi, alertApiRef } from '@backstage/core-plugin-api'; import { AsyncEntityProvider, catalogApiRef, - DefaultStarredEntitiesApi, EntityProvider, entityRouteRef, starredEntitiesApiRef, @@ -36,6 +35,7 @@ import { import { act, fireEvent } from '@testing-library/react'; import React from 'react'; import { Route, Routes } from 'react-router'; +import { DefaultStarredEntitiesApi } from '../../apis'; import { EntityLayout } from './EntityLayout'; const mockEntity = { diff --git a/plugins/catalog/src/index.ts b/plugins/catalog/src/index.ts index fef420d6d3..662a920f8b 100644 --- a/plugins/catalog/src/index.ts +++ b/plugins/catalog/src/index.ts @@ -20,6 +20,8 @@ * @packageDocumentation */ +export * from './apis'; + export * from './components/AboutCard'; export * from './components/CatalogKindHeader'; export * from './components/CatalogSearchResultListItem'; diff --git a/plugins/catalog/src/plugin.ts b/plugins/catalog/src/plugin.ts index b4d1a61bc1..520656e921 100644 --- a/plugins/catalog/src/plugin.ts +++ b/plugins/catalog/src/plugin.ts @@ -19,7 +19,6 @@ import { Entity } from '@backstage/catalog-model'; import { catalogApiRef, catalogRouteRef, - DefaultStarredEntitiesApi, entityRouteRef, starredEntitiesApiRef, } from '@backstage/plugin-catalog-react'; @@ -33,6 +32,7 @@ import { fetchApiRef, storageApiRef, } from '@backstage/core-plugin-api'; +import { DefaultStarredEntitiesApi } from './apis'; import { AboutCardProps } from './components/AboutCard'; import { DefaultCatalogPageProps } from './components/CatalogPage'; import { DependencyOfComponentsCardProps } from './components/DependencyOfComponentsCard'; diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx index ccc36f6964..ae4a84473f 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx @@ -21,8 +21,8 @@ import { import { starredEntitiesApiRef, entityRouteRef, - DefaultStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; +import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; import React from 'react'; import { Content } from './Content'; diff --git a/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx b/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx index 2e63762369..55486a5dcd 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx @@ -23,8 +23,8 @@ import { import { starredEntitiesApiRef, entityRouteRef, - DefaultStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; +import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; import { Grid } from '@material-ui/core'; import React, { ComponentType } from 'react'; diff --git a/plugins/home/src/templates/DefaultTemplate.stories.tsx b/plugins/home/src/templates/DefaultTemplate.stories.tsx index ba5e1f33fd..86d1b60022 100644 --- a/plugins/home/src/templates/DefaultTemplate.stories.tsx +++ b/plugins/home/src/templates/DefaultTemplate.stories.tsx @@ -26,8 +26,8 @@ import { Content, Page, InfoCard } from '@backstage/core-components'; import { starredEntitiesApiRef, entityRouteRef, - DefaultStarredEntitiesApi } from '@backstage/plugin-catalog-react'; +import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; import { HomePageSearchBar, SearchContextProvider, @@ -153,4 +153,3 @@ export const DefaultTemplate = () => { ); }; - diff --git a/plugins/scaffolder/dev/index.tsx b/plugins/scaffolder/dev/index.tsx index 78e6254e09..6ecc1a4564 100644 --- a/plugins/scaffolder/dev/index.tsx +++ b/plugins/scaffolder/dev/index.tsx @@ -20,7 +20,6 @@ import { scmIntegrationsApiRef } from '@backstage/integration-react'; import { catalogApiRef, starredEntitiesApiRef, - DefaultStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import React from 'react'; import { scaffolderApiRef, ScaffolderClient } from '../src'; @@ -30,7 +29,10 @@ import { fetchApiRef, storageApiRef, } from '@backstage/core-plugin-api'; -import { CatalogEntityPage } from '@backstage/plugin-catalog'; +import { + CatalogEntityPage, + DefaultStarredEntitiesApi, +} from '@backstage/plugin-catalog'; createDevApp() .addPage({ diff --git a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx index 8618c4432f..678c737547 100644 --- a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx +++ b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx @@ -20,10 +20,10 @@ import { configApiRef, storageApiRef, } from '@backstage/core-plugin-api'; +import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; import { CatalogApi, catalogApiRef, - DefaultStarredEntitiesApi, starredEntitiesApiRef, } from '@backstage/plugin-catalog-react'; import { From 2c5d38e337548cf3078addfebcee20894bca0c68 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 28 Feb 2022 16:19:05 +0100 Subject: [PATCH 025/150] chore: don't export the components that back the field extensions Signed-off-by: blam --- plugins/scaffolder/api-report.md | 62 ++++++------------- .../fields/EntityNamePicker/index.ts | 1 - .../components/fields/EntityPicker/index.ts | 1 - .../fields/EntityTagsPicker/index.ts | 1 - .../fields/OwnedEntityPicker/index.ts | 1 - .../fields/OwnerPicker/OwnerPicker.tsx | 2 +- .../components/fields/OwnerPicker/index.ts | 1 - .../components/fields/RepoUrlPicker/index.ts | 1 - plugins/scaffolder/src/extensions/default.ts | 20 +++--- plugins/scaffolder/src/extensions/index.tsx | 6 +- plugins/scaffolder/src/plugin.ts | 20 +++--- 11 files changed, 40 insertions(+), 76 deletions(-) diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index 82dfecfaa5..eed65578d3 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -28,6 +28,7 @@ import { ScmIntegrationRegistry } from '@backstage/integration'; import { TaskSpec } from '@backstage/plugin-scaffolder-common'; import { TemplateEntityV1beta2 } from '@backstage/plugin-scaffolder-common'; +// Warning: (ae-forgotten-export) The symbol "FieldExtensionComponent" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "createScaffolderFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -36,7 +37,7 @@ export function createScaffolderFieldExtension< TInputProps = unknown, >( options: FieldExtensionOptions, -): Extension<() => null>; +): Extension>; // @public export type CustomFieldValidator = ( @@ -50,19 +51,16 @@ export type CustomFieldValidator = ( // Warning: (ae-missing-release-tag) "EntityNamePickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const EntityNamePickerFieldExtension: () => null; - -// Warning: (ae-missing-release-tag) "EntityPicker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export const EntityPicker: ( - props: FieldExtensionComponentProps, -) => JSX.Element; +export const EntityNamePickerFieldExtension: FieldExtensionComponent< + FieldExtensionComponentProps +>; // Warning: (ae-missing-release-tag) "EntityPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const EntityPickerFieldExtension: () => null; +export const EntityPickerFieldExtension: FieldExtensionComponent< + FieldExtensionComponentProps +>; // Warning: (ae-missing-release-tag) "EntityPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -76,15 +74,10 @@ export interface EntityPickerUiOptions { defaultKind?: string; } -// Warning: (ae-missing-release-tag) "EntityTagsPicker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public -export const EntityTagsPicker: ( - props: FieldExtensionComponentProps, -) => JSX.Element; - -// @public -export const EntityTagsPickerFieldExtension: () => null; +export const EntityTagsPickerFieldExtension: FieldExtensionComponent< + FieldExtensionComponentProps +>; // Warning: (ae-missing-release-tag) "EntityTagsPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -153,17 +146,12 @@ export type LogEvent = { taskId: string; }; -// Warning: (ae-missing-release-tag) "OwnedEntityPicker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export const OwnedEntityPicker: ( - props: FieldExtensionComponentProps, -) => JSX.Element; - // Warning: (ae-missing-release-tag) "OwnedEntityPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const OwnedEntityPickerFieldExtension: () => null; +export const OwnedEntityPickerFieldExtension: FieldExtensionComponent< + FieldExtensionComponentProps +>; // Warning: (ae-missing-release-tag) "OwnedEntityPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -175,17 +163,12 @@ export interface OwnedEntityPickerUiOptions { defaultKind?: string; } -// Warning: (ae-missing-release-tag) "OwnerPicker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export const OwnerPicker: ( - props: FieldExtensionComponentProps, -) => JSX.Element; - // Warning: (ae-missing-release-tag) "OwnerPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const OwnerPickerFieldExtension: () => null; +export const OwnerPickerFieldExtension: FieldExtensionComponent< + FieldExtensionComponentProps +>; // Warning: (ae-missing-release-tag) "OwnerPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -206,17 +189,12 @@ export const repoPickerValidation: ( }, ) => void; -// Warning: (ae-missing-release-tag) "RepoUrlPicker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export const RepoUrlPicker: ( - props: FieldExtensionComponentProps, -) => JSX.Element; - // Warning: (ae-missing-release-tag) "RepoUrlPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const RepoUrlPickerFieldExtension: () => null; +export const RepoUrlPickerFieldExtension: FieldExtensionComponent< + FieldExtensionComponentProps +>; // Warning: (ae-missing-release-tag) "RepoUrlPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/plugins/scaffolder/src/components/fields/EntityNamePicker/index.ts b/plugins/scaffolder/src/components/fields/EntityNamePicker/index.ts index 0ba88d1277..7076221480 100644 --- a/plugins/scaffolder/src/components/fields/EntityNamePicker/index.ts +++ b/plugins/scaffolder/src/components/fields/EntityNamePicker/index.ts @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { EntityNamePicker } from './EntityNamePicker'; export { entityNamePickerValidation } from './validation'; diff --git a/plugins/scaffolder/src/components/fields/EntityPicker/index.ts b/plugins/scaffolder/src/components/fields/EntityPicker/index.ts index eb7511bbef..891b5bef16 100644 --- a/plugins/scaffolder/src/components/fields/EntityPicker/index.ts +++ b/plugins/scaffolder/src/components/fields/EntityPicker/index.ts @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { EntityPicker } from './EntityPicker'; export type { EntityPickerUiOptions } from './EntityPicker'; diff --git a/plugins/scaffolder/src/components/fields/EntityTagsPicker/index.ts b/plugins/scaffolder/src/components/fields/EntityTagsPicker/index.ts index 3ba36a2409..9ff6e553a6 100644 --- a/plugins/scaffolder/src/components/fields/EntityTagsPicker/index.ts +++ b/plugins/scaffolder/src/components/fields/EntityTagsPicker/index.ts @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { EntityTagsPicker } from './EntityTagsPicker'; export type { EntityTagsPickerUiOptions } from './EntityTagsPicker'; diff --git a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/index.ts b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/index.ts index 125f6fdef0..2988ba8cdc 100644 --- a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/index.ts +++ b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/index.ts @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { OwnedEntityPicker } from './OwnedEntityPicker'; export type { OwnedEntityPickerUiOptions } from './OwnedEntityPicker'; diff --git a/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx b/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx index 9012ed149e..1824275380 100644 --- a/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx +++ b/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ import React from 'react'; -import { EntityPicker } from '../EntityPicker'; +import { EntityPicker } from '../EntityPicker/EntityPicker'; import { FieldExtensionComponentProps } from '../../../extensions'; export interface OwnerPickerUiOptions { diff --git a/plugins/scaffolder/src/components/fields/OwnerPicker/index.ts b/plugins/scaffolder/src/components/fields/OwnerPicker/index.ts index b8cb97eb97..aa26024b5b 100644 --- a/plugins/scaffolder/src/components/fields/OwnerPicker/index.ts +++ b/plugins/scaffolder/src/components/fields/OwnerPicker/index.ts @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { OwnerPicker } from './OwnerPicker'; export type { OwnerPickerUiOptions } from './OwnerPicker'; diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts index 2fdbf0aac9..c5f596a786 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts @@ -13,6 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { RepoUrlPicker } from './RepoUrlPicker'; export type { RepoUrlPickerUiOptions } from './RepoUrlPicker'; export { repoPickerValidation } from './validation'; diff --git a/plugins/scaffolder/src/extensions/default.ts b/plugins/scaffolder/src/extensions/default.ts index 10c0746e41..b3bb37065e 100644 --- a/plugins/scaffolder/src/extensions/default.ts +++ b/plugins/scaffolder/src/extensions/default.ts @@ -13,19 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityPicker } from '../components/fields/EntityPicker'; -import { - EntityNamePicker, - entityNamePickerValidation, -} from '../components/fields/EntityNamePicker'; -import { EntityTagsPicker } from '../components/fields/EntityTagsPicker'; -import { OwnerPicker } from '../components/fields/OwnerPicker'; -import { - repoPickerValidation, - RepoUrlPicker, -} from '../components/fields/RepoUrlPicker'; +import { EntityPicker } from '../components/fields/EntityPicker/EntityPicker'; +import { EntityNamePicker } from '../components/fields/EntityNamePicker/EntityNamePicker'; +import { entityNamePickerValidation } from '../components/fields/EntityNamePicker/validation'; +import { EntityTagsPicker } from '../components/fields/EntityTagsPicker/EntityTagsPicker'; +import { OwnerPicker } from '../components/fields/OwnerPicker/OwnerPicker'; +import { RepoUrlPicker } from '../components/fields/RepoUrlPicker/RepoUrlPicker'; +import { repoPickerValidation } from '../components/fields/RepoUrlPicker/validation'; import { FieldExtensionOptions } from './types'; -import { OwnedEntityPicker } from '../components/fields/OwnedEntityPicker'; +import { OwnedEntityPicker } from '../components/fields/OwnedEntityPicker/OwnedEntityPicker'; export const DEFAULT_SCAFFOLDER_FIELD_EXTENSIONS: FieldExtensionOptions[] = [ diff --git a/plugins/scaffolder/src/extensions/index.tsx b/plugins/scaffolder/src/extensions/index.tsx index 53db4cd927..72df8d4df3 100644 --- a/plugins/scaffolder/src/extensions/index.tsx +++ b/plugins/scaffolder/src/extensions/index.tsx @@ -25,14 +25,14 @@ import { Extension, attachComponentData } from '@backstage/core-plugin-api'; export const FIELD_EXTENSION_WRAPPER_KEY = 'scaffolder.extensions.wrapper.v1'; export const FIELD_EXTENSION_KEY = 'scaffolder.extensions.field.v1'; +export type FieldExtensionComponent<_TInputProps> = () => null; + export function createScaffolderFieldExtension< TReturnValue = unknown, TInputProps = unknown, >( options: FieldExtensionOptions, - // TODO: need know how to embed these types nicely so the api report looks nice. - // then we can remove the export of the components -): Extension<() => null> { +): Extension> { return { expose() { const FieldExtensionDataHolder: any = () => null; diff --git a/plugins/scaffolder/src/plugin.ts b/plugins/scaffolder/src/plugin.ts index ccf8d2ee34..0d44a40c80 100644 --- a/plugins/scaffolder/src/plugin.ts +++ b/plugins/scaffolder/src/plugin.ts @@ -16,16 +16,12 @@ import { scmIntegrationsApiRef } from '@backstage/integration-react'; import { scaffolderApiRef, ScaffolderClient } from './api'; -import { EntityPicker } from './components/fields/EntityPicker'; -import { - entityNamePickerValidation, - EntityNamePicker, -} from './components/fields/EntityNamePicker'; -import { OwnerPicker } from './components/fields/OwnerPicker'; -import { - repoPickerValidation, - RepoUrlPicker, -} from './components/fields/RepoUrlPicker'; +import { EntityPicker } from './components/fields/EntityPicker/EntityPicker'; +import { entityNamePickerValidation } from './components/fields/EntityNamePicker'; +import { EntityNamePicker } from './components/fields/EntityNamePicker/EntityNamePicker'; +import { OwnerPicker } from './components/fields/OwnerPicker/OwnerPicker'; +import { repoPickerValidation } from './components/fields/RepoUrlPicker'; +import { RepoUrlPicker } from './components/fields/RepoUrlPicker/RepoUrlPicker'; import { createScaffolderFieldExtension } from './extensions'; import { registerComponentRouteRef, rootRouteRef } from './routes'; import { @@ -35,8 +31,8 @@ import { discoveryApiRef, fetchApiRef, } from '@backstage/core-plugin-api'; -import { OwnedEntityPicker } from './components/fields/OwnedEntityPicker'; -import { EntityTagsPicker } from './components/fields/EntityTagsPicker'; +import { OwnedEntityPicker } from './components/fields/OwnedEntityPicker/OwnedEntityPicker'; +import { EntityTagsPicker } from './components/fields/EntityTagsPicker/EntityTagsPicker'; export const scaffolderPlugin = createPlugin({ id: 'scaffolder', From 86da51cec553f9bf86761d6240d1aef0f0c666d7 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 28 Feb 2022 16:21:42 +0100 Subject: [PATCH 026/150] ochore: added changeset Signed-off-by: blam --- .changeset/swift-roses-hug.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/swift-roses-hug.md diff --git a/.changeset/swift-roses-hug.md b/.changeset/swift-roses-hug.md new file mode 100644 index 0000000000..47a84e3247 --- /dev/null +++ b/.changeset/swift-roses-hug.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +**BREAKING**: Removing the exports of the raw components that back the `CustomFieldExtensions`. From 303063a655cc5dad3b59c45aa221bffecc41a83c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 28 Feb 2022 14:29:23 +0100 Subject: [PATCH 027/150] catalog-react: added MockStarredEntitiesApi + usage Signed-off-by: Patrik Oldsberg --- .changeset/silver-boxes-flash.md | 2 +- .../components/catalog/EntityPage.test.tsx | 14 +---- plugins/catalog-react/api-report.md | 8 +++ .../MockStarredEntitiesApi.test.ts | 61 +++++++++++++++++++ .../MockStarredEntitiesApi.ts | 54 ++++++++++++++++ .../src/apis/StarredEntitiesApi/index.ts | 1 + .../src/hooks/useEntityListProvider.test.tsx | 10 +-- .../src/hooks/useStarredEntities.test.tsx | 32 +++++----- .../CatalogPage/DefaultCatalogPage.test.tsx | 7 +-- .../CatalogTable/CatalogTable.test.tsx | 10 +-- .../EntityLayout/EntityLayout.test.tsx | 8 +-- .../StarredEntities/Content.test.tsx | 29 ++------- .../StarredEntities.stories.tsx | 11 +--- .../src/templates/DefaultTemplate.stories.tsx | 6 +- plugins/scaffolder/dev/index.tsx | 16 ++--- .../components/DefaultTechDocsHome.test.tsx | 4 +- 16 files changed, 168 insertions(+), 105 deletions(-) create mode 100644 plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.test.ts create mode 100644 plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.ts diff --git a/.changeset/silver-boxes-flash.md b/.changeset/silver-boxes-flash.md index 3b860a06d6..9f30c1a34b 100644 --- a/.changeset/silver-boxes-flash.md +++ b/.changeset/silver-boxes-flash.md @@ -2,7 +2,7 @@ '@backstage/plugin-catalog-react': patch --- -**BREAKING**: Moved **DefaultStarredEntitiesApi** to `@backstage/plugin-catalog`. If you were using this in tests, you can add `@backstage/plugin-catalog` your packages `devDependencies` instead. +**BREAKING**: Moved **DefaultStarredEntitiesApi** to `@backstage/plugin-catalog`. If you were using this in tests, you can use the new `MockStarredEntitiesApi` from `@backstage/plugin-catalog-react` instead. Fixed a risky behavior where `DefaultStarredEntitiesApi` forwarded values to observers that were later mutated. diff --git a/packages/app/src/components/catalog/EntityPage.test.tsx b/packages/app/src/components/catalog/EntityPage.test.tsx index 4608dc7533..68ce5fb497 100644 --- a/packages/app/src/components/catalog/EntityPage.test.tsx +++ b/packages/app/src/components/catalog/EntityPage.test.tsx @@ -14,19 +14,16 @@ * limitations under the License. */ -import { - EntityLayout, - DefaultStarredEntitiesApi, -} from '@backstage/plugin-catalog'; +import { EntityLayout } from '@backstage/plugin-catalog'; import { EntityProvider, starredEntitiesApiRef, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import { githubActionsApiRef } from '@backstage/plugin-github-actions'; import { permissionApiRef } from '@backstage/plugin-permission-react'; import { MockPermissionApi, - MockStorageApi, renderInTestApp, TestApiProvider, } from '@backstage/test-utils'; @@ -61,12 +58,7 @@ describe('EntityPage Test', () => { diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index b8f0fa5538..3e1136313f 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -476,6 +476,14 @@ export const MockEntityListContextProvider: ({ value?: Partial> | undefined; }>) => JSX.Element; +// @public +export class MockStarredEntitiesApi implements StarredEntitiesApi { + // (undocumented) + starredEntitie$(): Observable>; + // (undocumented) + toggleStarred(entityRef: string): Promise; +} + // @public @deprecated (undocumented) export function reduceCatalogFilters( filters: EntityFilter[], diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.test.ts new file mode 100644 index 0000000000..aadecefe98 --- /dev/null +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { MockStarredEntitiesApi } from './MockStarredEntitiesApi'; + +describe('MockStarredEntitiesApi', () => { + it('should toggle starred entities', async () => { + const api = new MockStarredEntitiesApi(); + + const updates1 = new Array>(); + const sub1 = api + .starredEntitie$() + .subscribe(entities => updates1.push(entities)); + + api.toggleStarred('k:ns/e1'); + api.toggleStarred('k:ns/e2'); + + await Promise.resolve(); + expect(updates1).toEqual([ + new Set(), + new Set(['k:ns/e1']), + new Set(['k:ns/e1', 'k:ns/e2']), + ]); + + const updates2 = new Array>(); + const sub2 = api + .starredEntitie$() + .subscribe(entities => updates2.push(entities)); + + api.toggleStarred('k:ns/e2'); + sub1.unsubscribe(); + api.toggleStarred('k:ns/e2'); + + await Promise.resolve(); + expect(updates1).toEqual([ + new Set(), + new Set(['k:ns/e1']), + new Set(['k:ns/e1', 'k:ns/e2']), + new Set(['k:ns/e1']), + ]); + expect(updates2).toEqual([ + new Set(['k:ns/e1', 'k:ns/e2']), + new Set(['k:ns/e1']), + new Set(['k:ns/e1', 'k:ns/e2']), + ]); + sub2.unsubscribe(); + }); +}); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.ts new file mode 100644 index 0000000000..9467153dd9 --- /dev/null +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.ts @@ -0,0 +1,54 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Observable } from '@backstage/types'; +import ObservableImpl from 'zen-observable'; +import { StarredEntitiesApi } from './StarredEntitiesApi'; + +/** + * An in-memory mock implementation of the StarredEntitiesApi. + * + * @public + */ +export class MockStarredEntitiesApi implements StarredEntitiesApi { + private readonly starredEntities = new Set(); + private readonly subscribers = new Set< + ZenObservable.SubscriptionObserver> + >(); + + private readonly observable = new ObservableImpl>(subscriber => { + subscriber.next(new Set(this.starredEntities)); + + this.subscribers.add(subscriber); + return () => { + this.subscribers.delete(subscriber); + }; + }); + + async toggleStarred(entityRef: string): Promise { + if (!this.starredEntities.delete(entityRef)) { + this.starredEntities.add(entityRef); + } + + for (const subscription of this.subscribers) { + subscription.next(new Set(this.starredEntities)); + } + } + + starredEntitie$(): Observable> { + return this.observable; + } +} diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts index 94a221b36f..ba44c38e1d 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts @@ -16,3 +16,4 @@ export { starredEntitiesApiRef } from './StarredEntitiesApi'; export type { StarredEntitiesApi } from './StarredEntitiesApi'; +export { MockStarredEntitiesApi } from './MockStarredEntitiesApi'; diff --git a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx index 5db9e2be9c..27fde81cc7 100644 --- a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx +++ b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx @@ -23,14 +23,13 @@ import { identityApiRef, storageApiRef, } from '@backstage/core-plugin-api'; -import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; import { MockStorageApi, TestApiProvider } from '@backstage/test-utils'; import { act, renderHook } from '@testing-library/react-hooks'; import qs from 'qs'; import React, { PropsWithChildren } from 'react'; import { MemoryRouter } from 'react-router'; import { catalogApiRef } from '../api'; -import { starredEntitiesApiRef } from '../apis'; +import { starredEntitiesApiRef, MockStarredEntitiesApi } from '../apis'; import { EntityKindPicker, UserListPicker } from '../components'; import { EntityKindFilter, EntityTypeFilter, UserListFilter } from '../filters'; import { UserListFilterKind } from '../types'; @@ -96,12 +95,7 @@ const wrapper = ({ [catalogApiRef, mockCatalogApi], [identityApiRef, mockIdentityApi], [storageApiRef, MockStorageApi.create()], - [ - starredEntitiesApiRef, - new DefaultStarredEntitiesApi({ - storageApi: MockStorageApi.create(), - }), - ], + [starredEntitiesApiRef, new MockStarredEntitiesApi()], ]} > diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx b/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx index 949ac6515b..16fa19a1cd 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx +++ b/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx @@ -15,16 +15,18 @@ */ import { Entity } from '@backstage/catalog-model'; -import { StorageApi } from '@backstage/core-plugin-api'; -import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; -import { MockStorageApi, TestApiProvider } from '@backstage/test-utils'; +import { TestApiProvider } from '@backstage/test-utils'; import { act, renderHook } from '@testing-library/react-hooks'; import React, { PropsWithChildren } from 'react'; -import { starredEntitiesApiRef } from '../apis'; +import { + starredEntitiesApiRef, + StarredEntitiesApi, + MockStarredEntitiesApi, +} from '../apis'; import { useStarredEntities } from './useStarredEntities'; describe('useStarredEntities', () => { - let mockStorage: StorageApi; + let mockApi: StarredEntitiesApi; let wrapper: React.ComponentType; const mockEntity: Entity = { @@ -45,22 +47,15 @@ describe('useStarredEntities', () => { }; beforeEach(() => { - mockStorage = MockStorageApi.create(); + mockApi = new MockStarredEntitiesApi(); wrapper = ({ children }: PropsWithChildren<{}>) => ( - + {children} ); }); - it('should return an empty set for when there is no items in storage', async () => { + it('should return an empty set', async () => { const { result, waitForNextUpdate } = renderHook( () => useStarredEntities(), { wrapper }, @@ -71,10 +66,11 @@ describe('useStarredEntities', () => { expect(result.current.starredEntities.size).toBe(0); }); - it('should return a set with the current items when there are items in storage', async () => { + it('should return a set with the current items', async () => { const expectedIds = ['i', 'am', 'some', 'test', 'ids']; - const store = mockStorage?.forBucket('starredEntities'); - await store?.set('entityRefs', expectedIds); + for (const id of expectedIds) { + mockApi.toggleStarred(id); + } const { result, waitForNextUpdate } = renderHook( () => useStarredEntities(), diff --git a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx index e36c1e9338..cf2eedb14c 100644 --- a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx +++ b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx @@ -31,6 +31,7 @@ import { catalogApiRef, entityRouteRef, starredEntitiesApiRef, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import { mockBreakpoint, @@ -42,7 +43,6 @@ import { import DashboardIcon from '@material-ui/icons/Dashboard'; import { fireEvent, screen } from '@testing-library/react'; import React from 'react'; -import { DefaultStarredEntitiesApi } from '../../apis'; import { createComponentRouteRef } from '../../routes'; import { CatalogTableRow } from '../CatalogTable'; import { DefaultCatalogPage } from './DefaultCatalogPage'; @@ -141,10 +141,7 @@ describe('DefaultCatalogPage', () => { [catalogApiRef, catalogApi], [identityApiRef, identityApi], [storageApiRef, storageApi], - [ - starredEntitiesApiRef, - new DefaultStarredEntitiesApi({ storageApi }), - ], + [starredEntitiesApiRef, new MockStarredEntitiesApi()], ]} > {children} diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx index 1d49638eed..cc05bbe2ae 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx @@ -25,15 +25,11 @@ import { MockEntityListContextProvider, starredEntitiesApiRef, UserListFilter, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; -import { - MockStorageApi, - renderInTestApp, - TestApiRegistry, -} from '@backstage/test-utils'; +import { renderInTestApp, TestApiRegistry } from '@backstage/test-utils'; import { act, fireEvent } from '@testing-library/react'; import * as React from 'react'; -import { DefaultStarredEntitiesApi } from '../../apis'; import { CatalogTable } from './CatalogTable'; const entities: Entity[] = [ @@ -57,7 +53,7 @@ const entities: Entity[] = [ describe('CatalogTable component', () => { const mockApis = TestApiRegistry.from([ starredEntitiesApiRef, - new DefaultStarredEntitiesApi({ storageApi: MockStorageApi.create() }), + new MockStarredEntitiesApi(), ]); beforeEach(() => { diff --git a/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx b/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx index b29261c28b..c1deaf4b74 100644 --- a/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx +++ b/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx @@ -24,18 +24,17 @@ import { EntityProvider, entityRouteRef, starredEntitiesApiRef, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import { permissionApiRef } from '@backstage/plugin-permission-react'; import { MockPermissionApi, - MockStorageApi, renderInTestApp, TestApiRegistry, } from '@backstage/test-utils'; import { act, fireEvent } from '@testing-library/react'; import React from 'react'; import { Route, Routes } from 'react-router'; -import { DefaultStarredEntitiesApi } from '../../apis'; import { EntityLayout } from './EntityLayout'; const mockEntity = { @@ -48,10 +47,7 @@ const mockEntity = { const mockApis = TestApiRegistry.from( [catalogApiRef, {} as CatalogApi], [alertApiRef, {} as AlertApi], - [ - starredEntitiesApiRef, - new DefaultStarredEntitiesApi({ storageApi: MockStorageApi.create() }), - ], + [starredEntitiesApiRef, new MockStarredEntitiesApi()], [permissionApiRef, new MockPermissionApi()], ); diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx index ae4a84473f..9087ed3605 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx @@ -13,40 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - renderInTestApp, - TestApiProvider, - MockStorageApi, -} from '@backstage/test-utils'; +import { renderInTestApp, TestApiProvider } from '@backstage/test-utils'; import { starredEntitiesApiRef, + MockStarredEntitiesApi, entityRouteRef, } from '@backstage/plugin-catalog-react'; -import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; import React from 'react'; import { Content } from './Content'; describe('StarredEntitiesContent', () => { it('should render list of tools', async () => { - const mockStorageApi = MockStorageApi.create(); - await mockStorageApi - .forBucket('starredEntities') - .set('entityRefs', [ - 'component:default/mock-starred-entity', - 'component:default/mock-starred-entity-2', - ]); + const mockedApi = new MockStarredEntitiesApi(); + mockedApi.toggleStarred('component:default/mock-starred-entity'); + mockedApi.toggleStarred('component:default/mock-starred-entity-2'); const { getByText } = await renderInTestApp( - + , { diff --git a/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx b/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx index 55486a5dcd..315c0d0d16 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx @@ -22,9 +22,9 @@ import { } from '@backstage/test-utils'; import { starredEntitiesApiRef, + MockStarredEntitiesApi, entityRouteRef, } from '@backstage/plugin-catalog-react'; -import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; import { Grid } from '@material-ui/core'; import React, { ComponentType } from 'react'; @@ -44,14 +44,7 @@ export default { (Story: ComponentType<{}>) => wrapInTestApp( , diff --git a/plugins/home/src/templates/DefaultTemplate.stories.tsx b/plugins/home/src/templates/DefaultTemplate.stories.tsx index 86d1b60022..5dc4ff61a0 100644 --- a/plugins/home/src/templates/DefaultTemplate.stories.tsx +++ b/plugins/home/src/templates/DefaultTemplate.stories.tsx @@ -25,9 +25,9 @@ import { wrapInTestApp, TestApiProvider, MockStorageApi} from '@backstage/test-u import { Content, Page, InfoCard } from '@backstage/core-components'; import { starredEntitiesApiRef, + MockStarredEntitiesApi, entityRouteRef, } from '@backstage/plugin-catalog-react'; -import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; import { HomePageSearchBar, SearchContextProvider, @@ -57,9 +57,7 @@ export default { apis={[ [ starredEntitiesApiRef, - new DefaultStarredEntitiesApi({ - storageApi: mockStorageApi, - }), + new MockStarredEntitiesApi(), ], [searchApiRef, { query: () => Promise.resolve({ results: [] }) }], ]} diff --git a/plugins/scaffolder/dev/index.tsx b/plugins/scaffolder/dev/index.tsx index 6ecc1a4564..03632f434d 100644 --- a/plugins/scaffolder/dev/index.tsx +++ b/plugins/scaffolder/dev/index.tsx @@ -20,19 +20,13 @@ import { scmIntegrationsApiRef } from '@backstage/integration-react'; import { catalogApiRef, starredEntitiesApiRef, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import React from 'react'; import { scaffolderApiRef, ScaffolderClient } from '../src'; import { ScaffolderPage } from '../src/plugin'; -import { - discoveryApiRef, - fetchApiRef, - storageApiRef, -} from '@backstage/core-plugin-api'; -import { - CatalogEntityPage, - DefaultStarredEntitiesApi, -} from '@backstage/plugin-catalog'; +import { discoveryApiRef, fetchApiRef } from '@backstage/core-plugin-api'; +import { CatalogEntityPage } from '@backstage/plugin-catalog'; createDevApp() .addPage({ @@ -46,8 +40,8 @@ createDevApp() }) .registerApi({ api: starredEntitiesApiRef, - deps: { storageApi: storageApiRef }, - factory: ({ storageApi }) => new DefaultStarredEntitiesApi({ storageApi }), + deps: {}, + factory: () => new MockStarredEntitiesApi(), }) .registerApi({ api: scaffolderApiRef, diff --git a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx index 678c737547..33f5e0a08a 100644 --- a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx +++ b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx @@ -20,11 +20,11 @@ import { configApiRef, storageApiRef, } from '@backstage/core-plugin-api'; -import { DefaultStarredEntitiesApi } from '@backstage/plugin-catalog'; import { CatalogApi, catalogApiRef, starredEntitiesApiRef, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import { MockStorageApi, @@ -73,7 +73,7 @@ describe('TechDocs Home', () => { [catalogApiRef, mockCatalogApi], [configApiRef, configApi], [storageApiRef, storageApi], - [starredEntitiesApiRef, new DefaultStarredEntitiesApi({ storageApi })], + [starredEntitiesApiRef, new MockStarredEntitiesApi()], ); it('should render a TechDocs home page', async () => { From 0c9cf2822d77e27e3e2967b3e5a71d93b5bee3dd Mon Sep 17 00:00:00 2001 From: Mike Lewis Date: Mon, 28 Feb 2022 18:31:30 +0000 Subject: [PATCH 028/150] catalog-backend: mark permission-related exports as alpha Marks all strictly permission-related exports in catalog-backend as alpha. There are permission-related properties on other exports, so this doesn't entirely protect us from future breaking changes, but it will certainly reduce the number. Signed-off-by: Mike Lewis --- .changeset/chilled-dolls-agree.md | 10 ++++++++++ plugins/catalog-backend/api-report.md | 8 ++++---- .../src/permissions/conditionExports.ts | 6 ++++-- plugins/catalog-backend/src/permissions/rules/index.ts | 3 ++- plugins/catalog-backend/src/permissions/rules/util.ts | 2 +- 5 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 .changeset/chilled-dolls-agree.md diff --git a/.changeset/chilled-dolls-agree.md b/.changeset/chilled-dolls-agree.md new file mode 100644 index 0000000000..0468bdfee0 --- /dev/null +++ b/.changeset/chilled-dolls-agree.md @@ -0,0 +1,10 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Mark permission-related exports as alpha. This means that the exports below should now be imported from `@backstage/plugin-catalog-backend/alpha` instead of `@backstage/plugin-catalog-backend`. + +- `catalogConditions` +- `createCatalogPolicyDecision` +- `permissionRules` +- `createCatalogPermissionRule` diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 9b4db418b3..5ae8a76d19 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -213,7 +213,7 @@ export class CatalogBuilder { setRefreshIntervalSeconds(seconds: number): CatalogBuilder; } -// @public +// @alpha export const catalogConditions: Conditions<{ hasAnnotation: PermissionRule< Entity, @@ -394,12 +394,12 @@ export class CodeOwnersProcessor implements CatalogProcessor { preProcessEntity(entity: Entity, location: LocationSpec): Promise; } -// @public +// @alpha export const createCatalogPermissionRule: ( rule: PermissionRule, ) => PermissionRule; -// @public +// @alpha export const createCatalogPolicyDecision: ( conditions: PermissionCriteria>, ) => ConditionalPolicyDecision; @@ -896,7 +896,7 @@ export function parseEntityYaml( location: LocationSpec, ): Iterable; -// @public +// @alpha export const permissionRules: { hasAnnotation: PermissionRule< Entity, diff --git a/plugins/catalog-backend/src/permissions/conditionExports.ts b/plugins/catalog-backend/src/permissions/conditionExports.ts index 28e74c53c2..2846c8c3ec 100644 --- a/plugins/catalog-backend/src/permissions/conditionExports.ts +++ b/plugins/catalog-backend/src/permissions/conditionExports.ts @@ -27,7 +27,8 @@ const conditionExports = createConditionExports({ /** * These conditions are used when creating conditional decisions that are returned * by authorization policies. - * @public + * + * @alpha */ export const catalogConditions = conditionExports.conditions; @@ -50,7 +51,8 @@ export const catalogConditions = conditionExports.conditions; * } * } * ``` - * @public + * + * @alpha */ export const createCatalogPolicyDecision = conditionExports.createPolicyDecision; diff --git a/plugins/catalog-backend/src/permissions/rules/index.ts b/plugins/catalog-backend/src/permissions/rules/index.ts index 4eec796c74..f9d192781e 100644 --- a/plugins/catalog-backend/src/permissions/rules/index.ts +++ b/plugins/catalog-backend/src/permissions/rules/index.ts @@ -24,7 +24,8 @@ import { hasSpec } from './hasSpec'; /** * These permission rules can be used to conditionally filter catalog entities * or describe a user's access to the entities. - * @public + * + * @alpha */ export const permissionRules = { hasAnnotation, diff --git a/plugins/catalog-backend/src/permissions/rules/util.ts b/plugins/catalog-backend/src/permissions/rules/util.ts index 7ef0ca3546..a1316c7fb2 100644 --- a/plugins/catalog-backend/src/permissions/rules/util.ts +++ b/plugins/catalog-backend/src/permissions/rules/util.ts @@ -23,7 +23,7 @@ import { EntitiesSearchFilter } from '../../catalog/types'; * {@link @backstage/plugin-permission-node#PermissionRule}s for the * catalog-backend. * - * @public + * @alpha */ export const createCatalogPermissionRule = makeCreatePermissionRule< Entity, From abc91a6843ea0f4807644694625efbc75269d2f5 Mon Sep 17 00:00:00 2001 From: Marcus Crane Date: Tue, 1 Mar 2022 09:13:12 +1300 Subject: [PATCH 029/150] Update Postgres tutorial SSL block to match Postgres app config template Signed-off-by: Marcus Crane --- docs/tutorials/switching-sqlite-postgres.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/switching-sqlite-postgres.md b/docs/tutorials/switching-sqlite-postgres.md index fafd2ad586..8a7f90b21f 100644 --- a/docs/tutorials/switching-sqlite-postgres.md +++ b/docs/tutorials/switching-sqlite-postgres.md @@ -43,9 +43,11 @@ backend: + user: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} + # https://node-postgres.com/features/ssl -+ #ssl: require # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require) -+ #ca: # if you have a CA file and want to verify it you can uncomment this section -+ #$file: /ca/server.crt ++ # you can set the sslmode configuration option via the `PGSSLMODE` environment variable ++ # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require) ++ # ssl: ++ # ca: # if you have a CA file and want to verify it you can uncomment this section ++ # $file: /ca/server.crt ``` If you have an `app-config.local.yaml` for local development, a similar update From 55031596f23c5ef65462594f4f23fc4755924c57 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Tue, 1 Mar 2022 09:05:59 +1100 Subject: [PATCH 030/150] Prettify how to Signed-off-by: Nikolas Skoufis --- docs/features/techdocs/how-to-guides.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/features/techdocs/how-to-guides.md b/docs/features/techdocs/how-to-guides.md index 5ec8aac307..5a5fe36526 100644 --- a/docs/features/techdocs/how-to-guides.md +++ b/docs/features/techdocs/how-to-guides.md @@ -566,6 +566,7 @@ To achieve this hybrid build model: a given entity. For example, to only build docs when an entity has the `company.com/techdocs-builder` annotation set to `'local'`: + ```typescript export class AnnotationBasedBuildStrategy { private readonly config: Config; @@ -575,10 +576,14 @@ To achieve this hybrid build model: } async shouldBuild(_: Entity): Promise { - return this.entity.metadata?.annotations?.["company.com/techdocs-builder"] === 'local' + return ( + this.entity.metadata?.annotations?.['company.com/techdocs-builder'] === + 'local' + ); } } ``` + 4. Pass an instance of this Build Strategy as the `docsBuildStrategy` parameter of the TechDocs backend `createRouter` method. From 588445b175d70a40810fbaa5724226254d833bf7 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Tue, 1 Mar 2022 09:18:44 +1100 Subject: [PATCH 031/150] Exporting interface and adding docs This makes the api reporter happy, with good reason Signed-off-by: Nikolas Skoufis --- plugins/techdocs-backend/api-report.md | 10 ++++++---- plugins/techdocs-backend/src/index.ts | 1 + .../techdocs-backend/src/service/DocsBuildStrategy.ts | 5 +++++ plugins/techdocs-backend/src/service/index.ts | 1 + 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index 28700b245b..3ce3a2504b 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -42,6 +42,12 @@ export class DefaultTechDocsCollator implements DocumentCollator { readonly visibilityPermission: Permission; } +// @public +export interface DocsBuildStrategy { + // (undocumented) + shouldBuild(entity: Entity): Promise; +} + // @public export type OutOfTheBoxDeploymentOptions = { preparers: PreparerBuilder; @@ -84,8 +90,4 @@ export type TechDocsCollatorOptions = { export { TechDocsDocument }; export * from '@backstage/techdocs-common'; - -// Warnings were encountered during analysis: -// -// src/service/router.d.ts:24:5 - (ae-forgotten-export) The symbol "DocsBuildStrategy" needs to be exported by the entry point index.d.ts ``` diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts index 12e7df1e9e..2a17bf2736 100644 --- a/plugins/techdocs-backend/src/index.ts +++ b/plugins/techdocs-backend/src/index.ts @@ -25,6 +25,7 @@ export type { RouterOptions, RecommendedDeploymentOptions, OutOfTheBoxDeploymentOptions, + DocsBuildStrategy, } from './service'; export { DefaultTechDocsCollator } from './search'; diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts index 69b0474a85..2e8cc52d3e 100644 --- a/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts @@ -16,6 +16,11 @@ import { Entity } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; +/** + * A strategy for when to build TechDocs locally, and when to skip building TechDocs (allowing for an external build) + * + * @public + */ export interface DocsBuildStrategy { shouldBuild(entity: Entity): Promise; } diff --git a/plugins/techdocs-backend/src/service/index.ts b/plugins/techdocs-backend/src/service/index.ts index 0065e33a2a..29db2d2038 100644 --- a/plugins/techdocs-backend/src/service/index.ts +++ b/plugins/techdocs-backend/src/service/index.ts @@ -20,3 +20,4 @@ export type { RecommendedDeploymentOptions, OutOfTheBoxDeploymentOptions, } from './router'; +export type { DocsBuildStrategy } from './DocsBuildStrategy'; From 7a22a78180fffc4a1c82f27367344ffc10c6c800 Mon Sep 17 00:00:00 2001 From: Nik Skoufis Date: Tue, 1 Mar 2022 09:05:09 +1100 Subject: [PATCH 032/150] Apply suggestions from code review Co-authored-by: Emma Indal Signed-off-by: Nikolas Skoufis --- plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts index 71e5377a03..029d8d223f 100644 --- a/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 5adcf26b0513d6958d736ab4117c810415093772 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Tue, 1 Mar 2022 09:23:47 +1100 Subject: [PATCH 033/150] Refactor to a private constructor and static fromConfig Signed-off-by: Nikolas Skoufis --- .../techdocs-backend/src/service/DocsBuildStrategy.test.ts | 6 ++++-- plugins/techdocs-backend/src/service/DocsBuildStrategy.ts | 6 +++++- plugins/techdocs-backend/src/service/router.ts | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts index 029d8d223f..76b70e67e9 100644 --- a/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts @@ -41,7 +41,8 @@ describe('DefaultDocsBuildStrategy', () => { describe('shouldBuild', () => { it('should return true when techdocs.build is set to local', async () => { - const defaultDocsBuildStrategy = new DefaultDocsBuildStrategy(config); + const defaultDocsBuildStrategy = + DefaultDocsBuildStrategy.fromConfig(config); MockedConfigReader.prototype.getString.mockReturnValue('local'); @@ -51,7 +52,8 @@ describe('DefaultDocsBuildStrategy', () => { }); it('should return false when techdocs.build is set to external', async () => { - const defaultDocsBuildStrategy = new DefaultDocsBuildStrategy(config); + const defaultDocsBuildStrategy = + DefaultDocsBuildStrategy.fromConfig(config); MockedConfigReader.prototype.getString.mockReturnValue('external'); diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts index 2e8cc52d3e..e0220d7a32 100644 --- a/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts @@ -28,10 +28,14 @@ export interface DocsBuildStrategy { export class DefaultDocsBuildStrategy { private readonly config: Config; - constructor(config: Config) { + private constructor(config: Config) { this.config = config; } + static fromConfig(config: Config): DefaultDocsBuildStrategy { + return new DefaultDocsBuildStrategy(config); + } + async shouldBuild(_: Entity): Promise { return this.config.getString('techdocs.builder') === 'local'; } diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index d620020245..4d51ebd485 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -106,7 +106,7 @@ export async function createRouter( const { publisher, config, logger, discovery } = options; const catalogClient = new CatalogClient({ discoveryApi: discovery }); const docsBuildStrategy = - options.docsBuildStrategy ?? new DefaultDocsBuildStrategy(config); + options.docsBuildStrategy ?? DefaultDocsBuildStrategy.fromConfig(config); // Entities are cached to optimize the /static/docs request path, which can be called many times // when loading a single techdocs page. From 40ef5d33d3cf9d05e7bb95d4aa750dedb07721ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Mar 2022 04:10:40 +0000 Subject: [PATCH 034/150] chore(deps): bump @types/react-sparklines from 1.7.0 to 1.7.2 Bumps [@types/react-sparklines](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/react-sparklines) from 1.7.0 to 1.7.2. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/react-sparklines) --- updated-dependencies: - dependency-name: "@types/react-sparklines" dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index a6b27ffa82..ab169bb33a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6236,9 +6236,9 @@ redux "^4.0.0" "@types/react-sparklines@^1.7.0": - version "1.7.0" - resolved "https://registry.npmjs.org/@types/react-sparklines/-/react-sparklines-1.7.0.tgz#f956d0f7b0e746ad445ce1cd250fe81f8a384684" - integrity sha512-Vd+cME7+Yy3kFNhnid9EBIKiyCQ/at8nqDczIs0UYfIB8AtaRJPqekigv02biOsIbQCvxyvIAIjiTKOC+hHNbA== + version "1.7.2" + resolved "https://registry.npmjs.org/@types/react-sparklines/-/react-sparklines-1.7.2.tgz#c14e80623abd3669a10f18d13f6fb9fbdc322f70" + integrity sha512-N1GwO7Ri5C5fE8+CxhiDntuSw1qYdGytBuedKrCxWpaojXm4WnfygbdBdc5sXGX7feMxDXBy9MNhxoUTwrMl4A== dependencies: "@types/react" "*" From 9b4b73719e09d22121e8ec5f1d923cc05684afc1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Mar 2022 04:11:05 +0000 Subject: [PATCH 035/150] chore(deps): bump postcss from 8.4.6 to 8.4.7 Bumps [postcss](https://github.com/postcss/postcss) from 8.4.6 to 8.4.7. - [Release notes](https://github.com/postcss/postcss/releases) - [Changelog](https://github.com/postcss/postcss/blob/main/CHANGELOG.md) - [Commits](https://github.com/postcss/postcss/compare/8.4.6...8.4.7) --- updated-dependencies: - dependency-name: postcss dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/yarn.lock b/yarn.lock index a6b27ffa82..2e31950dd5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18139,15 +18139,10 @@ nanoclone@^0.2.1: resolved "https://registry.npmjs.org/nanoclone/-/nanoclone-0.2.1.tgz#dd4090f8f1a110d26bb32c49ed2f5b9235209ed4" integrity sha512-wynEP02LmIbLpcYw8uBKpcfF6dmg2vcpKqxeH5UcoKEYdExslsdUA4ugFauuaeYdTB76ez6gJW8XAZ6CgkXYxA== -nanoid@^3.1.23: - version "3.2.0" - resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.2.0.tgz#62667522da6673971cca916a6d3eff3f415ff80c" - integrity sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA== - -nanoid@^3.2.0: - version "3.3.0" - resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.0.tgz#5906f776fd886c66c24f3653e0c46fcb1d4ad6b0" - integrity sha512-JzxqqT5u/x+/KOFSd7JP15DOo9nOoHpx6DYatqIHUW2+flybkm+mdcraotSQR5WcnZr+qhGVh8Ted0KdfSMxlg== +nanoid@^3.1.23, nanoid@^3.3.1: + version "3.3.1" + resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz#6347a18cac88af88f58af0b3594b723d5e99bb35" + integrity sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw== nanomatch@^1.2.9: version "1.2.13" @@ -20142,11 +20137,11 @@ postcss-value-parser@^4.0.2, postcss-value-parser@^4.1.0, postcss-value-parser@^ integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ== postcss@^8.1.0, postcss@^8.4.5: - version "8.4.6" - resolved "https://registry.npmjs.org/postcss/-/postcss-8.4.6.tgz#c5ff3c3c457a23864f32cb45ac9b741498a09ae1" - integrity sha512-OovjwIzs9Te46vlEx7+uXB0PLijpwjXGKXjVGGPIGubGpq7uh5Xgf6D6FiJ/SzJMBosHDp6a2hiXOS97iBXcaA== + version "8.4.7" + resolved "https://registry.npmjs.org/postcss/-/postcss-8.4.7.tgz#f99862069ec4541de386bf57f5660a6c7a0875a8" + integrity sha512-L9Ye3r6hkkCeOETQX6iOaWZgjp3LL6Lpqm6EtgbKrgqGGteRMNb9vzBfRL96YOSu8o7x3MfIH9Mo5cPJFGrW6A== dependencies: - nanoid "^3.2.0" + nanoid "^3.3.1" picocolors "^1.0.0" source-map-js "^1.0.2" From ea29f9279892bc54c044ea065085b544c0d1c921 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Mar 2022 04:11:36 +0000 Subject: [PATCH 036/150] chore(deps): bump apollo-server from 3.6.1 to 3.6.3 Bumps [apollo-server](https://github.com/apollographql/apollo-server/tree/HEAD/packages/apollo-server) from 3.6.1 to 3.6.3. - [Release notes](https://github.com/apollographql/apollo-server/releases) - [Changelog](https://github.com/apollographql/apollo-server/blob/main/CHANGELOG.md) - [Commits](https://github.com/apollographql/apollo-server/commits/apollo-server@3.6.3/packages/apollo-server) --- updated-dependencies: - dependency-name: apollo-server dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/yarn.lock b/yarn.lock index a6b27ffa82..df8c58bcab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7284,7 +7284,7 @@ apollo-server-caching@^3.3.0: dependencies: lru-cache "^6.0.0" -apollo-server-core@^3.6.1, apollo-server-core@^3.6.3: +apollo-server-core@^3.6.3: version "3.6.3" resolved "https://registry.npmjs.org/apollo-server-core/-/apollo-server-core-3.6.3.tgz#6b12ffa1af8bc8799930f72360090834915033d1" integrity sha512-TFJmAlI6vPp1MHOSXqYkE6leAyMekWv/D/3ma11uETkcd3EPjERGmxtTXPJElMVEkOK9BEElYKthCrH7bjYLuw== @@ -7322,7 +7322,7 @@ apollo-server-errors@^3.3.1: resolved "https://registry.npmjs.org/apollo-server-errors/-/apollo-server-errors-3.3.1.tgz#ba5c00cdaa33d4cbd09779f8cb6f47475d1cd655" integrity sha512-xnZJ5QWs6FixHICXHxUfm+ZWqqxrNuPlQ+kj5m6RtEgIpekOPssH/SD9gf2B4HuWV0QozorrygwZnux8POvyPA== -apollo-server-express@^3.0.0, apollo-server-express@^3.6.1: +apollo-server-express@^3.0.0, apollo-server-express@^3.6.3: version "3.6.3" resolved "https://registry.npmjs.org/apollo-server-express/-/apollo-server-express-3.6.3.tgz#5daf58bf0bdf0107ded7cd52c7e6ce6cd32c8b44" integrity sha512-3CjahZ+n+1T7pHH1qW1B6Ns0BzwOMeupAp2u0+M8ruOmE/e7VKn0OSOQQckZ8Z2AcWxWeno9K89fIv3PoSYgYA== @@ -7356,12 +7356,12 @@ apollo-server-types@^3.5.1: apollo-server-env "^4.2.1" apollo-server@^3.0.0: - version "3.6.1" - resolved "https://registry.npmjs.org/apollo-server/-/apollo-server-3.6.1.tgz#29420b1c0cddbf2e18147a3ca7299485f17137a2" - integrity sha512-Y2MY2/WvaTiofVoIR5ZIYt6c6wX8klZRaXI9x+7JBiFV9HMcOuLLpU3+P4r2EVXuN1LLe82m1PgiAYr+a1OmQg== + version "3.6.3" + resolved "https://registry.npmjs.org/apollo-server/-/apollo-server-3.6.3.tgz#0ba0ddb2835ccf27056d20b6f5b83b0ce9545a79" + integrity sha512-kNvOiDNkIaO+MsfR9v40Vz4ArlDdc9VwVKGJy5dniLW9AoDa/tSF99m8ItfGoMypqlRPMgrNGxkMuToBnvYXNQ== dependencies: - apollo-server-core "^3.6.1" - apollo-server-express "^3.6.1" + apollo-server-core "^3.6.3" + apollo-server-express "^3.6.3" express "^4.17.1" aproba@^1.0.3: From ff0a16fb1ad5bb072c96e4e83f1e2859d611faad Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Tue, 15 Feb 2022 17:01:46 +0100 Subject: [PATCH 037/150] Add techdocs-core plugin to techdocs builder automatically Currently users need to modify their mkdocs files to container techdocs-core plugin manually when they want to expose techdocs in Backstage. To have a standardized set of extensions without the need to modify existing files (and possibly pollute alternative/additional mkdocs pipelines) this should be added automatically. This PR adds a helper function to modify the mkdocs file to contain this plugin if it doesn't exist in the config file. Co-authored-by: @emmaindal Signed-off-by: Jussi Hallila --- .changeset/tiny-jobs-hunt.md | 5 + docs/features/techdocs/cli.md | 2 + docs/features/techdocs/configuration.md | 5 + .../techdocs/creating-and-publishing.md | 3 + .../src/commands/generate/generate.ts | 4 + packages/techdocs-cli/src/commands/index.ts | 5 + .../mkdocs_with_additional_plugins.yml | 6 + .../mkdocs_with_techdocs_plugin.yml | 5 + .../__fixtures__/mkdocs_without_plugins.yml | 3 + .../src/stages/generate/helpers.test.ts | 69 +++++++- .../src/stages/generate/helpers.ts | 97 +--------- .../src/stages/generate/mkDocsPatchers.ts | 166 ++++++++++++++++++ .../src/stages/generate/techdocs.ts | 13 +- .../src/stages/generate/types.ts | 1 + 14 files changed, 286 insertions(+), 98 deletions(-) create mode 100644 .changeset/tiny-jobs-hunt.md create mode 100644 packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_additional_plugins.yml create mode 100644 packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_techdocs_plugin.yml create mode 100644 packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_without_plugins.yml create mode 100644 packages/techdocs-common/src/stages/generate/mkDocsPatchers.ts diff --git a/.changeset/tiny-jobs-hunt.md b/.changeset/tiny-jobs-hunt.md new file mode 100644 index 0000000000..8cf9db20b8 --- /dev/null +++ b/.changeset/tiny-jobs-hunt.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Modify techdocs builder to automatically append techdocs-core plugin to mkdocs.yaml file if it is missing. Adds an optional configuration item if this plugin needs to be omitted. diff --git a/docs/features/techdocs/cli.md b/docs/features/techdocs/cli.md index a5dc8c3c54..2a11e1181a 100644 --- a/docs/features/techdocs/cli.md +++ b/docs/features/techdocs/cli.md @@ -130,6 +130,8 @@ Options: if not found. --etag A unique identifier for the prepared tree e.g. commit SHA. If provided it will be stored in techdocs_metadata.json. + --omitTechdocsCoreMkdocsPlugin An option to disable automatic addition of techdocs-core plugin to the mkdocs.yaml files. + Defaults to false, which means that the techdocs-core plugin is always added to the mkdocs file. -v --verbose Enable verbose output. (default: false) -h, --help display help for command ``` diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 6317ae365b..129aa8957e 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -37,6 +37,11 @@ techdocs: pullImage: true + mkdocs: + # (Optional) techdocs.generator.omitTechdocsCoreMkdocsPlugin can be used to disable automatic addition of techdocs-core plugin to the mkdocs.yaml files. + # Defaults to false, which means that the techdocs-core plugin is always added to the mkdocs file. + omitTechdocsCorePlugin: false + # techdocs.builder can be either 'local' or 'external. # If builder is set to 'local' and you open a TechDocs page, techdocs-backend will try to generate the docs, publish to storage # and show the generated docs afterwords. This is the "Basic" setup of the TechDocs Architecture. diff --git a/docs/features/techdocs/creating-and-publishing.md b/docs/features/techdocs/creating-and-publishing.md index 383c3ae6d2..f63ff43b7f 100644 --- a/docs/features/techdocs/creating-and-publishing.md +++ b/docs/features/techdocs/creating-and-publishing.md @@ -79,6 +79,9 @@ plugins: - techdocs-core ``` +> Note - The plugins section above is optional. Backstage automatically adds the `techdocs-core` plugin to the +> mkdocs file if it is missing. This functionality can be turned off with a [configuration option](./configuration.md) in Backstage. + Update your component's entity description by adding the following lines to its `catalog-info.yaml` in the root of its repository: diff --git a/packages/techdocs-cli/src/commands/generate/generate.ts b/packages/techdocs-cli/src/commands/generate/generate.ts index a255805139..0f5213df34 100644 --- a/packages/techdocs-cli/src/commands/generate/generate.ts +++ b/packages/techdocs-cli/src/commands/generate/generate.ts @@ -39,6 +39,7 @@ export default async function generate(cmd: Command) { const sourceDir = resolve(cmd.sourceDir); const outputDir = resolve(cmd.outputDir); + const omitTechdocsCorePlugin = cmd.omitTechdocsCoreMkdocsPlugin; const dockerImage = cmd.dockerImage; const pullImage = cmd.pull; @@ -55,6 +56,9 @@ export default async function generate(cmd: Command) { runIn: cmd.docker ? 'docker' : 'local', dockerImage, pullImage, + mkdocs: { + omitTechdocsCorePlugin, + }, }, }, }); diff --git a/packages/techdocs-cli/src/commands/index.ts b/packages/techdocs-cli/src/commands/index.ts index 115fe8e364..6f1d9847e1 100644 --- a/packages/techdocs-cli/src/commands/index.ts +++ b/packages/techdocs-cli/src/commands/index.ts @@ -54,6 +54,11 @@ export function registerCommands(program: CommanderStatic) { 'A unique identifier for the prepared tree e.g. commit SHA. If provided it will be stored in techdocs_metadata.json.', ) .option('-v --verbose', 'Enable verbose output.', false) + .option( + '--omitTechdocsCoreMkdocsPlugin', + "Don't patch MkDocs file automatically with techdocs-core plugin.", + false, + ) .alias('build') .action(lazy(() => import('./generate/generate').then(m => m.default))); diff --git a/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_additional_plugins.yml b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_additional_plugins.yml new file mode 100644 index 0000000000..09e8fd7ac7 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_additional_plugins.yml @@ -0,0 +1,6 @@ +site_name: Test site name +site_description: Test site description +docs_dir: docs/ +plugins: + - not-techdocs-core + - also-not-techdocs-core diff --git a/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_techdocs_plugin.yml b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_techdocs_plugin.yml new file mode 100644 index 0000000000..eea9a8a3d9 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_techdocs_plugin.yml @@ -0,0 +1,5 @@ +site_name: Test site name +site_description: Test site description +# This is a comment that is removed after editing +plugins: + - techdocs-core diff --git a/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_without_plugins.yml b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_without_plugins.yml new file mode 100644 index 0000000000..e75b06ada7 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_without_plugins.yml @@ -0,0 +1,3 @@ +site_name: Test site name +site_description: Test site description +docs_dir: docs/ diff --git a/packages/techdocs-common/src/stages/generate/helpers.test.ts b/packages/techdocs-common/src/stages/generate/helpers.test.ts index 9323ab4f18..7590df94fb 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.test.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.test.ts @@ -28,10 +28,14 @@ import { getMkdocsYml, getRepoUrlFromLocationAnnotation, patchIndexPreBuild, - patchMkdocsYmlPreBuild, storeEtagMetadata, validateMkdocsYaml, } from './helpers'; +import { + patchMkdocsYmlPreBuild, + pathMkdocsYmlWithTechdocsPlugin, +} from './mkDocsPatchers'; +import yaml from 'js-yaml'; const mockEntity = { apiVersion: 'version', @@ -65,6 +69,15 @@ const mkdocsYmlWithInvalidDocDir2 = fs.readFileSync( const mkdocsYmlWithComments = fs.readFileSync( resolvePath(__filename, '../__fixtures__/mkdocs_with_comments.yml'), ); +const mkdocsYmlWithTechdocsPlugins = fs.readFileSync( + resolvePath(__filename, '../__fixtures__/mkdocs_with_techdocs_plugin.yml'), +); +const mkdocsYmlWithoutPlugins = fs.readFileSync( + resolvePath(__filename, '../__fixtures__/mkdocs_without_plugins.yml'), +); +const mkdocsYmlWithAdditionalPlugins = fs.readFileSync( + resolvePath(__filename, '../__fixtures__/mkdocs_with_additional_plugins.yml'), +); const mockLogger = getVoidLogger(); const warn = jest.spyOn(mockLogger, 'warn'); @@ -289,6 +302,60 @@ describe('helpers', () => { }); }); + describe('pathMkdocsYmlWithTechdocsPlugin', () => { + beforeEach(() => { + mockFs({ + '/mkdocs_with_techdocs_plugin.yml': mkdocsYmlWithTechdocsPlugins, + '/mkdocs_without_plugins.yml': mkdocsYmlWithoutPlugins, + '/mkdocs_with_additional_plugins.yml': mkdocsYmlWithAdditionalPlugins, + }); + }); + it('should not add additional plugins if techdocs exists already in mkdocs file', async () => { + await pathMkdocsYmlWithTechdocsPlugin( + '/mkdocs_with_techdocs_plugin.yml', + mockLogger, + ); + + const updatedMkdocsYml = await fs.readFile( + '/mkdocs_with_techdocs_plugin.yml', + ); + const parsedYml = yaml.load(updatedMkdocsYml.toString()) as { + plugins: string[]; + }; + expect(parsedYml.plugins).toHaveLength(1); + expect(parsedYml.plugins).toContain('techdocs-core'); + }); + it("should add the needed plugin if it doesn't exist in mkdocs file", async () => { + await pathMkdocsYmlWithTechdocsPlugin( + '/mkdocs_without_plugins.yml', + mockLogger, + ); + + const updatedMkdocsYml = await fs.readFile('/mkdocs_without_plugins.yml'); + const parsedYml = yaml.load(updatedMkdocsYml.toString()) as { + plugins: string[]; + }; + expect(parsedYml.plugins).toHaveLength(1); + expect(parsedYml.plugins).toContain('techdocs-core'); + }); + it('should not override existing plugins', async () => { + await pathMkdocsYmlWithTechdocsPlugin( + '/mkdocs_with_additional_plugins.yml', + mockLogger, + ); + const updatedMkdocsYml = await fs.readFile( + '/mkdocs_with_additional_plugins.yml', + ); + const parsedYml = yaml.load(updatedMkdocsYml.toString()) as { + plugins: string[]; + }; + expect(parsedYml.plugins).toHaveLength(3); + expect(parsedYml.plugins).toContain('techdocs-core'); + expect(parsedYml.plugins).toContain('not-techdocs-core'); + expect(parsedYml.plugins).toContain('also-not-techdocs-core'); + }); + }); + describe('patchIndexPreBuild', () => { afterEach(() => { warn.mockClear(); diff --git a/packages/techdocs-common/src/stages/generate/helpers.ts b/packages/techdocs-common/src/stages/generate/helpers.ts index 2debd5822c..349783d2bd 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.ts @@ -125,7 +125,7 @@ class UnknownTag { constructor(public readonly data: any, public readonly type?: string) {} } -const MKDOCS_SCHEMA = DEFAULT_SCHEMA.extend([ +export const MKDOCS_SCHEMA = DEFAULT_SCHEMA.extend([ new Type('', { kind: 'scalar', multi: true, @@ -203,101 +203,6 @@ export const validateMkdocsYaml = async ( return parsedMkdocsYml.docs_dir; }; -/** - * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site. - * - * List of tasks: - * - Add repo_url or edit_uri if it does not exists - * If mkdocs.yml has a repo_url, the generated docs site gets an Edit button on the pages by default. - * If repo_url is missing in mkdocs.yml, we will use techdocs annotation of the entity to possibly get - * the repository URL. - * - * This function will not throw an error since this is not critical to the whole TechDocs pipeline. - * Instead it will log warnings if there are any errors in reading, parsing or writing YAML. - * - * @param mkdocsYmlPath - Absolute path to mkdocs.yml or equivalent of a docs site - * @param logger - A logger instance - * @param parsedLocationAnnotation - Object with location url and type - * @param scmIntegrations - the scmIntegration to do url transformations - */ -export const patchMkdocsYmlPreBuild = async ( - mkdocsYmlPath: string, - logger: Logger, - parsedLocationAnnotation: ParsedLocationAnnotation, - scmIntegrations: ScmIntegrationRegistry, -) => { - // We only want to override the mkdocs.yml if it has actually changed. This is relevant if - // used with a 'dir' location on the file system as this would permanently update the file. - let didEdit = false; - - let mkdocsYmlFileString; - try { - mkdocsYmlFileString = await fs.readFile(mkdocsYmlPath, 'utf8'); - } catch (error) { - assertError(error); - logger.warn( - `Could not read MkDocs YAML config file ${mkdocsYmlPath} before running the generator: ${error.message}`, - ); - return; - } - - let mkdocsYml: any; - try { - mkdocsYml = yaml.load(mkdocsYmlFileString, { schema: MKDOCS_SCHEMA }); - - // mkdocsYml should be an object type after successful parsing. - // But based on its type definition, it can also be a string or undefined, which we don't want. - if (typeof mkdocsYml === 'string' || typeof mkdocsYml === 'undefined') { - throw new Error('Bad YAML format.'); - } - } catch (error) { - assertError(error); - logger.warn( - `Error in parsing YAML at ${mkdocsYmlPath} before running the generator. ${error.message}`, - ); - return; - } - - // Add edit_uri and/or repo_url to mkdocs.yml if it is missing. - // This will enable the Page edit button generated by MkDocs. - // If the either has been set, keep the original value - if (!('repo_url' in mkdocsYml) && !('edit_uri' in mkdocsYml)) { - const result = getRepoUrlFromLocationAnnotation( - parsedLocationAnnotation, - scmIntegrations, - mkdocsYml.docs_dir, - ); - - if (result.repo_url || result.edit_uri) { - mkdocsYml.repo_url = result.repo_url; - mkdocsYml.edit_uri = result.edit_uri; - didEdit = true; - - logger.info( - `Set ${JSON.stringify( - result, - )}. You can disable this feature by manually setting 'repo_url' or 'edit_uri' according to the MkDocs documentation at https://www.mkdocs.org/user-guide/configuration/#repo_url`, - ); - } - } - - try { - if (didEdit) { - await fs.writeFile( - mkdocsYmlPath, - yaml.dump(mkdocsYml, { schema: MKDOCS_SCHEMA }), - 'utf8', - ); - } - } catch (error) { - assertError(error); - logger.warn( - `Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`, - ); - return; - } -}; - /** * Update docs/index.md file before TechDocs generator uses it to generate docs site, * falling back to docs/README.md or README.md in case a default docs/index.md diff --git a/packages/techdocs-common/src/stages/generate/mkDocsPatchers.ts b/packages/techdocs-common/src/stages/generate/mkDocsPatchers.ts new file mode 100644 index 0000000000..d03b5d83c2 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/mkDocsPatchers.ts @@ -0,0 +1,166 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Logger } from 'winston'; +import fs from 'fs-extra'; +import yaml from 'js-yaml'; +import { ParsedLocationAnnotation } from '../../helpers'; +import { getRepoUrlFromLocationAnnotation, MKDOCS_SCHEMA } from './helpers'; +import { assertError } from '@backstage/errors'; +import { ScmIntegrationRegistry } from '@backstage/integration'; + +type MkDocsObject = { + plugins?: string[]; + docs_dir: string; + repo_url?: string; + edit_uri?: string; +}; + +const patchMkdocsFile = async ( + mkdocsYmlPath: string, + logger: Logger, + updateAction: (mkdocsYml: MkDocsObject) => boolean, +) => { + // We only want to override the mkdocs.yml if it has actually changed. This is relevant if + // used with a 'dir' location on the file system as this would permanently update the file. + let didEdit = false; + + let mkdocsYmlFileString; + try { + mkdocsYmlFileString = await fs.readFile(mkdocsYmlPath, 'utf8'); + } catch (error) { + assertError(error); + logger.warn( + `Could not read MkDocs YAML config file ${mkdocsYmlPath} before running the generator: ${error.message}`, + ); + return; + } + + let mkdocsYml: any; + try { + mkdocsYml = yaml.load(mkdocsYmlFileString, { schema: MKDOCS_SCHEMA }); + + // mkdocsYml should be an object type after successful parsing. + // But based on its type definition, it can also be a string or undefined, which we don't want. + if (typeof mkdocsYml === 'string' || typeof mkdocsYml === 'undefined') { + throw new Error('Bad YAML format.'); + } + } catch (error) { + assertError(error); + logger.warn( + `Error in parsing YAML at ${mkdocsYmlPath} before running the generator. ${error.message}`, + ); + return; + } + + didEdit = updateAction(mkdocsYml); + + try { + if (didEdit) { + await fs.writeFile( + mkdocsYmlPath, + yaml.dump(mkdocsYml, { schema: MKDOCS_SCHEMA }), + 'utf8', + ); + } + } catch (error) { + assertError(error); + logger.warn( + `Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`, + ); + return; + } +}; + +/** + * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site. + * + * List of tasks: + * - Add repo_url or edit_uri if it does not exists + * If mkdocs.yml has a repo_url, the generated docs site gets an Edit button on the pages by default. + * If repo_url is missing in mkdocs.yml, we will use techdocs annotation of the entity to possibly get + * the repository URL. + * + * This function will not throw an error since this is not critical to the whole TechDocs pipeline. + * Instead it will log warnings if there are any errors in reading, parsing or writing YAML. + * + * @param mkdocsYmlPath - Absolute path to mkdocs.yml or equivalent of a docs site + * @param logger - A logger instance + * @param parsedLocationAnnotation - Object with location url and type + * @param scmIntegrations - the scmIntegration to do url transformations + */ +export const patchMkdocsYmlPreBuild = async ( + mkdocsYmlPath: string, + logger: Logger, + parsedLocationAnnotation: ParsedLocationAnnotation, + scmIntegrations: ScmIntegrationRegistry, +) => { + await patchMkdocsFile(mkdocsYmlPath, logger, mkdocsYml => { + if (!('repo_url' in mkdocsYml) && !('edit_uri' in mkdocsYml)) { + // Add edit_uri and/or repo_url to mkdocs.yml if it is missing. + // This will enable the Page edit button generated by MkDocs. + // If the either has been set, keep the original value + const result = getRepoUrlFromLocationAnnotation( + parsedLocationAnnotation, + scmIntegrations, + mkdocsYml.docs_dir, + ); + + if (result.repo_url || result.edit_uri) { + mkdocsYml.repo_url = result.repo_url; + mkdocsYml.edit_uri = result.edit_uri; + + logger.info( + `Set ${JSON.stringify( + result, + )}. You can disable this feature by manually setting 'repo_url' or 'edit_uri' according to the MkDocs documentation at https://www.mkdocs.org/user-guide/configuration/#repo_url`, + ); + return true; + } + } + return false; + }); +}; + +/** + * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site. + * + * List of tasks: + * - Add techdocs-core plugin to mkdocs file if it doesn't exist + * + * This function will not throw an error since this is not critical to the whole TechDocs pipeline. + * Instead it will log warnings if there are any errors in reading, parsing or writing YAML. + * + * @param mkdocsYmlPath - Absolute path to mkdocs.yml or equivalent of a docs site + * @param logger - A logger instance + */ +export const pathMkdocsYmlWithTechdocsPlugin = async ( + mkdocsYmlPath: string, + logger: Logger, +) => { + await patchMkdocsFile(mkdocsYmlPath, logger, mkdocsYml => { + // Modify mkdocs.yaml to contain the needed techdocs-core plugin if it is not there + if (!('plugins' in mkdocsYml)) { + mkdocsYml.plugins = ['techdocs-core']; + return true; + } + + if (mkdocsYml.plugins && !mkdocsYml.plugins.includes('techdocs-core')) { + mkdocsYml.plugins.push('techdocs-core'); + return true; + } + return false; + }); +}; diff --git a/packages/techdocs-common/src/stages/generate/techdocs.ts b/packages/techdocs-common/src/stages/generate/techdocs.ts index baa2e008a1..b46f52f99a 100644 --- a/packages/techdocs-common/src/stages/generate/techdocs.ts +++ b/packages/techdocs-common/src/stages/generate/techdocs.ts @@ -26,11 +26,15 @@ import { createOrUpdateMetadata, getMkdocsYml, patchIndexPreBuild, - patchMkdocsYmlPreBuild, runCommand, storeEtagMetadata, validateMkdocsYaml, } from './helpers'; + +import { + patchMkdocsYmlPreBuild, + pathMkdocsYmlWithTechdocsPlugin, +} from './mkDocsPatchers'; import { GeneratorBase, GeneratorConfig, @@ -110,6 +114,10 @@ export class TechdocsGenerator implements GeneratorBase { await patchIndexPreBuild({ inputDir, logger: childLogger, docsDir }); } + if (!this.options.omitTechdocsCoreMkdocsPlugin) { + await pathMkdocsYmlWithTechdocsPlugin(mkdocsYmlPath, childLogger); + } + // Directories to bind on container const mountDirs = { [inputDir]: '/input', @@ -207,5 +215,8 @@ export function readGeneratorConfig( 'docker', dockerImage: config.getOptionalString('techdocs.generator.dockerImage'), pullImage: config.getOptionalBoolean('techdocs.generator.pullImage'), + omitTechdocsCoreMkdocsPlugin: config.getOptionalBoolean( + 'techdocs.generator.mkdocs.omitTechdocsCorePlugin', + ), }; } diff --git a/packages/techdocs-common/src/stages/generate/types.ts b/packages/techdocs-common/src/stages/generate/types.ts index 2ff3064991..f46dbdc56a 100644 --- a/packages/techdocs-common/src/stages/generate/types.ts +++ b/packages/techdocs-common/src/stages/generate/types.ts @@ -39,6 +39,7 @@ export type GeneratorConfig = { runIn: GeneratorRunInType; dockerImage?: string; pullImage?: boolean; + omitTechdocsCoreMkdocsPlugin?: boolean; }; /** From 44a1a447cc37100cd6e175308e8b77b92f8f7a71 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Tue, 1 Mar 2022 19:28:53 +1100 Subject: [PATCH 038/150] Fix up links to other pages Signed-off-by: Nikolas Skoufis --- docs/features/techdocs/concepts.md | 2 +- docs/features/techdocs/how-to-guides.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/features/techdocs/concepts.md b/docs/features/techdocs/concepts.md index f8e813d89e..e34f58bc70 100644 --- a/docs/features/techdocs/concepts.md +++ b/docs/features/techdocs/concepts.md @@ -62,7 +62,7 @@ However any logic that satisfies the Build Strategy interface can be implemented config as well as the entity being processed to make a decision. For an example of how the Build Strategy can be used to implement a 'hybrid' build model, refer to -the [How to implement a hybrid build strategy](./how-to-guides#how-to-implement-a-hybrid-build-strategy) guide. +the [How to implement a hybrid build strategy](./how-to-guides.md#how-to-implement-a-hybrid-build-strategy) guide. ## TechDocs Container diff --git a/docs/features/techdocs/how-to-guides.md b/docs/features/techdocs/how-to-guides.md index 5a5fe36526..56ca2252d0 100644 --- a/docs/features/techdocs/how-to-guides.md +++ b/docs/features/techdocs/how-to-guides.md @@ -541,7 +541,7 @@ Done! Now you have a support of the following diagrams along with mermaid: ## How to implement a hybrid build strategy -One limitation of the [Recommended deployment](./architecture#recommended-deployment) is that +One limitation of the [Recommended deployment](./architecture.md#recommended-deployment) is that the experience for users requires modifying their CI/CD process to publish their TechDocs. For some users, this may be unnecessary, and provides a barrier to entry for onboarding users to Backstage. However, a purely local TechDocs @@ -549,7 +549,7 @@ build restricts TechDocs creators to using the tooling provided in Backstage, as well as the plugins and features provided in the Backstage-included `mkdocs` installation. -To accommodate both of these use-cases, users can implement a custom [Build Strategy](./concepts#techdocs-build-strategy) +To accommodate both of these use-cases, users can implement a custom [Build Strategy](./concepts.md#techdocs-build-strategy) with logic to encode which TechDocs should be built locally, and which will be built externally. From 2d5783aa5b24ce465df0e58ccb70af6e33a41d87 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Mar 2022 09:44:44 +0100 Subject: [PATCH 039/150] chore: sort out the exporting Signed-off-by: blam --- plugins/scaffolder/src/extensions/index.tsx | 4 ++-- plugins/scaffolder/src/index.ts | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/plugins/scaffolder/src/extensions/index.tsx b/plugins/scaffolder/src/extensions/index.tsx index 72df8d4df3..75a166beff 100644 --- a/plugins/scaffolder/src/extensions/index.tsx +++ b/plugins/scaffolder/src/extensions/index.tsx @@ -25,14 +25,14 @@ import { Extension, attachComponentData } from '@backstage/core-plugin-api'; export const FIELD_EXTENSION_WRAPPER_KEY = 'scaffolder.extensions.wrapper.v1'; export const FIELD_EXTENSION_KEY = 'scaffolder.extensions.field.v1'; -export type FieldExtensionComponent<_TInputProps> = () => null; +export type FieldExtensionComponent<_TReturnValue, _TInputProps> = () => null; export function createScaffolderFieldExtension< TReturnValue = unknown, TInputProps = unknown, >( options: FieldExtensionOptions, -): Extension> { +): Extension> { return { expose() { const FieldExtensionDataHolder: any = () => null; diff --git a/plugins/scaffolder/src/index.ts b/plugins/scaffolder/src/index.ts index 71cd1d8f84..9776549091 100644 --- a/plugins/scaffolder/src/index.ts +++ b/plugins/scaffolder/src/index.ts @@ -40,11 +40,7 @@ export { createScaffolderFieldExtension, ScaffolderFieldExtensions, } from './extensions'; -export type { - CustomFieldValidator, - FieldExtensionOptions, - FieldExtensionComponentProps, -} from './extensions'; + export { EntityPickerFieldExtension, EntityNamePickerFieldExtension, @@ -56,4 +52,5 @@ export { scaffolderPlugin, } from './plugin'; export * from './components'; +export * from './extensions'; export type { TaskPageProps } from './components/TaskPage'; From 5f489049693d6b0f5525b7cb8cd1460b8438e160 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 1 Mar 2022 09:44:46 +0100 Subject: [PATCH 040/150] scripts: Update list-deprecations script to use new release tags Signed-off-by: Johan Haals --- scripts/list-deprecations.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/list-deprecations.js b/scripts/list-deprecations.js index 9a159927fc..8047158138 100755 --- a/scripts/list-deprecations.js +++ b/scripts/list-deprecations.js @@ -57,9 +57,7 @@ class ReleaseProvider { ); // Filter out just the releases - const releases = tagOutput - .split('\n') - .filter(l => l.startsWith('release-')); + const releases = tagOutput.split('\n').filter(l => l.startsWith('v')); // Then find the earliest release that affected our package for (const release of releases) { From bb2bb3665190bb19f8a5da2d01c00c43c1a29da8 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 1 Mar 2022 09:28:50 +0100 Subject: [PATCH 041/150] core-plugin-api: Remove deprecated get method from StorageApi Signed-off-by: Johan Haals --- .changeset/new-foxes-matter.md | 7 ++++ .changeset/two-lobsters-hammer.md | 6 ++++ .../StorageApi/WebStorage.test.ts | 28 ++++++--------- .../implementations/StorageApi/WebStorage.ts | 2 +- .../DismissableBanner.test.tsx | 2 +- .../DismissableBanner/DismissableBanner.tsx | 8 ++--- packages/core-plugin-api/api-report.md | 8 ----- .../src/apis/definitions/StorageApi.ts | 19 ---------- packages/test-utils/api-report.md | 2 -- .../apis/StorageApi/MockStorageApi.test.ts | 35 ++++++++----------- .../apis/StorageApi/MockStorageApi.ts | 8 ----- .../DefaultStarredEntitiesApi.ts | 4 +-- .../apis/StarredEntitiesApi/migration.test.ts | 20 ++++++----- .../src/apis/StarredEntitiesApi/migration.ts | 6 ++-- 14 files changed, 60 insertions(+), 95 deletions(-) create mode 100644 .changeset/new-foxes-matter.md create mode 100644 .changeset/two-lobsters-hammer.md diff --git a/.changeset/new-foxes-matter.md b/.changeset/new-foxes-matter.md new file mode 100644 index 0000000000..234dd78451 --- /dev/null +++ b/.changeset/new-foxes-matter.md @@ -0,0 +1,7 @@ +--- +'@backstage/core-app-api': minor +'@backstage/core-plugin-api': minor +'@backstage/test-utils': minor +--- + +**BREAKING**: Removed the deprecated `get` method from `StorageAPI` and its implementations, this method has been replaced by the `snapshot` method. The return value from snapshot no longer includes `newValue` which has been replaced by `value`. For getting notified when a value changes, use `observe$`. diff --git a/.changeset/two-lobsters-hammer.md b/.changeset/two-lobsters-hammer.md new file mode 100644 index 0000000000..412901fb41 --- /dev/null +++ b/.changeset/two-lobsters-hammer.md @@ -0,0 +1,6 @@ +--- +'@backstage/core-components': patch +'@backstage/plugin-catalog-react': patch +--- + +Updated usage of `StorageApi` to use `snapshot` method instead of `get` diff --git a/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.test.ts b/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.test.ts index d81cba48d2..2f82ed5e88 100644 --- a/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.test.ts +++ b/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.test.ts @@ -33,12 +33,11 @@ describe('WebStorage Storage API', () => { it('should return undefined for values which are unset', async () => { const storage = createWebStorage(); - expect(storage.get('myfakekey')).toBeUndefined(); + expect(storage.snapshot('myfakekey').value).toBeUndefined(); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'absent', value: undefined, - newValue: undefined, }); }); @@ -48,26 +47,23 @@ describe('WebStorage Storage API', () => { await storage.set('myfakekey', 'helloimastring'); await storage.set('mysecondfakekey', 1234); await storage.set('mythirdfakekey', true); - expect(storage.get('myfakekey')).toBe('helloimastring'); - expect(storage.get('mysecondfakekey')).toBe(1234); - expect(storage.get('mythirdfakekey')).toBe(true); + expect(storage.snapshot('myfakekey').value).toBe('helloimastring'); + expect(storage.snapshot('mysecondfakekey').value).toBe(1234); + expect(storage.snapshot('mythirdfakekey').value).toBe(true); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'present', value: 'helloimastring', - newValue: 'helloimastring', }); expect(storage.snapshot('mysecondfakekey')).toEqual({ key: 'mysecondfakekey', presence: 'present', value: 1234, - newValue: 1234, }); expect(storage.snapshot('mythirdfakekey')).toEqual({ key: 'mythirdfakekey', presence: 'present', value: true, - newValue: true, }); }); @@ -81,12 +77,11 @@ describe('WebStorage Storage API', () => { await storage.set('myfakekey', mockData); - expect(storage.get('myfakekey')).toEqual(mockData); + expect(storage.snapshot('myfakekey').value).toEqual(mockData); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'present', value: mockData, - newValue: mockData, }); }); @@ -118,7 +113,6 @@ describe('WebStorage Storage API', () => { key: 'correctKey', presence: 'present', value: mockData, - newValue: mockData, }); }); @@ -152,7 +146,6 @@ describe('WebStorage Storage API', () => { key: 'correctKey', presence: 'absent', value: undefined, - newValue: undefined, }); }); @@ -166,9 +159,11 @@ describe('WebStorage Storage API', () => { await firstStorage.set(keyName, 'boop'); await secondStorage.set(keyName, 'deerp'); - expect(firstStorage.get(keyName)).not.toBe(secondStorage.get(keyName)); - expect(firstStorage.get(keyName)).toBe('boop'); - expect(secondStorage.get(keyName)).toBe('deerp'); + expect(firstStorage.snapshot(keyName)).not.toBe( + secondStorage.snapshot(keyName), + ); + expect(firstStorage.snapshot(keyName).value).toBe('boop'); + expect(secondStorage.snapshot(keyName).value).toBe('deerp'); expect(firstStorage.snapshot(keyName)).not.toEqual( secondStorage.snapshot(keyName), ); @@ -176,13 +171,11 @@ describe('WebStorage Storage API', () => { key: keyName, presence: 'present', value: 'boop', - newValue: 'boop', }); expect(secondStorage.snapshot(keyName)).toEqual({ key: keyName, presence: 'present', value: 'deerp', - newValue: 'deerp', }); }); @@ -217,7 +210,6 @@ describe('WebStorage Storage API', () => { key: 'key', presence: 'absent', value: undefined, - newValue: undefined, }); expect(mockErrorApi.post).toHaveBeenCalledWith(expect.any(Error)); expect(mockErrorApi.post).toHaveBeenCalledWith( diff --git a/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.ts b/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.ts index c4cffaf184..ac3d3f20f2 100644 --- a/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.ts +++ b/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.ts @@ -65,7 +65,7 @@ export class WebStorage implements StorageApi { new Error(`Error when parsing JSON config from storage for: ${key}`), ); } - return { key, value, newValue: value, presence }; + return { key, value, presence }; } forBucket(name: string): WebStorage { diff --git a/packages/core-components/src/components/DismissableBanner/DismissableBanner.test.tsx b/packages/core-components/src/components/DismissableBanner/DismissableBanner.test.tsx index e62b3fa82d..3348c278ee 100644 --- a/packages/core-components/src/components/DismissableBanner/DismissableBanner.test.tsx +++ b/packages/core-components/src/components/DismissableBanner/DismissableBanner.test.tsx @@ -75,7 +75,7 @@ describe('', () => { ); fireEvent.click(button); const dismissedBanners = - notifications?.get('dismissedBanners') ?? []; + notifications?.snapshot('dismissedBanners').value ?? []; expect( dismissedBanners.includes('catalog_page_welcome_banner'), ).toBeTruthy(); diff --git a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx index 566b326cb5..9ff7616c45 100644 --- a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx +++ b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx @@ -101,7 +101,7 @@ export const DismissableBanner = (props: Props) => { const storageApi = useApi(storageApiRef); const notificationsStore = storageApi.forBucket('notifications'); const rawDismissedBanners = - notificationsStore.get('dismissedBanners') ?? []; + notificationsStore.snapshot('dismissedBanners').value ?? []; const [dismissedBanners, setDismissedBanners] = useState( new Set(rawDismissedBanners), @@ -112,11 +112,11 @@ export const DismissableBanner = (props: Props) => { ); useEffect(() => { - if (observedItems?.newValue) { - const currentValue = observedItems?.newValue ?? []; + if (observedItems?.value) { + const currentValue = observedItems?.value ?? []; setDismissedBanners(new Set(currentValue)); } - }, [observedItems?.newValue]); + }, [observedItems?.value]); const handleClick = () => { notificationsStore.set('dismissedBanners', [...dismissedBanners, id]); diff --git a/packages/core-plugin-api/api-report.md b/packages/core-plugin-api/api-report.md index 83cca4477b..89361419ba 100644 --- a/packages/core-plugin-api/api-report.md +++ b/packages/core-plugin-api/api-report.md @@ -678,8 +678,6 @@ export type SignInPageProps = { // @public export interface StorageApi { forBucket(name: string): StorageApi; - // @deprecated - get(key: string): T | undefined; observe$( key: string, ): Observable>; @@ -691,23 +689,17 @@ export interface StorageApi { // @public export const storageApiRef: ApiRef; -// @public @deprecated (undocumented) -export type StorageValueChange = - StorageValueSnapshot; - // @public export type StorageValueSnapshot = | { key: string; presence: 'unknown' | 'absent'; value?: undefined; - newValue?: undefined; } | { key: string; presence: 'present'; value: TValue; - newValue?: TValue; }; // @public diff --git a/packages/core-plugin-api/src/apis/definitions/StorageApi.ts b/packages/core-plugin-api/src/apis/definitions/StorageApi.ts index 4bd49f9b83..1506e5f143 100644 --- a/packages/core-plugin-api/src/apis/definitions/StorageApi.ts +++ b/packages/core-plugin-api/src/apis/definitions/StorageApi.ts @@ -27,24 +27,13 @@ export type StorageValueSnapshot = key: string; presence: 'unknown' | 'absent'; value?: undefined; - /** @deprecated Use `value` instead */ - newValue?: undefined; } | { key: string; presence: 'present'; value: TValue; - /** @deprecated Use `value` instead */ - newValue?: TValue; }; -/** - * @public - * @deprecated Use StorageValueSnapshot instead - */ -export type StorageValueChange = - StorageValueSnapshot; - /** * Provides a key-value persistence API. * @@ -59,14 +48,6 @@ export interface StorageApi { */ forBucket(name: string): StorageApi; - /** - * Get the current value for persistent data, use observe$ to be notified of updates. - * - * @deprecated Use `snapshot` instead. - * @param key - Unique key associated with the data. - */ - get(key: string): T | undefined; - /** * Remove persistent data. * diff --git a/packages/test-utils/api-report.md b/packages/test-utils/api-report.md index 4e5361ff80..2f2671885e 100644 --- a/packages/test-utils/api-report.md +++ b/packages/test-utils/api-report.md @@ -164,8 +164,6 @@ export class MockStorageApi implements StorageApi { // (undocumented) forBucket(name: string): StorageApi; // (undocumented) - get(key: string): T | undefined; - // (undocumented) observe$(key: string): Observable>; // (undocumented) remove(key: string): Promise; diff --git a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts index c974198be1..63aa4235e1 100644 --- a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts +++ b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts @@ -24,7 +24,7 @@ describe('WebStorage Storage API', () => { it('should return undefined for values which are unset', async () => { const storage = createMockStorage(); - expect(storage.get('myfakekey')).toBeUndefined(); + expect(storage.snapshot('myfakekey').value).toBeUndefined(); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'absent', @@ -33,32 +33,29 @@ describe('WebStorage Storage API', () => { }); }); - it('should allow the setting and getting of the simple data structures', async () => { + it('should allow the setting and snapshotting of the simple data structures', async () => { const storage = createMockStorage(); await storage.set('myfakekey', 'helloimastring'); await storage.set('mysecondfakekey', 1234); await storage.set('mythirdfakekey', true); - expect(storage.get('myfakekey')).toBe('helloimastring'); - expect(storage.get('mysecondfakekey')).toBe(1234); - expect(storage.get('mythirdfakekey')).toBe(true); + expect(storage.snapshot('myfakekey').value).toBe('helloimastring'); + expect(storage.snapshot('mysecondfakekey').value).toBe(1234); + expect(storage.snapshot('mythirdfakekey').value).toBe(true); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'present', value: 'helloimastring', - newValue: 'helloimastring', }); expect(storage.snapshot('mysecondfakekey')).toEqual({ key: 'mysecondfakekey', presence: 'present', value: 1234, - newValue: 1234, }); expect(storage.snapshot('mythirdfakekey')).toEqual({ key: 'mythirdfakekey', presence: 'present', value: true, - newValue: true, }); }); @@ -72,12 +69,11 @@ describe('WebStorage Storage API', () => { await storage.set('myfakekey', mockData); - expect(storage.get('myfakekey')).toEqual(mockData); + expect(storage.snapshot('myfakekey').value).toEqual(mockData); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'present', value: mockData, - newValue: mockData, }); }); @@ -107,7 +103,6 @@ describe('WebStorage Storage API', () => { key: 'correctKey', presence: 'present', value: mockData, - newValue: mockData, }); }); @@ -153,9 +148,11 @@ describe('WebStorage Storage API', () => { await firstStorage.set(keyName, 'boop'); await secondStorage.set(keyName, 'deerp'); - expect(firstStorage.get(keyName)).not.toBe(secondStorage.get(keyName)); - expect(firstStorage.get(keyName)).toBe('boop'); - expect(secondStorage.get(keyName)).toBe('deerp'); + expect(firstStorage.snapshot(keyName)).not.toBe( + secondStorage.snapshot(keyName), + ); + expect(firstStorage.snapshot(keyName).value).toBe('boop'); + expect(secondStorage.snapshot(keyName).value).toBe('deerp'); expect(firstStorage.snapshot(keyName)).not.toEqual( secondStorage.snapshot(keyName), ); @@ -163,13 +160,11 @@ describe('WebStorage Storage API', () => { key: keyName, presence: 'present', value: 'boop', - newValue: 'boop', }); expect(secondStorage.snapshot(keyName)).toEqual({ key: keyName, presence: 'present', value: 'deerp', - newValue: 'deerp', }); }); @@ -186,7 +181,7 @@ describe('WebStorage Storage API', () => { await firstStorage.set('test2', { error: true }); - expect(secondStorage.get('deep/test2')).toBe(undefined); + expect(secondStorage.snapshot('deep/test2').value).toBe(undefined); expect(secondStorage.snapshot('deep/test2')).toMatchObject({ presence: 'absent', }); @@ -201,19 +196,17 @@ describe('WebStorage Storage API', () => { await firstStorage.set('test2', true); - expect(firstStorage.get('test2')).toBe(true); - expect(secondStorage.get('test2')).toBe(undefined); + expect(firstStorage.snapshot('test2').value).toBe(true); + expect(secondStorage.snapshot('test2').value).toBe(undefined); expect(firstStorage.snapshot('test2')).toEqual({ key: 'test2', presence: 'present', value: true, - newValue: true, }); expect(secondStorage.snapshot('test2')).toEqual({ key: 'test2', presence: 'absent', value: undefined, - newValue: undefined, }); }); diff --git a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts index 25024f2211..6ce91a4a31 100644 --- a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts +++ b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts @@ -61,10 +61,6 @@ export class MockStorageApi implements StorageApi { return this.bucketStorageApis.get(name)!; } - get(key: string): T | undefined { - return this.snapshot(key).value as T | undefined; - } - snapshot(key: string): StorageValueSnapshot { if (this.data.hasOwnProperty(this.getKeyName(key))) { const data = this.data[this.getKeyName(key)]; @@ -72,14 +68,12 @@ export class MockStorageApi implements StorageApi { key, presence: 'present', value: data, - newValue: data, }; } return { key, presence: 'absent', value: undefined, - newValue: undefined, }; } @@ -95,7 +89,6 @@ export class MockStorageApi implements StorageApi { key, presence: 'present', value: serialized, - newValue: serialized, }); } @@ -105,7 +98,6 @@ export class MockStorageApi implements StorageApi { key, presence: 'absent', value: undefined, - newValue: undefined, }); } diff --git a/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts index e9cf84f11b..f4b7771d04 100644 --- a/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts +++ b/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -36,12 +36,12 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { this.settingsStore = opts.storageApi.forBucket('starredEntities'); this.starredEntities = new Set( - this.settingsStore.get('entityRefs') ?? [], + this.settingsStore.snapshot('entityRefs').value ?? [], ); this.settingsStore.observe$('entityRefs').subscribe({ next: next => { - this.starredEntities = new Set(next.newValue ?? []); + this.starredEntities = new Set(next.value ?? []); this.notifyChanges(); }, }); diff --git a/plugins/catalog/src/apis/StarredEntitiesApi/migration.test.ts b/plugins/catalog/src/apis/StarredEntitiesApi/migration.test.ts index 7eb5648d13..b3bc5400d5 100644 --- a/plugins/catalog/src/apis/StarredEntitiesApi/migration.test.ts +++ b/plugins/catalog/src/apis/StarredEntitiesApi/migration.test.ts @@ -41,19 +41,19 @@ describe('performMigrationToTheNewBucket', () => { 'entity:Component:default:a', 'entity:template:custom:b', ]); - expect(oldBucket.get('starredEntities')).not.toBeUndefined(); + expect(oldBucket.snapshot('starredEntities').value).not.toBeUndefined(); await performMigrationToTheNewBucket({ storageApi: mockStorage }); // read NEW bucket - expect(await newBucket.get('entityRefs')).toEqual([ + expect(await newBucket.snapshot('entityRefs').value).toEqual([ 'component:default/c', 'component:default/a', 'template:custom/b', ]); // OLD bucket should be removed - expect(oldBucket.get('starredEntities')).toBeUndefined(); + expect(oldBucket.snapshot('starredEntities').value).toBeUndefined(); }); it('should ignore invalid entries', async () => { @@ -67,15 +67,17 @@ describe('performMigrationToTheNewBucket', () => { 'entity:Component:a', 'invalid', ]); - expect(oldBucket.get('starredEntities')).not.toBeUndefined(); + expect(oldBucket.snapshot('starredEntities')).not.toBeUndefined(); await performMigrationToTheNewBucket({ storageApi: mockStorage }); // read NEW bucket - expect(await newBucket.get('entityRefs')).toEqual(['component:default/a']); + expect(await newBucket.snapshot('entityRefs').value).toEqual([ + 'component:default/a', + ]); // OLD bucket should be removed - expect(oldBucket.get('starredEntities')).toBeUndefined(); + expect(oldBucket.snapshot('starredEntities').value).toBeUndefined(); }); it('should skip migration without old starred entities', async () => { @@ -88,7 +90,7 @@ describe('performMigrationToTheNewBucket', () => { await performMigrationToTheNewBucket({ storageApi: mockStorage }); // read NEW bucket - expect(newBucket.get('entityRefs')).toEqual(expectedEntries); + expect(newBucket.snapshot('entityRefs').value).toEqual(expectedEntries); }); it('should skip migration with non-array old starred entities', async () => { @@ -105,9 +107,9 @@ describe('performMigrationToTheNewBucket', () => { await performMigrationToTheNewBucket({ storageApi: mockStorage }); // read NEW bucket - expect(newBucket.get('entityRefs')).toEqual(expectedEntries); + expect(newBucket.snapshot('entityRefs').value).toEqual(expectedEntries); // OLD bucket should be unchanged - expect(oldBucket.get('starredEntities')).toBe('invalid'); + expect(oldBucket.snapshot('starredEntities').value).toBe('invalid'); }); }); diff --git a/plugins/catalog/src/apis/StarredEntitiesApi/migration.ts b/plugins/catalog/src/apis/StarredEntitiesApi/migration.ts index 8405d9e08d..51dcd6e7ce 100644 --- a/plugins/catalog/src/apis/StarredEntitiesApi/migration.ts +++ b/plugins/catalog/src/apis/StarredEntitiesApi/migration.ts @@ -35,13 +35,15 @@ export async function performMigrationToTheNewBucket({ const source = storageApi.forBucket('settings'); const target = storageApi.forBucket('starredEntities'); - const oldStarredEntities = source.get('starredEntities'); + const oldStarredEntities = source.snapshot('starredEntities').value; if (!isArray(oldStarredEntities)) { // nothing to do return; } - const targetEntities = new Set(target.get('entityRefs') ?? []); + const targetEntities = new Set( + target.snapshot('entityRefs').value ?? [], + ); oldStarredEntities .filter(isString) From be68702834758f8dec018d413d9ff774026250a4 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 1 Mar 2022 11:13:28 +0100 Subject: [PATCH 042/150] catalog-backend: inline and deprecate BitbucketRepositoryParser type Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- plugins/catalog-backend/api-report.md | 14 +++++++-- .../BitbucketDiscoveryProcessor.test.ts | 21 +++++++------ .../processors/BitbucketDiscoveryProcessor.ts | 29 ++++++++++++++---- .../BitbucketRepositoryParser.test.ts | 4 --- .../bitbucket/BitbucketRepositoryParser.ts | 30 +++++++++++-------- 5 files changed, 63 insertions(+), 35 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 9b4db418b3..b69e1d5841 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -137,14 +137,22 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { export class BitbucketDiscoveryProcessor implements CatalogProcessor { constructor(options: { integrations: ScmIntegrationRegistry; - parser?: BitbucketRepositoryParser; + parser?: (options: { + integration: BitbucketIntegration; + target: string; + logger: Logger_2; + }) => AsyncIterable; logger: Logger_2; }); // (undocumented) static fromConfig( config: Config, options: { - parser?: BitbucketRepositoryParser; + parser?: (options: { + integration: BitbucketIntegration; + target: string; + logger: Logger_2; + }) => AsyncIterable; logger: Logger_2; }, ): BitbucketDiscoveryProcessor; @@ -158,7 +166,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { ): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) export type BitbucketRepositoryParser = (options: { integration: BitbucketIntegration; target: string; diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 9ee8bf1b57..4a261386c0 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -18,7 +18,6 @@ import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; import { ConfigReader } from '@backstage/config'; import { BitbucketRepository20, - BitbucketRepositoryParser, PagedResponse, PagedResponse20, } from './bitbucket'; @@ -742,15 +741,6 @@ describe('BitbucketDiscoveryProcessor', () => { }); describe('Custom repository parser', () => { - const customRepositoryParser: BitbucketRepositoryParser = - async function* customRepositoryParser({}) { - yield results.location({ - type: 'custom-location-type', - target: 'custom-target', - presence: 'optional', - }); - }; - const processor = BitbucketDiscoveryProcessor.fromConfig( new ConfigReader({ integrations: { @@ -763,7 +753,16 @@ describe('BitbucketDiscoveryProcessor', () => { ], }, }), - { parser: customRepositoryParser, logger: getVoidLogger() }, + { + parser: async function* customRepositoryParser({}) { + yield results.location({ + type: 'custom-location-type', + target: 'custom-target', + presence: 'optional', + }); + }, + logger: getVoidLogger(), + }, ); it('use custom repository parser', async () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index 8fcf64dc55..25f52b7dd7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -22,7 +22,6 @@ import { ScmIntegrations, } from '@backstage/integration'; import { - BitbucketRepositoryParser, BitbucketClient, defaultRepositoryParser, paginated, @@ -30,7 +29,12 @@ import { BitbucketRepository, BitbucketRepository20, } from './bitbucket'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { + CatalogProcessor, + CatalogProcessorEmit, + CatalogProcessorResult, + LocationSpec, +} from './types'; const DEFAULT_BRANCH = 'master'; const DEFAULT_CATALOG_LOCATION = '/catalog-info.yaml'; @@ -39,12 +43,23 @@ const EMPTY_CATALOG_LOCATION = '/'; /** @public */ export class BitbucketDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrationRegistry; - private readonly parser: BitbucketRepositoryParser; + private readonly parser: (options: { + integration: BitbucketIntegration; + target: string; + logger: Logger; + }) => AsyncIterable; private readonly logger: Logger; static fromConfig( config: Config, - options: { parser?: BitbucketRepositoryParser; logger: Logger }, + options: { + parser?: (options: { + integration: BitbucketIntegration; + target: string; + logger: Logger; + }) => AsyncIterable; + logger: Logger; + }, ) { const integrations = ScmIntegrations.fromConfig(config); @@ -56,7 +71,11 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { constructor(options: { integrations: ScmIntegrationRegistry; - parser?: BitbucketRepositoryParser; + parser?: (options: { + integration: BitbucketIntegration; + target: string; + logger: Logger; + }) => AsyncIterable; logger: Logger; }) { this.integrations = options.integrations; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts index 35e24ed4cb..e010865041 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts @@ -15,8 +15,6 @@ */ import { defaultRepositoryParser } from './BitbucketRepositoryParser'; import { results } from '../index'; -import { getVoidLogger } from '@backstage/backend-common'; -import { BitbucketIntegration } from '@backstage/integration'; describe('BitbucketRepositoryParser', () => { describe('defaultRepositoryParser', () => { @@ -32,9 +30,7 @@ describe('BitbucketRepositoryParser', () => { }), ]; const actual = await defaultRepositoryParser({ - integration: {} as BitbucketIntegration, target: `${browseUrl}${path}`, - logger: getVoidLogger(), }); let i = 0; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts index 9eaec221b5..6671735f57 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts @@ -18,21 +18,27 @@ import { results } from '../index'; import { Logger } from 'winston'; import { BitbucketIntegration } from '@backstage/integration'; -/** @public */ +/** + * @public + * @deprecated type inlined. + */ export type BitbucketRepositoryParser = (options: { integration: BitbucketIntegration; target: string; logger: Logger; }) => AsyncIterable; -export const defaultRepositoryParser: BitbucketRepositoryParser = - async function* defaultRepositoryParser({ target }) { - yield results.location({ - type: 'url', - target: target, - // Not all locations may actually exist, since the user defined them as a wildcard pattern. - // Thus, we emit them as optional and let the downstream processor find them while not outputting - // an error if it couldn't. - presence: 'optional', - }); - }; +export const defaultRepositoryParser = async function* defaultRepositoryParser({ + target, +}: { + target: string; +}) { + yield results.location({ + type: 'url', + target: target, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + presence: 'optional', + }); +}; From fc6d31b5c3826531840c87bb35a85835385d1dc7 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 1 Mar 2022 11:15:06 +0100 Subject: [PATCH 043/150] add changeset Signed-off-by: Johan Haals --- .changeset/plenty-tables-mix.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/plenty-tables-mix.md diff --git a/.changeset/plenty-tables-mix.md b/.changeset/plenty-tables-mix.md new file mode 100644 index 0000000000..5dffdf6946 --- /dev/null +++ b/.changeset/plenty-tables-mix.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Deprecated the `BitbucketRepositoryParser` type. From babb280c77ec445fd7376484d69faabb8debbd83 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Mar 2022 11:27:37 +0100 Subject: [PATCH 044/150] chore: deprecate some of the loading things from useEntity and force entity to be present Signed-off-by: blam --- plugins/catalog-react/src/hooks/index.ts | 3 + plugins/catalog-react/src/hooks/useEntity.tsx | 58 +++++++++++++++---- .../components/EntitySwitch/EntitySwitch.tsx | 7 ++- .../src/components/EntitySwitch/conditions.ts | 6 +- 4 files changed, 58 insertions(+), 16 deletions(-) diff --git a/plugins/catalog-react/src/hooks/index.ts b/plugins/catalog-react/src/hooks/index.ts index 3e6ef7c945..8e0fc48e72 100644 --- a/plugins/catalog-react/src/hooks/index.ts +++ b/plugins/catalog-react/src/hooks/index.ts @@ -18,11 +18,14 @@ export { useEntityFromUrl, EntityProvider, AsyncEntityProvider, + useAsyncEntity, } from './useEntity'; export type { EntityLoadingStatus, EntityProviderProps, AsyncEntityProviderProps, + UseEntityResponse, + UseAsyncEntityResponse, } from './useEntity'; export { useEntityCompoundName } from './useEntityCompoundName'; export { diff --git a/plugins/catalog-react/src/hooks/useEntity.tsx b/plugins/catalog-react/src/hooks/useEntity.tsx index 5cb67cb79a..1db14adc22 100644 --- a/plugins/catalog-react/src/hooks/useEntity.tsx +++ b/plugins/catalog-react/src/hooks/useEntity.tsx @@ -128,26 +128,35 @@ export const useEntityFromUrl = (): EntityLoadingStatus => { return { entity, loading, error, refresh }; }; +export interface UseEntityResponse { + entity: T; + /** @deprecated use useAsyncEntity instead */ + loading: boolean; + /** @deprecated use useAsyncEntity instead */ + error?: Error; + /** @deprecated use useAsyncEntity instead */ + refresh?: VoidFunction; +} + +export interface UseAsyncEntityResponse { + entity?: T; + loading: boolean; + error?: Error; + refresh?: VoidFunction; +} /** - * Grab the current entity from the context and its current loading state. + * Grab the current entity from the context, throws if the entity has not yet been loaded + * or is not available. * * @public */ -export function useEntity() { +export function useEntity(): UseEntityResponse { const versionedHolder = useVersionedContext<{ 1: EntityLoadingStatus }>('entity-context'); if (!versionedHolder) { - // TODO(Rugvip): Throw this once we fully migrate to the new context - // throw new Error('Entity context is not available'); - - return { - entity: undefined as unknown as T, - loading: true, - error: undefined, - refresh: () => {}, - }; + throw new Error('Entity context is not available'); } const value = versionedHolder.atVersion(1); @@ -155,6 +164,33 @@ export function useEntity() { throw new Error('EntityContext v1 not available'); } + if (!value.entity) { + throw new Error('Entity has not been loaded yet'); + } + + const { entity, loading, error, refresh } = value; + return { entity: entity as T, loading, error, refresh }; +} + +/** + * Grab the current entity from the context, provides loading state and errors, and the ability to refresh. + * + * @public + */ +export function useAsyncEntity< + T extends Entity = Entity, +>(): UseAsyncEntityResponse { + const versionedHolder = + useVersionedContext<{ 1: EntityLoadingStatus }>('entity-context'); + + if (!versionedHolder) { + throw new Error('Entity context is not available'); + } + const value = versionedHolder.atVersion(1); + if (!value) { + throw new Error('EntityContext v1 not available'); + } + const { entity, loading, error, refresh } = value; return { entity: entity as T, loading, error, refresh }; } diff --git a/plugins/catalog/src/components/EntitySwitch/EntitySwitch.tsx b/plugins/catalog/src/components/EntitySwitch/EntitySwitch.tsx index 4433517247..1b8667ce2a 100644 --- a/plugins/catalog/src/components/EntitySwitch/EntitySwitch.tsx +++ b/plugins/catalog/src/components/EntitySwitch/EntitySwitch.tsx @@ -15,7 +15,7 @@ */ import { Entity } from '@backstage/catalog-model'; -import { useEntity } from '@backstage/plugin-catalog-react'; +import { useAsyncEntity } from '@backstage/plugin-catalog-react'; import React, { ReactNode, ReactElement } from 'react'; import { attachComponentData, @@ -63,7 +63,7 @@ export interface EntitySwitchProps { /** @public */ export const EntitySwitch = (props: EntitySwitchProps) => { - const { entity } = useEntity(); + const { entity } = useAsyncEntity(); const apis = useApiHolder(); const results = useElementFilter( props.children, @@ -75,6 +75,9 @@ export const EntitySwitch = (props: EntitySwitchProps) => { }) .getElements() .flatMap((element: ReactElement) => { + if (!entity) { + return []; + } const { if: condition, children: elementsChildren } = element.props as EntitySwitchCase; return [ diff --git a/plugins/catalog/src/components/EntitySwitch/conditions.ts b/plugins/catalog/src/components/EntitySwitch/conditions.ts index a02170cc6f..907aa45afe 100644 --- a/plugins/catalog/src/components/EntitySwitch/conditions.ts +++ b/plugins/catalog/src/components/EntitySwitch/conditions.ts @@ -27,7 +27,7 @@ function strCmp(a: string | undefined, b: string | undefined): boolean { * @public */ export function isKind(kind: string) { - return (entity: Entity) => strCmp(entity?.kind, kind); + return (entity: Entity) => strCmp(entity.kind, kind); } /** @@ -36,7 +36,7 @@ export function isKind(kind: string) { */ export function isComponentType(type: string) { return (entity: Entity) => { - if (!strCmp(entity?.kind, 'component')) { + if (!strCmp(entity.kind, 'component')) { return false; } const componentEntity = entity as ComponentEntity; @@ -49,5 +49,5 @@ export function isComponentType(type: string) { * @public */ export function isNamespace(namespace: string) { - return (entity: Entity) => strCmp(entity?.metadata?.namespace, namespace); + return (entity: Entity) => strCmp(entity.metadata?.namespace, namespace); } From c85292b768c07e24554e3033a41f41f33fe20c63 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 1 Mar 2022 11:28:59 +0100 Subject: [PATCH 045/150] catalog-backend: Remove catalogProcessor.handleError Signed-off-by: Johan Haals --- .changeset/serious-mayflies-thank.md | 5 +++++ plugins/catalog-backend/api-report.md | 5 ----- .../src/ingestion/processors/types.ts | 14 -------------- 3 files changed, 5 insertions(+), 19 deletions(-) create mode 100644 .changeset/serious-mayflies-thank.md diff --git a/.changeset/serious-mayflies-thank.md b/.changeset/serious-mayflies-thank.md new file mode 100644 index 0000000000..b3c5e31527 --- /dev/null +++ b/.changeset/serious-mayflies-thank.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +**Breaking**: Removed optional `handleError()` from `CatalogProcessor`. This optional method is never called by the catalog processing engine and can therefore be removed. diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 9b4db418b3..23aae02610 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -300,11 +300,6 @@ export type CatalogProcessor = { emit: CatalogProcessorEmit, cache: CatalogProcessorCache, ): Promise; - handleError?( - error: Error, - location: LocationSpec, - emit: CatalogProcessorEmit, - ): Promise; }; // @public diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/ingestion/processors/types.ts index 879b4d32a8..80beda4213 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/types.ts @@ -116,20 +116,6 @@ export type CatalogProcessor = { emit: CatalogProcessorEmit, cache: CatalogProcessorCache, ): Promise; - - /** - * Handles an emitted error. - * - * @param error - The error - * @param location - The location where the error occurred - * @param emit - A sink for items resulting from this handling - * @returns Nothing - */ - handleError?( - error: Error, - location: LocationSpec, - emit: CatalogProcessorEmit, - ): Promise; }; /** From 44403296e7c9fff1709548f3bdcff7bd8eceb854 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Mar 2022 11:39:49 +0100 Subject: [PATCH 046/150] chore: added changeset Signed-off-by: blam --- .changeset/sour-eggs-kick.md | 9 +++++ plugins/catalog-react/api-report.md | 34 ++++++++++++++++--- plugins/catalog-react/src/hooks/useEntity.tsx | 26 +++++++++++--- 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 .changeset/sour-eggs-kick.md diff --git a/.changeset/sour-eggs-kick.md b/.changeset/sour-eggs-kick.md new file mode 100644 index 0000000000..6e814415ab --- /dev/null +++ b/.changeset/sour-eggs-kick.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-react': patch +--- + +Added the following deprecations to the `catalog-react` package: + +- **DEPRECATION**: `useEntity` will now warn if the entity has not yet been loaded. This hook is now designed only to be used inside of an `EntityPage` where the `entity` prop is guaranteed to be defined. If you would like to use it outside, please use `useAsyncEntity` instead. +- **DEPRECATION**: the `loading`, `error` and `refresh` properties that are returned from `useEntity` have been deprecated, and are available on `useAsyncEntity` instead. diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index e78ec75daa..0103ac2f0e 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -524,12 +524,24 @@ export type UnregisterEntityDialogProps = { }; // @public -export function useEntity(): { - entity: T; +export function useAsyncEntity< + T extends Entity = Entity, +>(): UseAsyncEntityResponse; + +// @public +export interface UseAsyncEntityResponse { + // (undocumented) + entity?: T; + // (undocumented) + error?: Error; + // (undocumented) loading: boolean; - error: Error | undefined; - refresh: VoidFunction | undefined; -}; + // (undocumented) + refresh?: VoidFunction; +} + +// @public +export function useEntity(): UseEntityResponse; // @public @deprecated export const useEntityCompoundName: () => { @@ -571,6 +583,18 @@ export function useEntityPermission(permission: Permission): { error?: Error; }; +// @public +export interface UseEntityResponse { + // (undocumented) + entity: T; + // @deprecated (undocumented) + error?: Error; + // @deprecated (undocumented) + loading: boolean; + // @deprecated (undocumented) + refresh?: VoidFunction; +} + // @public export function useEntityTypeFilter(): { loading: boolean; diff --git a/plugins/catalog-react/src/hooks/useEntity.tsx b/plugins/catalog-react/src/hooks/useEntity.tsx index 1db14adc22..9becedfee8 100644 --- a/plugins/catalog-react/src/hooks/useEntity.tsx +++ b/plugins/catalog-react/src/hooks/useEntity.tsx @@ -128,16 +128,27 @@ export const useEntityFromUrl = (): EntityLoadingStatus => { return { entity, loading, error, refresh }; }; + +/** + * @public + * + * The response shape for {@link useEntity} + */ export interface UseEntityResponse { entity: T; - /** @deprecated use useAsyncEntity instead */ + /** @deprecated use {@link useAsyncEntity} instead */ loading: boolean; - /** @deprecated use useAsyncEntity instead */ + /** @deprecated use {@link useAsyncEntity} instead */ error?: Error; - /** @deprecated use useAsyncEntity instead */ + /** @deprecated use {@link useAsyncEntity} instead */ refresh?: VoidFunction; } +/** + * @public + * + * The response shape for {@link useAsyncEntity} + */ export interface UseAsyncEntityResponse { entity?: T; loading: boolean; @@ -165,7 +176,14 @@ export function useEntity(): UseEntityResponse { } if (!value.entity) { - throw new Error('Entity has not been loaded yet'); + // Once we have removed the additional fields from being returned we can drop this deprecation + // and move to the error instead. + // throw new Error('useEntity hook is being called outside of an EntityPage where the entity has not been loaded. If this is intentional, please use useAsyncEntity instead.'); + + // eslint-disable-next-line no-console + console.warn( + 'DEPRECATION: useEntity hook is being called outside of an EntityPage where the entity has not been loaded. If this is intentional, please use useAsyncEntity instead. This warning will be replaced with an error in future releases.', + ); } const { entity, loading, error, refresh } = value; From f2b4b747a6bd2311f0a51a327fdbdb7436733e68 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Mar 2022 11:57:36 +0100 Subject: [PATCH 047/150] chore: removing deprecated routeRefs from catalog-react Signed-off-by: blam --- plugins/catalog-react/api-report.md | 13 ------------ plugins/catalog-react/src/index.ts | 8 +------ plugins/catalog-react/src/routes.ts | 21 ------------------- .../EntityOrphanWarning.test.tsx | 9 +++----- .../EntityOrphanWarning.tsx | 5 +++-- plugins/catalog/src/plugin.ts | 6 +++--- plugins/catalog/src/routes.ts | 9 +++++++- 7 files changed, 18 insertions(+), 53 deletions(-) diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 3e1136313f..ffddfaedc6 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -95,9 +95,6 @@ export type CatalogReactUserListPickerClassKey = | 'menuItem' | 'groupWrapper'; -// @public @deprecated (undocumented) -export const catalogRouteRef: RouteRef; - // @public (undocumented) export const columnFactories: Readonly<{ createEntityRefColumn(options: { @@ -272,13 +269,6 @@ export type EntityRefLinksProps = { defaultKind?: string; } & Omit; -// @public @deprecated (undocumented) -export const entityRoute: RouteRef<{ - name: string; - kind: string; - namespace: string; -}>; - // @public export function entityRouteParams(entity: Entity): { readonly kind: string; @@ -494,9 +484,6 @@ export function reduceEntityFilters( filters: EntityFilter[], ): (entity: Entity) => boolean; -// @public @deprecated (undocumented) -export const rootRoute: RouteRef; - // @public export interface StarredEntitiesApi { starredEntitie$(): Observable>; diff --git a/plugins/catalog-react/src/index.ts b/plugins/catalog-react/src/index.ts index 7bf1bd1221..2279d49a2c 100644 --- a/plugins/catalog-react/src/index.ts +++ b/plugins/catalog-react/src/index.ts @@ -27,13 +27,7 @@ export * from './apis'; export * from './components'; export * from './hooks'; export * from './filters'; -export { - catalogRouteRef, - entityRoute, - entityRouteParams, - entityRouteRef, - rootRoute, -} from './routes'; +export { entityRouteParams, entityRouteRef } from './routes'; export * from './testUtils'; export * from './types'; export * from './utils'; diff --git a/plugins/catalog-react/src/routes.ts b/plugins/catalog-react/src/routes.ts index b9aa0de531..013bf7b5be 100644 --- a/plugins/catalog-react/src/routes.ts +++ b/plugins/catalog-react/src/routes.ts @@ -18,21 +18,6 @@ import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { createRouteRef } from '@backstage/core-plugin-api'; import { getOrCreateGlobalSingleton } from '@backstage/version-bridge'; -// TODO(Rugvip): Move these route refs back to the catalog plugin once we're all ported to using external routes -/** - * @deprecated Use an `ExternalRouteRef` instead, which can point to `catalogPlugin.routes.catalogIndex`. - * @public - */ -export const rootRoute = createRouteRef({ - id: 'catalog', -}); - -/** - * @deprecated Use an `ExternalRouteRef` instead, which can point to `catalogPlugin.routes.catalogIndex`. - * @public - */ -export const catalogRouteRef = rootRoute; - /** * A stable route ref that points to the catalog page for an individual entity. * @@ -52,12 +37,6 @@ export const entityRouteRef = getOrCreateGlobalSingleton( }), ); -/** - * @deprecated use `entityRouteRef` instead. - * @public - */ -export const entityRoute = entityRouteRef; - /** * Utility function to get suitable route params for entityRoute, given an * @public diff --git a/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.test.tsx b/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.test.tsx index a8aac405fe..f00474868f 100644 --- a/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.test.tsx +++ b/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.test.tsx @@ -14,14 +14,11 @@ * limitations under the License. */ -import { - catalogApiRef, - catalogRouteRef, - EntityProvider, -} from '@backstage/plugin-catalog-react'; +import { catalogApiRef, EntityProvider } from '@backstage/plugin-catalog-react'; import { renderInTestApp, TestApiProvider } from '@backstage/test-utils'; import React from 'react'; +import { rootRouteRef } from '../../routes'; import { EntityOrphanWarning } from './EntityOrphanWarning'; describe('', () => { @@ -59,7 +56,7 @@ describe('', () => { , { mountedRoutes: { - '/create': catalogRouteRef, + '/create': rootRouteRef, }, }, ); diff --git a/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.tsx b/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.tsx index aba28c558f..26b0e5de45 100644 --- a/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.tsx +++ b/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.tsx @@ -15,12 +15,13 @@ */ import { Entity } from '@backstage/catalog-model'; -import { catalogRouteRef, useEntity } from '@backstage/plugin-catalog-react'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { Alert } from '@material-ui/lab'; import React, { useState } from 'react'; import { useNavigate } from 'react-router'; import { DeleteEntityDialog } from './DeleteEntityDialog'; import { useRouteRef } from '@backstage/core-plugin-api'; +import { rootRouteRef } from '../../routes'; /** * Returns true if the given entity has the orphan annotation given by the @@ -40,7 +41,7 @@ export function isOrphan(entity: Entity): boolean { */ export function EntityOrphanWarning() { const navigate = useNavigate(); - const catalogLink = useRouteRef(catalogRouteRef); + const catalogLink = useRouteRef(rootRouteRef); const [confirmationDialogOpen, setConfirmationDialogOpen] = useState(false); const { entity } = useEntity(); diff --git a/plugins/catalog/src/plugin.ts b/plugins/catalog/src/plugin.ts index 520656e921..1b9083e7ca 100644 --- a/plugins/catalog/src/plugin.ts +++ b/plugins/catalog/src/plugin.ts @@ -18,7 +18,6 @@ import { CatalogClient } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; import { catalogApiRef, - catalogRouteRef, entityRouteRef, starredEntitiesApiRef, } from '@backstage/plugin-catalog-react'; @@ -43,6 +42,7 @@ import { HasResourcesCardProps } from './components/HasResourcesCard'; import { HasSubcomponentsCardProps } from './components/HasSubcomponentsCard'; import { HasSystemsCardProps } from './components/HasSystemsCard'; import { RelatedEntitiesCardProps } from './components/RelatedEntitiesCard'; +import { rootRouteRef } from './routes'; /** @public */ export const catalogPlugin = createPlugin({ @@ -65,7 +65,7 @@ export const catalogPlugin = createPlugin({ }), ], routes: { - catalogIndex: catalogRouteRef, + catalogIndex: rootRouteRef, catalogEntity: entityRouteRef, }, externalRoutes: { @@ -81,7 +81,7 @@ export const CatalogIndexPage: (props: DefaultCatalogPageProps) => JSX.Element = name: 'CatalogIndexPage', component: () => import('./components/CatalogPage').then(m => m.CatalogPage), - mountPoint: catalogRouteRef, + mountPoint: rootRouteRef, }), ); diff --git a/plugins/catalog/src/routes.ts b/plugins/catalog/src/routes.ts index 63d9ccfb87..5c0d195d74 100644 --- a/plugins/catalog/src/routes.ts +++ b/plugins/catalog/src/routes.ts @@ -14,7 +14,10 @@ * limitations under the License. */ -import { createExternalRouteRef } from '@backstage/core-plugin-api'; +import { + createExternalRouteRef, + createRouteRef, +} from '@backstage/core-plugin-api'; export const createComponentRouteRef = createExternalRouteRef({ id: 'create-component', @@ -26,3 +29,7 @@ export const viewTechDocRouteRef = createExternalRouteRef({ optional: true, params: ['namespace', 'kind', 'name'], }); + +export const rootRouteRef = createRouteRef({ + id: 'catalog', +}); From 7e0a0109bf84e9cee035a9f5bbc40a18273e323b Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Tue, 1 Mar 2022 12:44:06 +0100 Subject: [PATCH 048/150] Export permission criteria utilities Signed-off-by: Vincenzo Scamporlino --- plugins/permission-node/api-report.md | 18 +++++++ .../permission-node/src/integration/index.ts | 1 + .../permission-node/src/integration/util.ts | 48 ++++++++++++++----- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/plugins/permission-node/api-report.md b/plugins/permission-node/api-report.md index 534ad45110..f6de7f4909 100644 --- a/plugins/permission-node/api-report.md +++ b/plugins/permission-node/api-report.md @@ -3,6 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { AllOfCriteria } from '@backstage/plugin-permission-common'; +import { AnyOfCriteria } from '@backstage/plugin-permission-common'; import { AuthorizeDecision } from '@backstage/plugin-permission-common'; import { AuthorizeQuery } from '@backstage/plugin-permission-common'; import { AuthorizeRequestOptions } from '@backstage/plugin-permission-common'; @@ -11,6 +13,7 @@ import { BackstageIdentityResponse } from '@backstage/plugin-auth-node'; import { Config } from '@backstage/config'; import express from 'express'; import { Identified } from '@backstage/plugin-permission-common'; +import { NotCriteria } from '@backstage/plugin-permission-common'; import { PermissionAuthorizer } from '@backstage/plugin-permission-common'; import { PermissionCondition } from '@backstage/plugin-permission-common'; import { PermissionCriteria } from '@backstage/plugin-permission-common'; @@ -118,6 +121,21 @@ export type DefinitivePolicyDecision = { result: AuthorizeResult.ALLOW | AuthorizeResult.DENY; }; +// @alpha +export const isAndCriteria: ( + criteria: PermissionCriteria, +) => criteria is AllOfCriteria; + +// @alpha +export const isNotCriteria: ( + criteria: PermissionCriteria, +) => criteria is NotCriteria; + +// @alpha +export const isOrCriteria: ( + criteria: PermissionCriteria, +) => criteria is AnyOfCriteria; + // @public export const makeCreatePermissionRule: () => < TParams extends unknown[], diff --git a/plugins/permission-node/src/integration/index.ts b/plugins/permission-node/src/integration/index.ts index 978342e4ed..7702fea95b 100644 --- a/plugins/permission-node/src/integration/index.ts +++ b/plugins/permission-node/src/integration/index.ts @@ -19,3 +19,4 @@ export * from './createConditionExports'; export * from './createConditionTransformer'; export * from './createPermissionIntegrationRouter'; export * from './createPermissionRule'; +export { isAndCriteria, isOrCriteria, isNotCriteria } from './util'; diff --git a/plugins/permission-node/src/integration/util.ts b/plugins/permission-node/src/integration/util.ts index 3878e18895..9102f092c4 100644 --- a/plugins/permission-node/src/integration/util.ts +++ b/plugins/permission-node/src/integration/util.ts @@ -22,20 +22,44 @@ import { } from '@backstage/plugin-permission-common'; import { PermissionRule } from '../types'; -export const isAndCriteria = ( - filter: PermissionCriteria, -): filter is AllOfCriteria => - Object.prototype.hasOwnProperty.call(filter, 'allOf'); +/** + * Utility function used to parse a PermissionCriteria + * @param criteria - a PermissionCriteria + * @alpha + * + * @returns `true` if the permission criteria is of type allOf, + * narrowing down `criteria` to the specific type. + */ +export const isAndCriteria = ( + criteria: PermissionCriteria, +): criteria is AllOfCriteria => + Object.prototype.hasOwnProperty.call(criteria, 'allOf'); -export const isOrCriteria = ( - filter: PermissionCriteria, -): filter is AnyOfCriteria => - Object.prototype.hasOwnProperty.call(filter, 'anyOf'); +/** + * Utility function used to parse a PermissionCriteria of type + * @param criteria - a PermissionCriteria + * @alpha + * + * @returns `true` if the permission criteria is of type anyOf, + * narrowing down `criteria` to the specific type. + */ +export const isOrCriteria = ( + criteria: PermissionCriteria, +): criteria is AnyOfCriteria => + Object.prototype.hasOwnProperty.call(criteria, 'anyOf'); -export const isNotCriteria = ( - filter: PermissionCriteria, -): filter is NotCriteria => - Object.prototype.hasOwnProperty.call(filter, 'not'); +/** + * Utility function used to parse a PermissionCriteria + * @param criteria - a PermissionCriteria + * @alpha + * + * @returns `true` if the permission criteria is of type not, + * narrowing down `criteria` to the specific type. + */ +export const isNotCriteria = ( + criteria: PermissionCriteria, +): criteria is NotCriteria => + Object.prototype.hasOwnProperty.call(criteria, 'not'); export const createGetRule = ( rules: PermissionRule[], From 580f4e1df8f6af6616b0de24f51bfdc9c2a46b03 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Tue, 1 Mar 2022 12:46:17 +0100 Subject: [PATCH 049/150] Add changeset Signed-off-by: Vincenzo Scamporlino --- .changeset/popular-items-tan.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/popular-items-tan.md diff --git a/.changeset/popular-items-tan.md b/.changeset/popular-items-tan.md new file mode 100644 index 0000000000..b91773a5d8 --- /dev/null +++ b/.changeset/popular-items-tan.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-permission-node': patch +--- + +Export some utility functions for parsing PermissionCriteria + +`isAndCriteria`, `isOrCriteria`, `isNotCriteria` are now exported. From 6fe5d70cace2d1123c8f3dff3d84fe8b4eb5189a Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Mar 2022 12:49:04 +0100 Subject: [PATCH 050/150] chore: fix up the changeset Signed-off-by: blam --- .changeset/sour-eggs-kick.md | 3 ++- plugins/catalog-react/src/hooks/useEntity.tsx | 2 +- .../catalog/src/components/EntityLayout/EntityLayout.tsx | 8 +++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.changeset/sour-eggs-kick.md b/.changeset/sour-eggs-kick.md index 6e814415ab..f796b173fb 100644 --- a/.changeset/sour-eggs-kick.md +++ b/.changeset/sour-eggs-kick.md @@ -5,5 +5,6 @@ Added the following deprecations to the `catalog-react` package: -- **DEPRECATION**: `useEntity` will now warn if the entity has not yet been loaded. This hook is now designed only to be used inside of an `EntityPage` where the `entity` prop is guaranteed to be defined. If you would like to use it outside, please use `useAsyncEntity` instead. +- **DEPRECATION**: `useEntity` will now warn if the entity has not yet been loaded, and will soon throw errors instead. If you're using the default implementation of `EntityLayout` and `EntitySwitch` then these components will ensure that there is an entity loaded before rendering children. If you're implementing your own `EntityLayout` or `EntitySwitch` or something that operates outside or adjacent to them, then use `useAsyncEntity`. + - **DEPRECATION**: the `loading`, `error` and `refresh` properties that are returned from `useEntity` have been deprecated, and are available on `useAsyncEntity` instead. diff --git a/plugins/catalog-react/src/hooks/useEntity.tsx b/plugins/catalog-react/src/hooks/useEntity.tsx index 9becedfee8..e46ee680eb 100644 --- a/plugins/catalog-react/src/hooks/useEntity.tsx +++ b/plugins/catalog-react/src/hooks/useEntity.tsx @@ -182,7 +182,7 @@ export function useEntity(): UseEntityResponse { // eslint-disable-next-line no-console console.warn( - 'DEPRECATION: useEntity hook is being called outside of an EntityPage where the entity has not been loaded. If this is intentional, please use useAsyncEntity instead. This warning will be replaced with an error in future releases.', + 'DEPRECATION: useEntity hook is being called outside of an EntityLayout where the entity has not been loaded. If this is intentional, please use useAsyncEntity instead. This warning will be replaced with an error in future releases.', ); } diff --git a/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx b/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx index 634662f386..40c4f9e873 100644 --- a/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx +++ b/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx @@ -40,7 +40,7 @@ import { getEntityRelations, InspectEntityDialog, UnregisterEntityDialog, - useEntity, + useAsyncEntity, useEntityCompoundName, } from '@backstage/plugin-catalog-react'; import { Box, TabProps } from '@material-ui/core'; @@ -177,7 +177,7 @@ export const EntityLayout = (props: EntityLayoutProps) => { children, } = props; const { kind, namespace, name } = useEntityCompoundName(); - const { entity, loading, error } = useEntity(); + const { entity, loading, error } = useAsyncEntity(); const location = useLocation(); const routes = useElementFilter( children, @@ -190,7 +190,9 @@ export const EntityLayout = (props: EntityLayoutProps) => { }) .getElements() // all nodes, element data, maintain structure or not? .flatMap(({ props: elementProps }) => { - if (elementProps.if && entity && !elementProps.if(entity)) { + if (!entity) { + return []; + } else if (elementProps.if && !elementProps.if(entity)) { return []; } From 862e41623974165913fdd4244bd04fd7ed986c02 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 1 Mar 2022 13:34:38 +0100 Subject: [PATCH 051/150] catalog-backend: Remove entityRef from CatalogProcessorRelationResult Signed-off-by: Johan Haals --- .changeset/real-kids-hide.md | 5 +++++ plugins/catalog-backend/src/ingestion/processors/types.ts | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 .changeset/real-kids-hide.md diff --git a/.changeset/real-kids-hide.md b/.changeset/real-kids-hide.md new file mode 100644 index 0000000000..188a4061c9 --- /dev/null +++ b/.changeset/real-kids-hide.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +**Breaking**: Removed `entityRef` from `CatalogProcessorRelationResult`. The field is not used by the catalog and relation information is already available inside the `reation` property. diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/ingestion/processors/types.ts index 879b4d32a8..8a66d3eeb1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/types.ts @@ -193,7 +193,6 @@ export type CatalogProcessorEntityResult = { export type CatalogProcessorRelationResult = { type: 'relation'; relation: EntityRelationSpec; - entityRef?: string; }; /** @public */ From b19061a5471a3ff63574f934ab9021c6019a88ae Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 1 Mar 2022 13:47:25 +0100 Subject: [PATCH 052/150] update api report Signed-off-by: Johan Haals --- plugins/catalog-backend/api-report.md | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 9b4db418b3..604f92a4c3 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -347,7 +347,6 @@ export type CatalogProcessorParser = (options: { export type CatalogProcessorRelationResult = { type: 'relation'; relation: EntityRelationSpec; - entityRef?: string; }; // @public (undocumented) From e02960f30f31f78af1661e82bf812ca07726bdb7 Mon Sep 17 00:00:00 2001 From: Mike Lewis Date: Tue, 1 Mar 2022 12:56:42 +0000 Subject: [PATCH 053/150] catalog-backend: add alpha setup to package.json Signed-off-by: Mike Lewis --- plugins/catalog-backend/package.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 55c92c92be..be0772e199 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -9,7 +9,8 @@ "publishConfig": { "access": "public", "main": "dist/index.cjs.js", - "types": "dist/index.d.ts" + "types": "dist/index.d.ts", + "alphaTypes": "dist/index.alpha.d.ts" }, "backstage": { "role": "backend-plugin" @@ -25,7 +26,7 @@ ], "scripts": { "start": "backstage-cli package start", - "build": "backstage-cli package build", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", @@ -87,6 +88,7 @@ }, "files": [ "dist", + "alpha", "migrations/**/*.{js,d.ts}", "config.d.ts" ], From aa69677928adc0467a44b8f47589fec00e898b11 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Mar 2022 14:03:08 +0100 Subject: [PATCH 054/150] chore: fixing tests Signed-off-by: blam --- .../AllureReportComponent.test.tsx | 11 ++++++++++- plugins/todo/src/plugin.test.tsx | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/plugins/allure/src/components/AllureReportComponent/AllureReportComponent.test.tsx b/plugins/allure/src/components/AllureReportComponent/AllureReportComponent.test.tsx index 34d12aa032..e6355d861f 100644 --- a/plugins/allure/src/components/AllureReportComponent/AllureReportComponent.test.tsx +++ b/plugins/allure/src/components/AllureReportComponent/AllureReportComponent.test.tsx @@ -23,6 +23,7 @@ import { setupRequestMockHandlers, renderInTestApp, } from '@backstage/test-utils'; +import { EntityProvider } from '@backstage/plugin-catalog-react'; describe('ExampleComponent', () => { const server = setupServer(); @@ -39,7 +40,15 @@ describe('ExampleComponent', () => { it('should render', async () => { const rendered = await renderInTestApp( - + + + , ); expect(rendered.getByText('Missing Annotation')).toBeInTheDocument(); diff --git a/plugins/todo/src/plugin.test.tsx b/plugins/todo/src/plugin.test.tsx index 57b9be291d..847ab7f65c 100644 --- a/plugins/todo/src/plugin.test.tsx +++ b/plugins/todo/src/plugin.test.tsx @@ -19,6 +19,7 @@ import { Route } from 'react-router'; import { renderInTestApp, TestApiProvider } from '@backstage/test-utils'; import { todoPlugin, EntityTodoContent } from './plugin'; import { todoApiRef } from './api'; +import { EntityProvider } from '@backstage/plugin-catalog-react'; describe('todo', () => { it('should export plugin', () => { @@ -47,7 +48,15 @@ describe('todo', () => { ], ]} > - } /> + + } /> + , ); From dd7e34b4b8685389b31c5162dac3f38bc11c8ca8 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Mar 2022 14:04:15 +0100 Subject: [PATCH 055/150] chore: added additional changeset Signed-off-by: blam --- .changeset/tall-pillows-smash.md | 6 ++++++ plugins/todo/src/plugin.test.tsx | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changeset/tall-pillows-smash.md diff --git a/.changeset/tall-pillows-smash.md b/.changeset/tall-pillows-smash.md new file mode 100644 index 0000000000..3c1d18cbe3 --- /dev/null +++ b/.changeset/tall-pillows-smash.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-allure': patch +'@backstage/plugin-todo': patch +--- + +Fixing broken tests for the packages with the new `useEntity` change diff --git a/plugins/todo/src/plugin.test.tsx b/plugins/todo/src/plugin.test.tsx index 847ab7f65c..af4704c16f 100644 --- a/plugins/todo/src/plugin.test.tsx +++ b/plugins/todo/src/plugin.test.tsx @@ -50,7 +50,7 @@ describe('todo', () => { > Date: Tue, 1 Mar 2022 14:16:39 +0100 Subject: [PATCH 056/150] cli: install both v16 and v17 of @hot-loader/react-dom Signed-off-by: Patrik Oldsberg --- .changeset/poor-hounds-beam.md | 5 +++++ packages/cli/package.json | 3 ++- packages/cli/src/lib/bundler/config.ts | 7 +++++-- yarn.lock | 12 +++++++++++- 4 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 .changeset/poor-hounds-beam.md diff --git a/.changeset/poor-hounds-beam.md b/.changeset/poor-hounds-beam.md new file mode 100644 index 0000000000..925370c1b0 --- /dev/null +++ b/.changeset/poor-hounds-beam.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +The CLI now bundles both version 16 and 17 of the patched `@hot-loader/react-dom` dependency, and selects the appropriate one based on what version of `react-dom` is installed within the app. diff --git a/packages/cli/package.json b/packages/cli/package.json index b17d517ba1..e5551ce29e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -37,7 +37,8 @@ "@backstage/errors": "^0.2.2", "@backstage/release-manifests": "^0.0.2", "@backstage/types": "^0.1.3", - "@hot-loader/react-dom": "^17.0.2", + "@hot-loader/react-dom-v16": "npm:@hot-loader/react-dom@^16.0.2", + "@hot-loader/react-dom-v17": "npm:@hot-loader/react-dom@^17.0.2", "@manypkg/get-packages": "^1.1.3", "@octokit/request": "^5.4.12", "@rollup/plugin-commonjs": "^21.0.1", diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts index 3cfcafc005..fe64f23209 100644 --- a/packages/cli/src/lib/bundler/config.ts +++ b/packages/cli/src/lib/bundler/config.ts @@ -167,13 +167,16 @@ export async function createConfig( }), ); + // Detect and use the appropriate react-dom hot-loader patch based on what + // version of React is used within the target repo. const resolveAliases: Record = {}; try { // eslint-disable-next-line import/no-extraneous-dependencies const { version: reactDomVersion } = require('react-dom/package.json'); - // Only apply the alias for hook support if we're running with React 16 if (reactDomVersion.startsWith('16.')) { - resolveAliases['react-dom'] = '@hot-loader/react-dom'; + resolveAliases['react-dom'] = '@hot-loader/react-dom-v16'; + } else { + resolveAliases['react-dom'] = '@hot-loader/react-dom-v17'; } } catch (error) { console.warn(`WARNING: Failed to read react-dom version, ${error}`); diff --git a/yarn.lock b/yarn.lock index a6b27ffa82..3b14b3dcfa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2557,7 +2557,17 @@ dependencies: "@hapi/hoek" "^9.0.0" -"@hot-loader/react-dom@^17.0.2": +"@hot-loader/react-dom-v16@npm:@hot-loader/react-dom@^16.0.2": + version "16.14.0" + resolved "https://registry.npmjs.org/@hot-loader/react-dom/-/react-dom-16.14.0.tgz#3cfc64e40bb78fa623e59b582b8f09dcdaad648a" + integrity sha512-EN9czvcLsMYmSDo5yRKZOAq3ZGRlDpad1gPtX0NdMMomJXcPE3yFSeFzE94X/NjOaiSVimB7LuqPYpkWVaIi4Q== + dependencies: + loose-envify "^1.1.0" + object-assign "^4.1.1" + prop-types "^15.6.2" + scheduler "^0.19.1" + +"@hot-loader/react-dom-v17@npm:@hot-loader/react-dom@^17.0.2": version "17.0.2" resolved "https://registry.npmjs.org/@hot-loader/react-dom/-/react-dom-17.0.2.tgz#0b24e484093e8f97eb5c72bebdda44fc20bc8400" integrity sha512-G2RZrFhsQClS+bdDh/Ojpk3SgocLPUGnvnJDTQYnmKSSwXtU+Yh+8QMs+Ia3zaAvBiOSpIIDSUxuN69cvKqrWg== From da79aac2a68a349d893144f9dd2df3dc17da60c4 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Mar 2022 14:17:38 +0100 Subject: [PATCH 057/150] chore: add some beautiful changesets Signed-off-by: blam --- .changeset/ninety-kids-drop.md | 8 ++++++++ .changeset/spicy-onions-kiss.md | 5 +++++ 2 files changed, 13 insertions(+) create mode 100644 .changeset/ninety-kids-drop.md create mode 100644 .changeset/spicy-onions-kiss.md diff --git a/.changeset/ninety-kids-drop.md b/.changeset/ninety-kids-drop.md new file mode 100644 index 0000000000..ed5cf6330b --- /dev/null +++ b/.changeset/ninety-kids-drop.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-react': minor +--- + +Removed some previously deprecated `routeRefs` as follows: + +- **BREAKING**: Removed `entityRoute` in favor of `entityRouteRef`. +- **BREAKING**: Removed the previously deprecated `rootRoute` and `catalogRouteRef`. If you want to refer to the catalog index page from a public plugin you now need to use an `ExternalRouteRef` instead. For private plugins it is possible to take the shortcut of referring directly to `catalogPlugin.routes.indexPage` instead. diff --git a/.changeset/spicy-onions-kiss.md b/.changeset/spicy-onions-kiss.md new file mode 100644 index 0000000000..777e233b88 --- /dev/null +++ b/.changeset/spicy-onions-kiss.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +- Replaced usage of the deprecated and now removed `rootRoute` and `catalogRouteRef`s from the `catalog-react` package From e3c2bfef11b268981627d1ffebfcd513549ecac6 Mon Sep 17 00:00:00 2001 From: Mike Lewis Date: Tue, 1 Mar 2022 13:35:41 +0000 Subject: [PATCH 058/150] catalog-common: remove resourceType from catalogEntityCreatePermission The resourceType on permissions refers to the resource whose ref is expected to be passed along with the permission during authorization. This allows the permission-backend to make the decision based on characteristics of the resource. Since the entity being created by definition doesn't yet exist, it's not correct for this permission to include a resourceType. Signed-off-by: Mike Lewis --- .changeset/giant-taxis-drop.md | 5 +++++ plugins/catalog-common/src/permissions.ts | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 .changeset/giant-taxis-drop.md diff --git a/.changeset/giant-taxis-drop.md b/.changeset/giant-taxis-drop.md new file mode 100644 index 0000000000..41f8a91d40 --- /dev/null +++ b/.changeset/giant-taxis-drop.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-common': minor +--- + +Remove resourceType property from catalogEntityCreatePermission. Resource type refers to the type of resources whose resourceRefs should be passed along with authorize requests, to allow conditional responses for that resource type. Since creation does not correspond to an entity (as the entity does not exist at the time of authorization), the resourceRef should not be included on the permission. diff --git a/plugins/catalog-common/src/permissions.ts b/plugins/catalog-common/src/permissions.ts index 52c225e441..51b09bab7e 100644 --- a/plugins/catalog-common/src/permissions.ts +++ b/plugins/catalog-common/src/permissions.ts @@ -49,7 +49,6 @@ export const catalogEntityCreatePermission: Permission = { attributes: { action: 'create', }, - resourceType: RESOURCE_TYPE_CATALOG_ENTITY, }; /** From 83a83381b09890e0c2f6f7a6a5b4edcda7ae2982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sun, 27 Feb 2022 19:16:01 +0100 Subject: [PATCH 059/150] rearrange MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/lemon-needles-applaud.md | 5 ++ .changeset/long-weeks-thank.md | 5 ++ .changeset/old-waves-wash.md | 8 +++ .../software-catalog/external-integrations.md | 4 +- .../AwsOrganizationCloudAccountProcessor.ts | 4 +- .../src/processors/LdapOrgReaderProcessor.ts | 6 +- .../MicrosoftGraphOrgReaderProcessor.ts | 6 +- plugins/catalog-backend/api-report.md | 37 ++++++++-- plugins/catalog-backend/src/api/common.ts | 56 +++++++++++++++ .../results.ts => api/deprecatedResult.ts} | 36 +++++++--- plugins/catalog-backend/src/api/index.ts | 38 ++++++++++ .../src/api/processingResult.ts | 71 +++++++++++++++++++ .../processors/types.ts => api/processor.ts} | 19 +---- .../{providers/types.ts => api/provider.ts} | 2 +- plugins/catalog-backend/src/database/types.ts | 3 +- plugins/catalog-backend/src/index.ts | 3 +- .../src/ingestion/CatalogRules.test.ts | 2 +- .../src/ingestion/CatalogRules.ts | 2 +- .../catalog-backend/src/ingestion/index.ts | 2 - .../catalog-backend/src/ingestion/types.ts | 2 +- .../aws}/AwsS3DiscoveryProcessor.test.ts | 13 ++-- .../aws}/AwsS3DiscoveryProcessor.ts | 8 +-- .../aws/__fixtures__}/awsS3-mock-object.txt | 0 .../providers => modules/aws}/index.ts | 4 +- .../AzureDevOpsDiscoveryProcessor.test.ts | 6 +- .../azure}/AzureDevOpsDiscoveryProcessor.ts | 12 ++-- .../src/{providers => modules/azure}/index.ts | 8 +-- .../azure => modules/azure/lib}/azure.test.ts | 0 .../azure => modules/azure/lib}/azure.ts | 0 .../azure => modules/azure/lib}/index.ts | 0 .../BitbucketDiscoveryProcessor.test.ts | 12 ++-- .../bitbucket}/BitbucketDiscoveryProcessor.ts | 5 +- .../src/modules/bitbucket/index.ts | 18 +++++ .../lib}/BitbucketRepositoryParser.test.ts | 5 +- .../lib}/BitbucketRepositoryParser.ts | 31 ++++---- .../bitbucket/lib}/client.ts | 2 +- .../bitbucket/lib}/index.ts | 1 + .../bitbucket/lib}/types.ts | 0 .../codeowners}/CodeOwnersProcessor.test.ts | 2 +- .../codeowners}/CodeOwnersProcessor.ts | 5 +- .../src/modules/codeowners/index.ts | 17 +++++ .../codeowners/lib}/index.ts | 0 .../codeowners/lib}/read.test.ts | 0 .../codeowners/lib}/read.ts | 0 .../codeowners/lib}/resolve.test.ts | 0 .../codeowners/lib}/resolve.ts | 0 .../codeowners/lib}/scm.ts | 0 .../AnnotateLocationEntityProcessor.test.ts | 2 +- .../core}/AnnotateLocationEntityProcessor.ts | 6 +- .../AnnotateScmSlugEntityProcessor.test.ts | 2 +- .../core}/AnnotateScmSlugEntityProcessor.ts | 2 +- .../core}/BuiltinKindsEntityProcessor.test.ts | 0 .../core}/BuiltinKindsEntityProcessor.ts | 12 ++-- .../ConfigLocationEntityProvider.test.ts | 2 +- .../core}/ConfigLocationEntityProvider.ts | 6 +- .../core}/DefaultLocationStore.test.ts | 2 +- .../core}/DefaultLocationStore.ts | 10 +-- .../core}/FileReaderProcessor.test.ts | 10 ++- .../core}/FileReaderProcessor.ts | 8 +-- .../core}/LocationEntityProcessor.test.ts | 2 +- .../core}/LocationEntityProcessor.ts | 12 ++-- .../core}/PlaceholderProcessor.test.ts | 0 .../core}/PlaceholderProcessor.ts | 2 +- .../core}/StaticLocationProcessor.ts | 9 ++- .../core}/UrlReaderProcessor.test.ts | 4 +- .../core}/UrlReaderProcessor.ts | 8 +-- .../awsS3/awsS3-mock-object.txt | 1 + .../fileReaderProcessor/component.yaml | 0 .../fileReaderProcessor/dir/api.yaml | 0 .../__fixtures__/fileReaderProcessor/test.txt | 0 .../processors => modules/core}/index.ts | 17 +---- .../github}/GitHubOrgEntityProvider.test.ts | 8 ++- .../github}/GitHubOrgEntityProvider.ts | 7 +- .../github}/GithubDiscoveryProcessor.test.ts | 10 +-- .../github}/GithubDiscoveryProcessor.ts | 12 ++-- .../github}/GithubMultiOrgReaderProcessor.ts | 16 +++-- .../github}/GithubOrgReaderProcessor.test.ts | 3 +- .../github}/GithubOrgReaderProcessor.ts | 16 +++-- .../src/modules/github/index.ts | 21 ++++++ .../github/lib}/config.test.ts | 0 .../github => modules/github/lib}/config.ts | 0 .../github/lib}/github.test.ts | 0 .../github => modules/github/lib}/github.ts | 0 .../github => modules/github/lib}/index.ts | 0 .../github/lib}/util.test.ts | 1 + .../github => modules/github/lib}/util.ts | 1 + .../gitlab}/GitLabDiscoveryProcessor.test.ts | 4 +- .../gitlab}/GitLabDiscoveryProcessor.ts | 22 +++--- .../src/modules/gitlab/index.ts | 17 +++++ .../gitlab/lib}/client.test.ts | 2 +- .../gitlab => modules/gitlab/lib}/client.ts | 0 .../gitlab => modules/gitlab/lib}/index.ts | 0 .../gitlab => modules/gitlab/lib}/types.ts | 0 plugins/catalog-backend/src/modules/index.ts | 23 ++++++ .../processors => modules}/util/org.test.ts | 0 .../processors => modules}/util/org.ts | 0 .../processors => modules}/util/parse.test.ts | 21 +++--- .../processors => modules}/util/parse.ts | 12 ++-- ...faultCatalogProcessingOrchestrator.test.ts | 12 ++-- .../DefaultCatalogProcessingOrchestrator.ts | 22 +++--- .../processing/ProcessorCacheManager.test.ts | 2 +- .../src/processing/ProcessorCacheManager.ts | 3 +- .../processing/ProcessorOutputCollector.ts | 4 +- .../src/processing/connectEntityProviders.ts | 2 +- .../catalog-backend/src/processing/index.ts | 1 - .../catalog-backend/src/processing/types.ts | 25 +------ .../catalog-backend/src/processing/util.ts | 2 +- .../src/service/AuthorizedRefreshService.ts | 1 + .../src/service/CatalogBuilder.ts | 18 ++--- .../src/service/DefaultLocationService.ts | 1 + .../catalog-backend/src/util/conversion.ts | 2 +- .../processor/ScaffolderEntitiesProcessor.ts | 6 +- 112 files changed, 594 insertions(+), 280 deletions(-) create mode 100644 .changeset/lemon-needles-applaud.md create mode 100644 .changeset/long-weeks-thank.md create mode 100644 .changeset/old-waves-wash.md create mode 100644 plugins/catalog-backend/src/api/common.ts rename plugins/catalog-backend/src/{ingestion/processors/results.ts => api/deprecatedResult.ts} (69%) create mode 100644 plugins/catalog-backend/src/api/index.ts create mode 100644 plugins/catalog-backend/src/api/processingResult.ts rename plugins/catalog-backend/src/{ingestion/processors/types.ts => api/processor.ts} (92%) rename plugins/catalog-backend/src/{providers/types.ts => api/provider.ts} (96%) rename plugins/catalog-backend/src/{ingestion/processors => modules/aws}/AwsS3DiscoveryProcessor.test.ts (89%) rename plugins/catalog-backend/src/{ingestion/processors => modules/aws}/AwsS3DiscoveryProcessor.ts (92%) rename plugins/catalog-backend/src/{ingestion/processors/__fixtures__/fileReaderProcessor/awsS3 => modules/aws/__fixtures__}/awsS3-mock-object.txt (100%) rename plugins/catalog-backend/src/{ingestion/providers => modules/aws}/index.ts (83%) rename plugins/catalog-backend/src/{ingestion/processors => modules/azure}/AzureDevOpsDiscoveryProcessor.test.ts (98%) rename plugins/catalog-backend/src/{ingestion/processors => modules/azure}/AzureDevOpsDiscoveryProcessor.ts (95%) rename plugins/catalog-backend/src/{providers => modules/azure}/index.ts (79%) rename plugins/catalog-backend/src/{ingestion/processors/azure => modules/azure/lib}/azure.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/azure => modules/azure/lib}/azure.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/azure => modules/azure/lib}/index.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/bitbucket}/BitbucketDiscoveryProcessor.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/bitbucket}/BitbucketDiscoveryProcessor.ts (99%) create mode 100644 plugins/catalog-backend/src/modules/bitbucket/index.ts rename plugins/catalog-backend/src/{ingestion/processors/bitbucket => modules/bitbucket/lib}/BitbucketRepositoryParser.test.ts (94%) rename plugins/catalog-backend/src/{ingestion/processors/bitbucket => modules/bitbucket/lib}/BitbucketRepositoryParser.ts (62%) rename plugins/catalog-backend/src/{ingestion/processors/bitbucket => modules/bitbucket/lib}/client.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/bitbucket => modules/bitbucket/lib}/index.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors/bitbucket => modules/bitbucket/lib}/types.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/codeowners}/CodeOwnersProcessor.test.ts (98%) rename plugins/catalog-backend/src/{ingestion/processors => modules/codeowners}/CodeOwnersProcessor.ts (95%) create mode 100644 plugins/catalog-backend/src/modules/codeowners/index.ts rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/index.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/read.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/read.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/resolve.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/resolve.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/scm.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/AnnotateLocationEntityProcessor.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/AnnotateLocationEntityProcessor.ts (96%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/AnnotateScmSlugEntityProcessor.test.ts (98%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/AnnotateScmSlugEntityProcessor.ts (97%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/BuiltinKindsEntityProcessor.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/BuiltinKindsEntityProcessor.ts (97%) rename plugins/catalog-backend/src/{providers => modules/core}/ConfigLocationEntityProvider.test.ts (98%) rename plugins/catalog-backend/src/{providers => modules/core}/ConfigLocationEntityProvider.ts (90%) rename plugins/catalog-backend/src/{providers => modules/core}/DefaultLocationStore.test.ts (98%) rename plugins/catalog-backend/src/{providers => modules/core}/DefaultLocationStore.ts (93%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/FileReaderProcessor.test.ts (94%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/FileReaderProcessor.ts (92%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/LocationEntityProcessor.test.ts (98%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/LocationEntityProcessor.ts (92%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/PlaceholderProcessor.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/PlaceholderProcessor.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/StaticLocationProcessor.ts (91%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/UrlReaderProcessor.test.ts (98%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/UrlReaderProcessor.ts (95%) create mode 100644 plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/__fixtures__/fileReaderProcessor/component.yaml (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/__fixtures__/fileReaderProcessor/dir/api.yaml (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/__fixtures__/fileReaderProcessor/test.txt (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/index.ts (63%) rename plugins/catalog-backend/src/{ingestion/providers => modules/github}/GitHubOrgEntityProvider.test.ts (97%) rename plugins/catalog-backend/src/{ingestion/providers => modules/github}/GitHubOrgEntityProvider.ts (97%) rename plugins/catalog-backend/src/{ingestion/processors => modules/github}/GithubDiscoveryProcessor.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/github}/GithubDiscoveryProcessor.ts (96%) rename plugins/catalog-backend/src/{ingestion/processors => modules/github}/GithubMultiOrgReaderProcessor.ts (95%) rename plugins/catalog-backend/src/{ingestion/processors => modules/github}/GithubOrgReaderProcessor.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/github}/GithubOrgReaderProcessor.ts (92%) create mode 100644 plugins/catalog-backend/src/modules/github/index.ts rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/config.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/config.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/github.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/github.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/index.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/util.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/util.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/gitlab}/GitLabDiscoveryProcessor.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/gitlab}/GitLabDiscoveryProcessor.ts (92%) create mode 100644 plugins/catalog-backend/src/modules/gitlab/index.ts rename plugins/catalog-backend/src/{ingestion/processors/gitlab => modules/gitlab/lib}/client.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/gitlab => modules/gitlab/lib}/client.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/gitlab => modules/gitlab/lib}/index.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/gitlab => modules/gitlab/lib}/types.ts (100%) create mode 100644 plugins/catalog-backend/src/modules/index.ts rename plugins/catalog-backend/src/{ingestion/processors => modules}/util/org.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules}/util/org.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules}/util/parse.test.ts (91%) rename plugins/catalog-backend/src/{ingestion/processors => modules}/util/parse.ts (87%) diff --git a/.changeset/lemon-needles-applaud.md b/.changeset/lemon-needles-applaud.md new file mode 100644 index 0000000000..a77d3fd33c --- /dev/null +++ b/.changeset/lemon-needles-applaud.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +**DEPRECATED**: The `results` export, and instead adding `processingResult` with the same shape and purpose. diff --git a/.changeset/long-weeks-thank.md b/.changeset/long-weeks-thank.md new file mode 100644 index 0000000000..a6c3a0719f --- /dev/null +++ b/.changeset/long-weeks-thank.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Internal restructuring to collect the various provider files in a `modules` folder while waiting to be externalized diff --git a/.changeset/old-waves-wash.md b/.changeset/old-waves-wash.md new file mode 100644 index 0000000000..34ba0760b5 --- /dev/null +++ b/.changeset/old-waves-wash.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-aws': patch +'@backstage/plugin-catalog-backend-module-ldap': patch +'@backstage/plugin-catalog-backend-module-msgraph': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Use the new `processingResult` export from the catalog backend diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index 59caf9475c..9f592e7cca 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -57,7 +57,7 @@ The recommended way of instantiating the catalog backend classes is to use the `CatalogBuilder`, as illustrated in the [example backend here](https://github.com/backstage/backstage/blob/master/packages/backend/src/plugins/catalog.ts). We will create a new -[`EntityProvider`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/providers/types.ts) +[`EntityProvider`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/api/provider.ts) subclass that can be added to this catalog builder. Let's make a simple provider that can refresh a set of entities based on a @@ -355,7 +355,7 @@ The recommended way of instantiating the catalog backend classes is to use the `CatalogBuilder`, as illustrated in the [example backend here](https://github.com/backstage/backstage/blob/master/packages/backend/src/plugins/catalog.ts). We will create a new -[`CatalogProcessor`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/ingestion/processors/types.ts) +[`CatalogProcessor`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/api/processor.ts) subclass that can be added to this catalog builder. It is up to you where you put the code for this new processor class. For quick diff --git a/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts b/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts index e5e72d5fa8..1018e49a41 100644 --- a/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts +++ b/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts @@ -20,7 +20,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; import AWS, { Credentials, Organizations } from 'aws-sdk'; import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations'; @@ -112,7 +112,7 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { return true; }) .forEach(entity => { - emit(results.entity(location, entity)); + emit(processingResult.entity(location, entity)); }); return true; diff --git a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts index 6c31d5a27d..bec24a1744 100644 --- a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts @@ -28,7 +28,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; /** @@ -119,10 +119,10 @@ export class LdapOrgReaderProcessor implements CatalogProcessor { // Done! for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } for (const user of users) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 4db3a9b1a0..942bde19eb 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -19,7 +19,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import { @@ -125,10 +125,10 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { // Done! for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } for (const user of users) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index c9e08964b6..00e09d1169 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -524,7 +524,7 @@ export type EntitiesSearchFilter = { values?: string[]; }; -// @public (undocumented) +// @public @deprecated (undocumented) function entity( atLocation: LocationSpec, newEntity: Entity, @@ -641,7 +641,7 @@ export class FileReaderProcessor implements CatalogProcessor { ): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function generalError( atLocation: LocationSpec, message: string, @@ -776,13 +776,13 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { ): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function inputError( atLocation: LocationSpec, message: string, ): CatalogProcessorResult; -// @public (undocumented) +// @public @deprecated (undocumented) function location_2( newLocation: LocationSpec, optional?: boolean, @@ -876,7 +876,7 @@ export interface LocationStore { listLocations(): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function notFoundError( atLocation: LocationSpec, message: string, @@ -963,6 +963,31 @@ export type PlaceholderResolverResolveUrl = ( base: string, ) => string; +// @public +export const processingResult: Readonly<{ + readonly notFoundError: ( + atLocation: LocationSpec, + message: string, + ) => CatalogProcessorResult; + readonly inputError: ( + atLocation: LocationSpec, + message: string, + ) => CatalogProcessorResult; + readonly generalError: ( + atLocation: LocationSpec, + message: string, + ) => CatalogProcessorResult; + readonly location: ( + newLocation: LocationSpec, + optional?: boolean | undefined, + ) => CatalogProcessorResult; + readonly entity: ( + atLocation: LocationSpec, + newEntity: Entity, + ) => CatalogProcessorResult; + readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult; +}>; + // @public export type RecursivePartial = { [P in keyof T]?: T[P] extends (infer U)[] @@ -986,7 +1011,7 @@ export interface RefreshService { refresh(options: RefreshOptions): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function relation(spec: EntityRelationSpec): CatalogProcessorResult; declare namespace results { diff --git a/plugins/catalog-backend/src/api/common.ts b/plugins/catalog-backend/src/api/common.ts new file mode 100644 index 0000000000..1fd09aec32 --- /dev/null +++ b/plugins/catalog-backend/src/api/common.ts @@ -0,0 +1,56 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityName } from '@backstage/catalog-model'; + +/** + * Holds the entity location information. + * + * @remarks + * + * `presence` flag: when using repo importer plugin, location is being created before the component yaml file is merged to the main branch. + * This flag is then set to indicate that the file can be not present. + * default value: 'required'. + * + * @public + */ +export type LocationSpec = { + type: string; + target: string; + presence?: 'optional' | 'required'; +}; + +/** + * Holds the relation data for entities. + * + * @public + */ +export type EntityRelationSpec = { + /** + * The source entity of this relation. + */ + source: EntityName; + + /** + * The type of the relation. + */ + type: string; + + /** + * The target entity of this relation. + */ + target: EntityName; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/results.ts b/plugins/catalog-backend/src/api/deprecatedResult.ts similarity index 69% rename from plugins/catalog-backend/src/ingestion/processors/results.ts rename to plugins/catalog-backend/src/api/deprecatedResult.ts index 01583678df..ecacc62c65 100644 --- a/plugins/catalog-backend/src/ingestion/processors/results.ts +++ b/plugins/catalog-backend/src/api/deprecatedResult.ts @@ -16,10 +16,15 @@ import { InputError, NotFoundError } from '@backstage/errors'; import { Entity } from '@backstage/catalog-model'; -import { CatalogProcessorResult, LocationSpec } from './types'; -import { EntityRelationSpec } from '../../processing/types'; +import { CatalogProcessorResult } from './processor'; +import { EntityRelationSpec, LocationSpec } from './common'; -/** @public */ +// NOTE: This entire file is deprecated and should be eventually removed along with the `result` export + +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function notFoundError( atLocation: LocationSpec, message: string, @@ -31,7 +36,10 @@ export function notFoundError( }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function inputError( atLocation: LocationSpec, message: string, @@ -43,7 +51,10 @@ export function inputError( }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function generalError( atLocation: LocationSpec, message: string, @@ -51,7 +62,10 @@ export function generalError( return { type: 'error', location: atLocation, error: new Error(message) }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function location( newLocation: LocationSpec, optional?: boolean, @@ -59,7 +73,10 @@ export function location( return { type: 'location', location: newLocation, optional }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function entity( atLocation: LocationSpec, newEntity: Entity, @@ -67,7 +84,10 @@ export function entity( return { type: 'entity', location: atLocation, entity: newEntity }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function relation(spec: EntityRelationSpec): CatalogProcessorResult { return { type: 'relation', relation: spec }; } diff --git a/plugins/catalog-backend/src/api/index.ts b/plugins/catalog-backend/src/api/index.ts new file mode 100644 index 0000000000..3f1c627743 --- /dev/null +++ b/plugins/catalog-backend/src/api/index.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as results from './deprecatedResult'; + +export { results }; + +export { processingResult } from './processingResult'; +export type { EntityRelationSpec, LocationSpec } from './common'; +export type { + CatalogProcessor, + CatalogProcessorParser, + CatalogProcessorCache, + CatalogProcessorEmit, + CatalogProcessorLocationResult, + CatalogProcessorEntityResult, + CatalogProcessorRelationResult, + CatalogProcessorErrorResult, + CatalogProcessorResult, +} from './processor'; +export type { + EntityProvider, + EntityProviderConnection, + EntityProviderMutation, +} from './provider'; diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts new file mode 100644 index 0000000000..5ad908dc8b --- /dev/null +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError, NotFoundError } from '@backstage/errors'; +import { Entity } from '@backstage/catalog-model'; +import { CatalogProcessorResult } from './processor'; +import { EntityRelationSpec, LocationSpec } from './common'; + +/** + * Factory functions for the standard processing result types. + * + * @public + */ +export const processingResult = Object.freeze({ + notFoundError( + atLocation: LocationSpec, + message: string, + ): CatalogProcessorResult { + return { + type: 'error', + location: atLocation, + error: new NotFoundError(message), + }; + }, + + inputError( + atLocation: LocationSpec, + message: string, + ): CatalogProcessorResult { + return { + type: 'error', + location: atLocation, + error: new InputError(message), + }; + }, + + generalError( + atLocation: LocationSpec, + message: string, + ): CatalogProcessorResult { + return { type: 'error', location: atLocation, error: new Error(message) }; + }, + + location( + newLocation: LocationSpec, + optional?: boolean, + ): CatalogProcessorResult { + return { type: 'location', location: newLocation, optional }; + }, + + entity(atLocation: LocationSpec, newEntity: Entity): CatalogProcessorResult { + return { type: 'entity', location: atLocation, entity: newEntity }; + }, + + relation(spec: EntityRelationSpec): CatalogProcessorResult { + return { type: 'relation', relation: spec }; + }, +} as const); diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/api/processor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/types.ts rename to plugins/catalog-backend/src/api/processor.ts index 9b3bb05e2d..12a008edc7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -16,24 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; -import { EntityRelationSpec } from '../../processing/types'; - -/** - * Holds the entity location information. - * - * @remarks - * - * `presence` flag: when using repo importer plugin, location is being created before the component yaml file is merged to the main branch. - * This flag is then set to indicate that the file can be not present. - * default value: 'required'. - * - * @public - */ -export type LocationSpec = { - type: string; - target: string; - presence?: 'optional' | 'required'; -}; +import { EntityRelationSpec, LocationSpec } from './common'; /** * @public diff --git a/plugins/catalog-backend/src/providers/types.ts b/plugins/catalog-backend/src/api/provider.ts similarity index 96% rename from plugins/catalog-backend/src/providers/types.ts rename to plugins/catalog-backend/src/api/provider.ts index 226c021e3c..fa7659ee32 100644 --- a/plugins/catalog-backend/src/providers/types.ts +++ b/plugins/catalog-backend/src/api/provider.ts @@ -44,7 +44,7 @@ export interface EntityProviderConnection { * @public */ export interface EntityProvider { - /** Unique name provider name used internally for caching. */ + /** Unique provider name used internally for caching. */ getProviderName(): string; /** Connect is called upon initialization by the catalog engine. */ connect(connection: EntityProviderConnection): Promise; diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 08201323ea..1affb14b45 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -17,7 +17,8 @@ import { Entity } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/types'; import { DateTime } from 'luxon'; -import { DeferredEntity, EntityRelationSpec } from '../processing/types'; +import { EntityRelationSpec } from '../api'; +import { DeferredEntity } from '../processing/types'; /** * An abstraction for transactions of the underlying database technology. diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index efdaa4b8c5..4c1e753300 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -20,11 +20,12 @@ * @packageDocumentation */ +export * from './api'; export * from './catalog'; export * from './ingestion'; +export * from './modules'; export * from './search'; export * from './util'; export * from './processing'; -export * from './providers'; export * from './service'; export * from './permissions'; diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index d29a445a8c..13430f51dd 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -17,7 +17,7 @@ import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { DefaultCatalogRulesEnforcer } from './CatalogRules'; -import { LocationSpec } from './processors'; +import { LocationSpec } from '../api'; const entity = { user: { diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index b4ad394168..e4bfdb8d0f 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -17,7 +17,7 @@ import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import path from 'path'; -import { LocationSpec } from './processors'; +import { LocationSpec } from '../api'; /** * Rules to apply to catalog entities. diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index fcb634914b..1f4729b041 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -16,8 +16,6 @@ export { DefaultCatalogRulesEnforcer } from './CatalogRules'; export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules'; -export * from './processors'; -export * from './providers'; export type { AnalyzeLocationEntityField, AnalyzeLocationExistingEntity, diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index eccf583e4b..37d12d12b1 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -16,7 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { RecursivePartial } from '../util/RecursivePartial'; -import { LocationSpec } from './processors'; +import { LocationSpec } from '../api'; /** @public */ export type LocationAnalyzer = { diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts similarity index 89% rename from plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts index 064a2ad724..249de71591 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts @@ -13,11 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; -import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types'; -import { defaultEntityDataParser } from './util/parse'; +import { + CatalogProcessorEntityResult, + CatalogProcessorResult, +} from '../../api'; +import { defaultEntityDataParser } from '../util/parse'; import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; @@ -36,10 +40,7 @@ AWSMock.mock( 'getObject', Buffer.from( require('fs').readFileSync( - path.resolve( - __dirname, - '__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt', - ), + path.resolve(__dirname, '__fixtures__/awsS3-mock-object.txt'), ), ), ); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts index 0dc44cc157..737f8f7c2f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts @@ -17,13 +17,13 @@ import { UrlReader } from '@backstage/backend-common'; import { isError } from '@backstage/errors'; import limiterFactory from 'p-limit'; -import * as result from './results'; import { CatalogProcessor, CatalogProcessorEmit, CatalogProcessorParser, LocationSpec, -} from './types'; + processingResult, +} from '../../api'; /** @public */ export class AwsS3DiscoveryProcessor implements CatalogProcessor { @@ -58,10 +58,10 @@ export class AwsS3DiscoveryProcessor implements CatalogProcessor { if (isError(error) && error.name === 'NotFoundError') { if (!optional) { - emit(result.notFoundError(location, message)); + emit(processingResult.notFoundError(location, message)); } } else { - emit(result.generalError(location, message)); + emit(processingResult.generalError(location, message)); } } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt b/plugins/catalog-backend/src/modules/aws/__fixtures__/awsS3-mock-object.txt similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt rename to plugins/catalog-backend/src/modules/aws/__fixtures__/awsS3-mock-object.txt diff --git a/plugins/catalog-backend/src/ingestion/providers/index.ts b/plugins/catalog-backend/src/modules/aws/index.ts similarity index 83% rename from plugins/catalog-backend/src/ingestion/providers/index.ts rename to plugins/catalog-backend/src/modules/aws/index.ts index cd1bc5cb36..9477d4104d 100644 --- a/plugins/catalog-backend/src/ingestion/providers/index.ts +++ b/plugins/catalog-backend/src/modules/aws/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,4 +14,4 @@ * limitations under the License. */ -export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; +export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.test.ts index a406feaa74..450a0482ed 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.test.ts @@ -16,14 +16,14 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { codeSearch } from './azure'; +import { codeSearch } from './lib'; import { AzureDevOpsDiscoveryProcessor, parseUrl, } from './AzureDevOpsDiscoveryProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; -jest.mock('./azure'); +jest.mock('./lib'); const mockCodeSearch = codeSearch as jest.MockedFunction; describe('AzureDevOpsDiscoveryProcessor', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.ts index bc74621f1d..8e36e810b8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.ts @@ -20,9 +20,13 @@ import { ScmIntegrations, } from '@backstage/integration'; import { Logger } from 'winston'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { codeSearch } from './azure'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { codeSearch } from './lib'; /** * Extracts repositories out of an Azure DevOps org. @@ -102,7 +106,7 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { for (const file of files) { emit( - results.location({ + processingResult.location({ type: 'url', target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`, // Not all locations may actually exist, since the user defined them as a wildcard pattern. diff --git a/plugins/catalog-backend/src/providers/index.ts b/plugins/catalog-backend/src/modules/azure/index.ts similarity index 79% rename from plugins/catalog-backend/src/providers/index.ts rename to plugins/catalog-backend/src/modules/azure/index.ts index 3b4a1c4b7e..9c76336804 100644 --- a/plugins/catalog-backend/src/providers/index.ts +++ b/plugins/catalog-backend/src/modules/azure/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,4 @@ * limitations under the License. */ -export type { - EntityProvider, - EntityProviderConnection, - EntityProviderMutation, -} from './types'; +export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts b/plugins/catalog-backend/src/modules/azure/lib/azure.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts rename to plugins/catalog-backend/src/modules/azure/lib/azure.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts b/plugins/catalog-backend/src/modules/azure/lib/azure.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/azure/azure.ts rename to plugins/catalog-backend/src/modules/azure/lib/azure.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/index.ts b/plugins/catalog-backend/src/modules/azure/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/azure/index.ts rename to plugins/catalog-backend/src/modules/azure/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.test.ts index 4a261386c0..857f999a70 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.test.ts @@ -13,18 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger } from '@backstage/backend-common'; import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; import { ConfigReader } from '@backstage/config'; -import { - BitbucketRepository20, - PagedResponse, - PagedResponse20, -} from './bitbucket'; -import { LocationSpec } from './types'; -import { results } from './index'; import { RequestHandler, rest } from 'msw'; import { setupServer } from 'msw/node'; +import { BitbucketRepository20, PagedResponse, PagedResponse20 } from './lib'; +import { LocationSpec, processingResult } from '../../api'; const server = setupServer(); @@ -755,7 +751,7 @@ describe('BitbucketDiscoveryProcessor', () => { }), { parser: async function* customRepositoryParser({}) { - yield results.location({ + yield processingResult.location({ type: 'custom-location-type', target: 'custom-target', presence: 'optional', diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.ts index 25f52b7dd7..daf24e2427 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Logger } from 'winston'; import { Config } from '@backstage/config'; @@ -28,13 +29,13 @@ import { paginated20, BitbucketRepository, BitbucketRepository20, -} from './bitbucket'; +} from './lib'; import { CatalogProcessor, CatalogProcessorEmit, CatalogProcessorResult, LocationSpec, -} from './types'; +} from '../../api'; const DEFAULT_BRANCH = 'master'; const DEFAULT_CATALOG_LOCATION = '/catalog-info.yaml'; diff --git a/plugins/catalog-backend/src/modules/bitbucket/index.ts b/plugins/catalog-backend/src/modules/bitbucket/index.ts new file mode 100644 index 0000000000..0e39083520 --- /dev/null +++ b/plugins/catalog-backend/src/modules/bitbucket/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; +export type { BitbucketRepositoryParser } from './lib'; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.test.ts similarity index 94% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.test.ts index e010865041..9f2d8b2daa 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.test.ts @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import { processingResult } from '../../../api'; import { defaultRepositoryParser } from './BitbucketRepositoryParser'; -import { results } from '../index'; describe('BitbucketRepositoryParser', () => { describe('defaultRepositoryParser', () => { @@ -23,7 +24,7 @@ describe('BitbucketRepositoryParser', () => { 'https://bitbucket.mycompany.com/projects/project-key/repos/repo-slug/browse'; const path = '/catalog-info.yaml'; const expected = [ - results.location({ + processingResult.location({ type: 'url', target: `${browseUrl}${path}`, presence: 'optional', diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.ts similarity index 62% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.ts index 6671735f57..c72e560c21 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.ts @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { CatalogProcessorResult } from '../types'; -import { results } from '../index'; -import { Logger } from 'winston'; + import { BitbucketIntegration } from '@backstage/integration'; +import { Logger } from 'winston'; +import { CatalogProcessorResult, processingResult } from '../../../api'; /** * @public @@ -28,17 +28,14 @@ export type BitbucketRepositoryParser = (options: { logger: Logger; }) => AsyncIterable; -export const defaultRepositoryParser = async function* defaultRepositoryParser({ - target, -}: { - target: string; -}) { - yield results.location({ - type: 'url', - target: target, - // Not all locations may actually exist, since the user defined them as a wildcard pattern. - // Thus, we emit them as optional and let the downstream processor find them while not outputting - // an error if it couldn't. - presence: 'optional', - }); -}; +export const defaultRepositoryParser = + async function* defaultRepositoryParser(options: { target: string }) { + yield processingResult.location({ + type: 'url', + target: options.target, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + presence: 'optional', + }); + }; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/client.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/client.ts index 0c132bb61b..65cd28d5ab 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/client.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import fetch from 'node-fetch'; +import fetch from 'node-fetch'; import { BitbucketIntegrationConfig, getBitbucketRequestOptions, diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/index.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/index.ts index 4b28c7f115..a819bb3c64 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/index.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + export { BitbucketClient, paginated, paginated20 } from './client'; export { defaultRepositoryParser } from './BitbucketRepositoryParser'; export type { PagedResponse, PagedResponse20 } from './client'; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/types.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/types.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts rename to plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.test.ts index 8d6d1f8ff7..8cf250de2a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { CodeOwnersProcessor } from './CodeOwnersProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; const mockCodeOwnersText = () => ` * @acme/team-foo @acme/team-bar diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts rename to plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.ts index 93e3c1618a..6f6aee7161 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts +++ b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.ts @@ -22,11 +22,10 @@ import { ScmIntegrations, } from '@backstage/integration'; import { Logger } from 'winston'; -import { findCodeOwnerByTarget } from './codeowners'; -import { CatalogProcessor, LocationSpec } from './types'; +import { CatalogProcessor, LocationSpec } from '../../api'; +import { findCodeOwnerByTarget } from './lib'; const ALLOWED_KINDS = ['API', 'Component', 'Domain', 'Resource', 'System']; - const ALLOWED_LOCATION_TYPES = ['url']; /** @public */ diff --git a/plugins/catalog-backend/src/modules/codeowners/index.ts b/plugins/catalog-backend/src/modules/codeowners/index.ts new file mode 100644 index 0000000000..6b7569ff07 --- /dev/null +++ b/plugins/catalog-backend/src/modules/codeowners/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { CodeOwnersProcessor } from './CodeOwnersProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/index.ts b/plugins/catalog-backend/src/modules/codeowners/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/index.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/read.test.ts b/plugins/catalog-backend/src/modules/codeowners/lib/read.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/read.test.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/read.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts b/plugins/catalog-backend/src/modules/codeowners/lib/read.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/read.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.test.ts b/plugins/catalog-backend/src/modules/codeowners/lib/resolve.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.test.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/resolve.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.ts b/plugins/catalog-backend/src/modules/codeowners/lib/resolve.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/resolve.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/scm.ts b/plugins/catalog-backend/src/modules/codeowners/lib/scm.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/scm.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/scm.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.test.ts index 5f5b20d1c2..3911f7f743 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.test.ts @@ -17,7 +17,7 @@ import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; import { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; describe('AnnotateLocationEntityProcessor', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.ts similarity index 96% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.ts index b8c31ed5a2..4af0b8cf61 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.ts @@ -25,7 +25,11 @@ import { } from '@backstage/catalog-model'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { identity, merge, pickBy } from 'lodash'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, +} from '../../api'; /** @public */ export class AnnotateLocationEntityProcessor implements CatalogProcessor { diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.test.ts index 311dd13789..1e553a44f8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.test.ts @@ -16,7 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; describe('AnnotateScmSlugEntityProcessor', () => { describe('github', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.ts index 4d3b3aff6c..e44db4d44d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.ts @@ -21,7 +21,7 @@ import { } from '@backstage/integration'; import parseGitUrl from 'git-url-parse'; import { identity, merge, pickBy } from 'lodash'; -import { CatalogProcessor, LocationSpec } from './types'; +import { CatalogProcessor, LocationSpec } from '../../api'; const GITHUB_ACTIONS_ANNOTATION = 'github.com/project-slug'; diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts index 89b0bb1bb5..548cdf3115 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts @@ -52,8 +52,12 @@ import { TemplateEntityV1beta2, templateEntityV1beta2Validator, } from '@backstage/plugin-scaffolder-common'; -import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; /** @public */ export class BuiltinKindsEntityProcessor implements CatalogProcessor { @@ -107,7 +111,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { for (const target of [targets].flat()) { const targetRef = parseEntityRef(target, context); emit( - result.relation({ + processingResult.relation({ source: selfRef, type: outgoingRelation, target: { @@ -118,7 +122,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { }), ); emit( - result.relation({ + processingResult.relation({ source: { kind: targetRef.kind, namespace: targetRef.namespace, diff --git a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.test.ts similarity index 98% rename from plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts rename to plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.test.ts index 0663ad8d04..d61342e3d0 100644 --- a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts +++ b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.test.ts @@ -17,7 +17,7 @@ import { ConfigReader } from '@backstage/config'; import path from 'path'; import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider'; -import { EntityProviderConnection } from './types'; +import { EntityProviderConnection } from '../../api'; describe('ConfigLocationEntityProvider', () => { it('should apply mutation with the correct paths in the config', async () => { diff --git a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.ts similarity index 90% rename from plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts rename to plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.ts index 389cd8d9ff..9529a5b608 100644 --- a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts +++ b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.ts @@ -16,9 +16,9 @@ import { Config } from '@backstage/config'; import path from 'path'; -import { getEntityLocationRef } from '../processing/util'; -import { EntityProvider, EntityProviderConnection } from './types'; -import { locationSpecToLocationEntity } from '../util/conversion'; +import { getEntityLocationRef } from '../../processing/util'; +import { EntityProvider, EntityProviderConnection } from '../../api'; +import { locationSpecToLocationEntity } from '../../util/conversion'; export class ConfigLocationEntityProvider implements EntityProvider { constructor(private readonly config: Config) {} diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts similarity index 98% rename from plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts rename to plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts index b9d129de5f..dfbc37f6c9 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts @@ -15,7 +15,7 @@ */ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { v4 as uuid } from 'uuid'; -import { applyDatabaseMigrations } from '../database/migrations'; +import { applyDatabaseMigrations } from '../../database/migrations'; import { DefaultLocationStore } from './DefaultLocationStore'; describe('DefaultLocationStore', () => { diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.ts similarity index 93% rename from plugins/catalog-backend/src/providers/DefaultLocationStore.ts rename to plugins/catalog-backend/src/modules/core/DefaultLocationStore.ts index 9d5971508f..d637f4eda0 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.ts @@ -18,11 +18,11 @@ import { Location } from '@backstage/catalog-client'; import { ConflictError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; -import { DbLocationsRow } from '../database/tables'; -import { getEntityLocationRef } from '../processing/util'; -import { EntityProvider, EntityProviderConnection } from './types'; -import { locationSpecToLocationEntity } from '../util/conversion'; -import { LocationInput, LocationStore } from '../service'; +import { DbLocationsRow } from '../../database/tables'; +import { getEntityLocationRef } from '../../processing/util'; +import { EntityProvider, EntityProviderConnection } from '../../api'; +import { locationSpecToLocationEntity } from '../../util/conversion'; +import { LocationInput, LocationStore } from '../../service'; export class DefaultLocationStore implements LocationStore, EntityProvider { private _connection: EntityProviderConnection | undefined; diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts similarity index 94% rename from plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index c2cce66bc6..a3eb5e4542 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -19,12 +19,16 @@ import { CatalogProcessorEntityResult, CatalogProcessorErrorResult, CatalogProcessorResult, -} from './types'; +} from '../../api'; import path from 'path'; -import { defaultEntityDataParser } from './util/parse'; +import { defaultEntityDataParser } from '../util/parse'; describe('FileReaderProcessor', () => { - const fixturesRoot = path.join(__dirname, '__fixtures__/fileReaderProcessor'); + const fixturesRoot = path.join( + __dirname, + '__fixtures__', + 'fileReaderProcessor', + ); it('should load from file', async () => { const processor = new FileReaderProcessor(); diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts rename to plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 996beff24a..66c6479fa7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -18,13 +18,13 @@ import fs from 'fs-extra'; import g from 'glob'; import path from 'path'; import { promisify } from 'util'; -import * as result from './results'; import { CatalogProcessor, CatalogProcessorEmit, CatalogProcessorParser, LocationSpec, -} from './types'; + processingResult, +} from '../../api'; const glob = promisify(g); @@ -65,11 +65,11 @@ export class FileReaderProcessor implements CatalogProcessor { } } else if (!optional) { const message = `${location.type} ${location.target} does not exist`; - emit(result.notFoundError(location, message)); + emit(processingResult.notFoundError(location, message)); } } catch (e) { const message = `${location.type} ${location.target} could not be read, ${e}`; - emit(result.generalError(location, message)); + emit(processingResult.generalError(location, message)); } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/LocationEntityProcessor.test.ts index 83f1ca1925..92ba1fcebf 100644 --- a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.test.ts @@ -21,7 +21,7 @@ import { } from '@backstage/integration'; import path from 'path'; import { toAbsoluteUrl } from './LocationEntityProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; describe('LocationEntityProcessor', () => { describe('toAbsoluteUrl', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/LocationEntityProcessor.ts index 533a366f57..c168f54bf1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.ts @@ -17,8 +17,12 @@ import { Entity, LocationEntity } from '@backstage/catalog-model'; import { ScmIntegrationRegistry } from '@backstage/integration'; import path from 'path'; -import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { + processingResult, + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, +} from '../../api'; export function toAbsoluteUrl( integrations: ScmIntegrationRegistry, @@ -62,7 +66,7 @@ export class LocationEntityProcessor implements CatalogProcessor { const type = locationEntity.spec.type || location.type; if (type === 'file' && location.target.endsWith(path.sep)) { emit( - result.inputError( + processingResult.inputError( location, `LocationEntityProcessor cannot handle ${type} type location with target ${location.target} that ends with a path separator`, ), @@ -83,7 +87,7 @@ export class LocationEntityProcessor implements CatalogProcessor { location, maybeRelativeTarget, ); - emit(result.location({ type, target })); + emit(processingResult.location({ type, target })); } } diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts rename to plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index 0d572a024d..ddd5f951db 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -19,7 +19,7 @@ import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; import { ScmIntegrationRegistry } from '@backstage/integration'; import yaml from 'yaml'; -import { CatalogProcessor, LocationSpec } from './types'; +import { CatalogProcessor, LocationSpec } from '../../api'; /** @public */ export type PlaceholderResolverRead = (url: string) => Promise; diff --git a/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts b/plugins/catalog-backend/src/modules/core/StaticLocationProcessor.ts similarity index 91% rename from plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts rename to plugins/catalog-backend/src/modules/core/StaticLocationProcessor.ts index 1c2c27f425..e22d011695 100644 --- a/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/StaticLocationProcessor.ts @@ -15,8 +15,11 @@ */ import { Config } from '@backstage/config'; -import * as result from './results'; -import { CatalogProcessorEmit, LocationSpec } from './types'; +import { + processingResult, + CatalogProcessorEmit, + LocationSpec, +} from '../../api'; /** * @deprecated no longer in use, replaced by the ConfigLocationEntityProvider. @@ -48,7 +51,7 @@ export class StaticLocationProcessor implements StaticLocationProcessor { } for (const staticLocation of this.staticLocations) { - emit(result.location(staticLocation)); + emit(processingResult.location(staticLocation)); } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index b7dc0084f2..a96d392c52 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -28,9 +28,9 @@ import { CatalogProcessorEntityResult, CatalogProcessorErrorResult, CatalogProcessorResult, -} from './types'; +} from '../../api'; +import { defaultEntityDataParser } from '../util/parse'; import { UrlReaderProcessor } from './UrlReaderProcessor'; -import { defaultEntityDataParser } from './util/parse'; describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost'; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts rename to plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index c2be454bcf..7f27bcd19d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -20,7 +20,6 @@ import { assertError } from '@backstage/errors'; import parseGitUrl from 'git-url-parse'; import limiterFactory from 'p-limit'; import { Logger } from 'winston'; -import * as result from './results'; import { CatalogProcessor, CatalogProcessorCache, @@ -29,7 +28,8 @@ import { CatalogProcessorParser, CatalogProcessorResult, LocationSpec, -} from './types'; + processingResult, +} from '../../api'; const CACHE_KEY = 'v1'; @@ -102,10 +102,10 @@ export class UrlReaderProcessor implements CatalogProcessor { } } else if (error.name === 'NotFoundError') { if (!optional) { - emit(result.notFoundError(location, message)); + emit(processingResult.notFoundError(location, message)); } } else { - emit(result.generalError(location, message)); + emit(processingResult.generalError(location, message)); } } diff --git a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt new file mode 100644 index 0000000000..7470c0e8a3 --- /dev/null +++ b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt @@ -0,0 +1 @@ +site_name: Test diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/component.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/component.yaml rename to plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/dir/api.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/dir/api.yaml rename to plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/test.txt b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/test.txt similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/test.txt rename to plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/test.txt diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/modules/core/index.ts similarity index 63% rename from plugins/catalog-backend/src/ingestion/processors/index.ts rename to plugins/catalog-backend/src/modules/core/index.ts index 84a92fb1ba..9d582bfb34 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/modules/core/index.ts @@ -14,20 +14,10 @@ * limitations under the License. */ -import * as results from './results'; - export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; -export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; -export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; -export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; -export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; -export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor'; -export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; -export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; -export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; export { LocationEntityProcessor } from './LocationEntityProcessor'; export type { LocationEntityProcessorOptions } from './LocationEntityProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; @@ -39,10 +29,5 @@ export type { PlaceholderResolverResolveUrl, } from './PlaceholderProcessor'; export { StaticLocationProcessor } from './StaticLocationProcessor'; -export * from './types'; export { UrlReaderProcessor } from './UrlReaderProcessor'; -export { parseEntityYaml } from './util/parse'; -export { results }; - -export type { BitbucketRepositoryParser } from './bitbucket'; -export type { GithubMultiOrgConfig } from './github'; +export { parseEntityYaml } from '../util/parse'; diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts rename to plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts index e7aec1520d..dd3bf490fb 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts @@ -20,10 +20,12 @@ import { GithubCredentialsProvider, GitHubIntegrationConfig, } from '@backstage/integration'; -import { GitHubOrgEntityProvider } from '.'; -import { EntityProviderConnection } from '../../providers'; -import { withLocations } from './GitHubOrgEntityProvider'; import { graphql } from '@octokit/graphql'; +import { EntityProviderConnection } from '../../api'; +import { + GitHubOrgEntityProvider, + withLocations, +} from './GitHubOrgEntityProvider'; jest.mock('@octokit/graphql'); diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts rename to plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts index a20f7fdb0b..76389968ae 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, @@ -29,13 +30,13 @@ import { import { graphql } from '@octokit/graphql'; import { merge } from 'lodash'; import { Logger } from 'winston'; -import { EntityProvider, EntityProviderConnection } from '../../providers'; +import { EntityProvider, EntityProviderConnection } from '../../api'; import { getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, -} from '../processors/github'; -import { assignGroupsToUsers, buildOrgHierarchy } from '../processors/util/org'; +} from './lib'; +import { assignGroupsToUsers, buildOrgHierarchy } from '../util/org'; // TODO: Consider supporting an (optional) webhook that reacts on org changes /** @public */ diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts index 55e5560451..5440539b27 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts @@ -15,16 +15,16 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; -import { getOrganizationRepositories } from './github'; -import { LocationSpec } from './types'; import { ConfigReader } from '@backstage/config'; import { - ScmIntegrations, DefaultGithubCredentialsProvider, + ScmIntegrations, } from '@backstage/integration'; +import { LocationSpec } from '../../api'; +import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; +import { getOrganizationRepositories } from './lib'; -jest.mock('./github'); +jest.mock('./lib'); const mockGetOrganizationRepositories = getOrganizationRepositories as jest.MockedFunction< typeof getOrganizationRepositories diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts similarity index 96% rename from plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts index e790134011..4aff470f52 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts @@ -23,9 +23,13 @@ import { } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; -import { getOrganizationRepositories } from './github'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { getOrganizationRepositories } from './lib'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; /** * Extracts repositories out of a GitHub org. @@ -141,7 +145,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { const path = `/blob/${branchName}${catalogPath}`; emit( - results.location({ + processingResult.location({ type: 'url', target: `${repository.url}${path}`, // Not all locations may actually exist, since the user defined them as a wildcard pattern. diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts rename to plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts index 383b82c019..339119137a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts @@ -30,10 +30,14 @@ import { getOrganizationUsers, GithubMultiOrgConfig, readGithubMultiOrgConfig, -} from './github'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { buildOrgHierarchy } from './util/org'; +} from './lib'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { buildOrgHierarchy } from '../util/org'; /** * @alpha @@ -158,7 +162,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { buildOrgHierarchy(groups); for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } } catch (e) { this.logger.error( @@ -169,7 +173,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { const allUsers = Array.from(allUsersMap.values()); for (const user of allUsers) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts rename to plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts index c01db88d3a..9ce278ac44 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { @@ -21,7 +22,7 @@ import { } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; jest.mock('@octokit/graphql'); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts rename to plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts index 8b5914cc80..e85e7b874d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts @@ -28,10 +28,14 @@ import { getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, -} from './github'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { assignGroupsToUsers, buildOrgHierarchy } from './util/org'; +} from './lib'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { assignGroupsToUsers, buildOrgHierarchy } from '../util/org'; type GraphQL = typeof graphql; @@ -106,10 +110,10 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { // Done! for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } for (const user of users) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend/src/modules/github/index.ts b/plugins/catalog-backend/src/modules/github/index.ts new file mode 100644 index 0000000000..7958818096 --- /dev/null +++ b/plugins/catalog-backend/src/modules/github/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; +export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; +export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; +export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; +export type { GithubMultiOrgConfig } from './lib'; diff --git a/plugins/catalog-backend/src/ingestion/processors/github/config.test.ts b/plugins/catalog-backend/src/modules/github/lib/config.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/config.test.ts rename to plugins/catalog-backend/src/modules/github/lib/config.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/config.ts b/plugins/catalog-backend/src/modules/github/lib/config.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/config.ts rename to plugins/catalog-backend/src/modules/github/lib/config.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/modules/github/lib/github.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/github.test.ts rename to plugins/catalog-backend/src/modules/github/lib/github.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/modules/github/lib/github.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/github.ts rename to plugins/catalog-backend/src/modules/github/lib/github.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/index.ts b/plugins/catalog-backend/src/modules/github/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/index.ts rename to plugins/catalog-backend/src/modules/github/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts b/plugins/catalog-backend/src/modules/github/lib/util.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/github/util.test.ts rename to plugins/catalog-backend/src/modules/github/lib/util.test.ts index 02f54e0675..c73c11bd2d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts +++ b/plugins/catalog-backend/src/modules/github/lib/util.test.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { parseGitHubOrgUrl } from './util'; describe('parseGitHubOrgUrl', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.ts b/plugins/catalog-backend/src/modules/github/lib/util.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/github/util.ts rename to plugins/catalog-backend/src/modules/github/lib/util.ts index d8df376038..a38225ff98 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/util.ts +++ b/plugins/catalog-backend/src/modules/github/lib/util.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + export function parseGitHubOrgUrl(urlString: string): { org: string } { const path = new URL(urlString).pathname.substr(1).split('/'); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.test.ts index 8261690e8f..cc0f368aee 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.test.ts @@ -19,8 +19,8 @@ import { getVoidLogger } from '@backstage/backend-common'; import { GitLabDiscoveryProcessor, parseUrl } from './GitLabDiscoveryProcessor'; import { setupServer } from 'msw/node'; import { rest } from 'msw'; -import { GitLabProject } from './gitlab'; -import { LocationSpec } from './types'; +import { GitLabProject } from './lib'; +import { LocationSpec } from '../../api'; const server = setupServer(); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.ts index b534d50044..d3916f45d1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.ts @@ -20,9 +20,13 @@ import { ScmIntegrations, } from '@backstage/integration'; import { Logger } from 'winston'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { GitLabClient, GitLabProject, paginated } from './gitlab'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { GitLabClient, GitLabProject, paginated } from './lib'; import { CacheClient, CacheManager, @@ -95,12 +99,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { page: 1, }); - const result: Result = { + const res: Result = { scanned: 0, matches: [], }; for await (const project of projects) { - result.scanned++; + res.scanned++; if (project.archived) { continue; @@ -110,14 +114,14 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { continue; } - result.matches.push(project); + res.matches.push(project); } - for (const project of result.matches) { + for (const project of res.matches) { const project_branch = branch === '*' ? project.default_branch : branch; emit( - results.location({ + processingResult.location({ type: 'url', // The format expected by the GitLabUrlReader: // https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath @@ -133,7 +137,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); this.logger.debug( - `Read ${result.scanned} GitLab repositories in ${duration} seconds`, + `Read ${res.scanned} GitLab repositories in ${duration} seconds`, ); return true; diff --git a/plugins/catalog-backend/src/modules/gitlab/index.ts b/plugins/catalog-backend/src/modules/gitlab/index.ts new file mode 100644 index 0000000000..f42a901f8f --- /dev/null +++ b/plugins/catalog-backend/src/modules/gitlab/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts b/plugins/catalog-backend/src/modules/gitlab/lib/client.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/client.test.ts index b249137740..a7c0d5161f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts +++ b/plugins/catalog-backend/src/modules/gitlab/lib/client.test.ts @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { ConfigReader } from '@backstage/config'; import { setupRequestMockHandlers } from '@backstage/test-utils'; import { readGitLabIntegrationConfig } from '@backstage/integration'; import { getVoidLogger } from '@backstage/backend-common'; import { rest } from 'msw'; import { setupServer, SetupServerApi } from 'msw/node'; - import { GitLabClient, paginated } from './client'; const server = setupServer(); diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/modules/gitlab/lib/client.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/client.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts b/plugins/catalog-backend/src/modules/gitlab/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts b/plugins/catalog-backend/src/modules/gitlab/lib/types.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/types.ts diff --git a/plugins/catalog-backend/src/modules/index.ts b/plugins/catalog-backend/src/modules/index.ts new file mode 100644 index 0000000000..6acf057ca3 --- /dev/null +++ b/plugins/catalog-backend/src/modules/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './aws'; +export * from './azure'; +export * from './bitbucket'; +export * from './codeowners'; +export * from './core'; +export * from './github'; +export * from './gitlab'; diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts b/plugins/catalog-backend/src/modules/util/org.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/util/org.test.ts rename to plugins/catalog-backend/src/modules/util/org.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.ts b/plugins/catalog-backend/src/modules/util/org.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/util/org.ts rename to plugins/catalog-backend/src/modules/util/org.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/util/parse.test.ts b/plugins/catalog-backend/src/modules/util/parse.test.ts similarity index 91% rename from plugins/catalog-backend/src/ingestion/processors/util/parse.test.ts rename to plugins/catalog-backend/src/modules/util/parse.test.ts index a42d160725..e87de1c9a3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/parse.test.ts +++ b/plugins/catalog-backend/src/modules/util/parse.test.ts @@ -15,7 +15,7 @@ */ import { parseEntityYaml } from './parse'; -import * as result from '../results'; +import { processingResult } from '../../api'; const testLoc = { target: 'my-loc-target', @@ -47,7 +47,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -92,7 +92,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -102,7 +102,7 @@ describe('parseEntityYaml', () => { type: 'website', }, }), - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -137,7 +137,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -157,7 +157,7 @@ describe('parseEntityYaml', () => { // Parse errors are always per document expect(results).toEqual([ - result.generalError( + processingResult.generalError( testLoc, 'YAML error at my-loc-type:my-loc-target, YAMLSemanticError: Plain value cannot start with reserved character `', ), @@ -186,7 +186,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -196,7 +196,7 @@ describe('parseEntityYaml', () => { type: 'website', }, }), - result.generalError( + processingResult.generalError( testLoc, 'YAML error at my-loc-type:my-loc-target, YAMLSemanticError: Nested mappings are not allowed in compact mappings', ), @@ -209,7 +209,10 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.generalError(testLoc, 'Expected object at root, got string'), + processingResult.generalError( + testLoc, + 'Expected object at root, got string', + ), ]); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts b/plugins/catalog-backend/src/modules/util/parse.ts similarity index 87% rename from plugins/catalog-backend/src/ingestion/processors/util/parse.ts rename to plugins/catalog-backend/src/modules/util/parse.ts index 0beaf1cd4c..65fad5986f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts +++ b/plugins/catalog-backend/src/modules/util/parse.ts @@ -17,12 +17,12 @@ import { Entity, stringifyLocationRef } from '@backstage/catalog-model'; import lodash from 'lodash'; import yaml from 'yaml'; -import * as result from '../results'; import { CatalogProcessorParser, CatalogProcessorResult, LocationSpec, -} from '../types'; + processingResult, +} from '../../api'; /** @public */ export function* parseEntityYaml( @@ -35,7 +35,7 @@ export function* parseEntityYaml( } catch (e) { const loc = stringifyLocationRef(location); const message = `Failed to parse YAML at ${loc}, ${e}`; - yield result.generalError(location, message); + yield processingResult.generalError(location, message); return; } @@ -43,17 +43,17 @@ export function* parseEntityYaml( if (document.errors?.length) { const loc = stringifyLocationRef(location); const message = `YAML error at ${loc}, ${document.errors[0]}`; - yield result.generalError(location, message); + yield processingResult.generalError(location, message); } else { const json = document.toJSON(); if (lodash.isPlainObject(json)) { - yield result.entity(location, json as Entity); + yield processingResult.entity(location, json as Entity); } else if (json === null) { // Ignore null values, these happen if there is an empty document in the // YAML file, for example if --- is added to the end of the file. } else { const message = `Expected object at root, got ${typeof json}`; - yield result.generalError(location, message); + yield processingResult.generalError(location, message); } } } diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts index 0853484408..6e494d784e 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts @@ -29,11 +29,11 @@ import { CatalogProcessorEmit, CatalogProcessorParser, LocationSpec, - results, -} from '../ingestion'; + processingResult, +} from '../api'; import { CatalogRulesEnforcer } from '../ingestion/CatalogRules'; import { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; -import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; +import { defaultEntityDataParser } from '../modules/util/parse'; import { ConfigReader } from '@backstage/config'; class FooBarProcessor implements CatalogProcessor { @@ -51,7 +51,7 @@ class FooBarProcessor implements CatalogProcessor { ) { if (await cache.get('emit')) { emit( - results.entity( + processingResult.entity( { type: 'url', target: './new-place' }, { apiVersion: 'my-api/v1', @@ -63,7 +63,7 @@ class FooBarProcessor implements CatalogProcessor { ), ); emit( - results.relation({ + processingResult.relation({ type: 'my-type', source: { kind: 'foobar', name: 'my-source', namespace: 'default' }, target: { kind: 'foobar', name: 'my-target', namespace: 'default' }, @@ -211,7 +211,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => { getProcessorName: jest.fn(), validateEntityKind: jest.fn(async () => true), readLocation: jest.fn(async (_l, _o, emit) => { - emit(results.entity({ type: 't', target: 't' }, entity)); + emit(processingResult.entity({ type: 't', target: 't' }, entity)); return true; }), }; diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts index 9649bc937b..c050779a8b 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts @@ -36,8 +36,8 @@ import { CatalogProcessor, CatalogProcessorParser, LocationSpec, -} from '../ingestion/processors'; -import * as results from '../ingestion/processors/results'; + processingResult, +} from '../api'; import { CatalogProcessingOrchestrator, EntityProcessingRequest, @@ -178,13 +178,13 @@ export class DefaultCatalogProcessingOrchestrator entity: Entity, context: Context, ): Promise { - let result = entity; + let res = entity; for (const processor of this.options.processors) { if (processor.preProcessEntity) { try { - result = await processor.preProcessEntity( - result, + res = await processor.preProcessEntity( + res, context.location, context.collector.onEmit, context.originLocation, @@ -199,7 +199,7 @@ export class DefaultCatalogProcessingOrchestrator } } - return result; + return res; } /** @@ -295,7 +295,7 @@ export class DefaultCatalogProcessingOrchestrator for (const maybeRelativeTarget of targets) { if (type === 'file' && maybeRelativeTarget.endsWith(path.sep)) { context.collector.onEmit( - results.inputError( + processingResult.inputError( context.location, `LocationEntityProcessor cannot handle ${type} type location with target ${context.location.target} that ends with a path separator`, ), @@ -351,13 +351,13 @@ export class DefaultCatalogProcessingOrchestrator entity: Entity, context: Context, ): Promise { - let result = entity; + let res = entity; for (const processor of this.options.processors) { if (processor.postProcessEntity) { try { - result = await processor.postProcessEntity( - result, + res = await processor.postProcessEntity( + res, context.location, context.collector.onEmit, context.cache.forProcessor(processor), @@ -371,6 +371,6 @@ export class DefaultCatalogProcessingOrchestrator } } - return result; + return res; } } diff --git a/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts b/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts index 71631b13d8..602557a224 100644 --- a/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts +++ b/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { CatalogProcessor } from '../ingestion/processors'; +import { CatalogProcessor } from '../api'; import { ProcessorCacheManager } from './ProcessorCacheManager'; class MyProcessor implements CatalogProcessor { diff --git a/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts b/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts index ab95fcedb7..e54641f572 100644 --- a/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts +++ b/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts @@ -15,8 +15,7 @@ */ import { JsonObject, JsonValue } from '@backstage/types'; -import { CatalogProcessor } from '../ingestion/processors'; -import { CatalogProcessorCache } from '../ingestion/processors/types'; +import { CatalogProcessor, CatalogProcessorCache } from '../api'; import { isObject } from './util'; class SingleProcessorSubCache implements CatalogProcessorCache { diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 351e68882b..c3cc6e288b 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -22,9 +22,9 @@ import { } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; import { Logger } from 'winston'; -import { CatalogProcessorResult } from '../ingestion'; +import { CatalogProcessorResult, EntityRelationSpec } from '../api'; import { locationSpecToLocationEntity } from '../util/conversion'; -import { DeferredEntity, EntityRelationSpec } from './types'; +import { DeferredEntity } from './types'; import { getEntityLocationRef, getEntityOriginLocationRef, diff --git a/plugins/catalog-backend/src/processing/connectEntityProviders.ts b/plugins/catalog-backend/src/processing/connectEntityProviders.ts index 02bdc2aee3..d7015ea52e 100644 --- a/plugins/catalog-backend/src/processing/connectEntityProviders.ts +++ b/plugins/catalog-backend/src/processing/connectEntityProviders.ts @@ -23,7 +23,7 @@ import { EntityProvider, EntityProviderConnection, EntityProviderMutation, -} from '../providers/types'; +} from '../api'; class Connection implements EntityProviderConnection { readonly validateEntityEnvelope = entityEnvelopeSchemaValidator(); diff --git a/plugins/catalog-backend/src/processing/index.ts b/plugins/catalog-backend/src/processing/index.ts index 83f8a9f566..dc8b07eceb 100644 --- a/plugins/catalog-backend/src/processing/index.ts +++ b/plugins/catalog-backend/src/processing/index.ts @@ -19,7 +19,6 @@ export type { CatalogProcessingEngine, EntityProcessingRequest, EntityProcessingResult, - EntityRelationSpec, DeferredEntity, } from './types'; export { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index b732df297e..484cd73939 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -14,30 +14,9 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/types'; - -/** - * Holds the relation data for entities. - * - * @public - */ -export type EntityRelationSpec = { - /** - * The source entity of this relation. - */ - source: EntityName; - - /** - * The type of the relation. - */ - type: string; - - /** - * The target entity of this relation. - */ - target: EntityName; -}; +import { EntityRelationSpec } from '../api'; /** * The request to process an entity. diff --git a/plugins/catalog-backend/src/processing/util.ts b/plugins/catalog-backend/src/processing/util.ts index 94634f794a..b1948cf559 100644 --- a/plugins/catalog-backend/src/processing/util.ts +++ b/plugins/catalog-backend/src/processing/util.ts @@ -27,7 +27,7 @@ import { JsonObject, JsonValue } from '@backstage/types'; import { InputError } from '@backstage/errors'; import { ScmIntegrationRegistry } from '@backstage/integration'; import path from 'path'; -import { LocationSpec } from '../ingestion'; +import { LocationSpec } from '../api'; export function isLocationEntity(entity: Entity): entity is LocationEntity { return entity.kind === 'Location'; diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts index 819451d854..17dfb58c54 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { NotAllowedError } from '@backstage/errors'; import { catalogEntityRefreshPermission } from '@backstage/plugin-catalog-common'; import { diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 9004cb4c56..f62377decb 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -38,12 +38,15 @@ import { Router } from 'express'; import lodash, { keyBy } from 'lodash'; import { EntitiesCatalog, EntitiesSearchFilter } from '../catalog'; +import { + CatalogProcessor, + CatalogProcessorParser, + EntityProvider, +} from '../api'; import { AnnotateLocationEntityProcessor, BitbucketDiscoveryProcessor, BuiltinKindsEntityProcessor, - CatalogProcessor, - CatalogProcessorParser, CodeOwnersProcessor, FileReaderProcessor, AzureDevOpsDiscoveryProcessor, @@ -53,23 +56,22 @@ import { PlaceholderProcessor, PlaceholderResolver, UrlReaderProcessor, -} from '../ingestion'; +} from '../modules'; +import { ConfigLocationEntityProvider } from '../modules/core/ConfigLocationEntityProvider'; +import { DefaultLocationStore } from '../modules/core/DefaultLocationStore'; import { RepoLocationAnalyzer } from '../ingestion/LocationAnalyzer'; import { jsonPlaceholderResolver, textPlaceholderResolver, yamlPlaceholderResolver, -} from '../ingestion/processors/PlaceholderProcessor'; -import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; +} from '../modules/core/PlaceholderProcessor'; +import { defaultEntityDataParser } from '../modules/util/parse'; import { LocationAnalyzer } from '../ingestion/types'; -import { EntityProvider } from '../providers/types'; import { CatalogProcessingEngine } from '../processing/types'; -import { ConfigLocationEntityProvider } from '../providers/ConfigLocationEntityProvider'; import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { applyDatabaseMigrations } from '../database/migrations'; import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; import { DefaultLocationService } from './DefaultLocationService'; -import { DefaultLocationStore } from '../providers/DefaultLocationStore'; import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; import { Stitcher } from '../stitching/Stitcher'; diff --git a/plugins/catalog-backend/src/service/DefaultLocationService.ts b/plugins/catalog-backend/src/service/DefaultLocationService.ts index c6b5eaf162..7a1b923068 100644 --- a/plugins/catalog-backend/src/service/DefaultLocationService.ts +++ b/plugins/catalog-backend/src/service/DefaultLocationService.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Entity, ANNOTATION_LOCATION, diff --git a/plugins/catalog-backend/src/util/conversion.ts b/plugins/catalog-backend/src/util/conversion.ts index 75ed4f5e96..63771d1028 100644 --- a/plugins/catalog-backend/src/util/conversion.ts +++ b/plugins/catalog-backend/src/util/conversion.ts @@ -23,7 +23,7 @@ import { stringifyLocationRef, } from '@backstage/catalog-model'; import { createHash } from 'crypto'; -import { LocationSpec } from '../ingestion'; +import { LocationSpec } from '../api'; export function locationSpecToMetadataName(location: LocationSpec) { const hash = createHash('sha1') diff --git a/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts b/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts index 43559fcd5f..5fcd89841e 100644 --- a/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts +++ b/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts @@ -25,7 +25,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; import { TemplateEntityV1beta3, @@ -70,7 +70,7 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { defaultNamespace: selfRef.namespace, }); emit( - results.relation({ + processingResult.relation({ source: selfRef, type: RELATION_OWNED_BY, target: { @@ -81,7 +81,7 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { }), ); emit( - results.relation({ + processingResult.relation({ source: { kind: targetRef.kind, namespace: targetRef.namespace, From 79b9d8a8611860a01ffa686bcf8e864ccfa7f637 Mon Sep 17 00:00:00 2001 From: Mike Lewis Date: Tue, 1 Mar 2022 14:02:24 +0000 Subject: [PATCH 060/150] permission-common: add apidocs for permission properties Signed-off-by: Mike Lewis --- .changeset/nice-dragons-collect.md | 5 +++++ plugins/permission-common/src/types/permission.ts | 14 ++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 .changeset/nice-dragons-collect.md diff --git a/.changeset/nice-dragons-collect.md b/.changeset/nice-dragons-collect.md new file mode 100644 index 0000000000..040b030539 --- /dev/null +++ b/.changeset/nice-dragons-collect.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-permission-common': patch +--- + +Add api doc comments to Permission type properties. diff --git a/plugins/permission-common/src/types/permission.ts b/plugins/permission-common/src/types/permission.ts index 805aefa3de..16934d8771 100644 --- a/plugins/permission-common/src/types/permission.ts +++ b/plugins/permission-common/src/types/permission.ts @@ -37,8 +37,22 @@ export type PermissionAttributes = { * @public */ export type Permission = { + /** + * The name of the permission. + */ name: string; + /** + * {@link PermissionAttributes} which describe characteristics of the permission, to help + * policy authors make consistent decisions for similar permissions without referring to them + * all by name. + */ attributes: PermissionAttributes; + /** + * Some permissions can be authorized based on characteristics of a resource + * such a catalog entity. For these permissions, the resourceType field + * denotes the type of the resource whose resourceRef should be passed when + * authorizing. + */ resourceType?: string; }; From 194011ce84cb3a9643b29620818e806e37553128 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Mar 2022 10:40:48 +0100 Subject: [PATCH 061/150] chore: fixing up the types to actually be correct and use the returnValue types Signed-off-by: blam --- .changeset/swift-roses-hug.md | 2 +- plugins/scaffolder/api-report.md | 38 +++++++----- .../components/TemplatePage/TemplatePage.tsx | 2 +- plugins/scaffolder/src/extensions/default.ts | 58 +++++++++---------- plugins/scaffolder/src/extensions/index.tsx | 15 +++++ plugins/scaffolder/src/extensions/types.ts | 6 +- plugins/scaffolder/src/index.ts | 8 ++- 7 files changed, 77 insertions(+), 52 deletions(-) diff --git a/.changeset/swift-roses-hug.md b/.changeset/swift-roses-hug.md index 47a84e3247..0e6d76ed87 100644 --- a/.changeset/swift-roses-hug.md +++ b/.changeset/swift-roses-hug.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-scaffolder': patch +'@backstage/plugin-scaffolder': minor --- **BREAKING**: Removing the exports of the raw components that back the `CustomFieldExtensions`. diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index eed65578d3..a385e13534 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -28,16 +28,13 @@ import { ScmIntegrationRegistry } from '@backstage/integration'; import { TaskSpec } from '@backstage/plugin-scaffolder-common'; import { TemplateEntityV1beta2 } from '@backstage/plugin-scaffolder-common'; -// Warning: (ae-forgotten-export) The symbol "FieldExtensionComponent" needs to be exported by the entry point index.d.ts -// Warning: (ae-missing-release-tag) "createScaffolderFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export function createScaffolderFieldExtension< TReturnValue = unknown, TInputProps = unknown, >( options: FieldExtensionOptions, -): Extension>; +): Extension>; // @public export type CustomFieldValidator = ( @@ -52,14 +49,16 @@ export type CustomFieldValidator = ( // // @public (undocumented) export const EntityNamePickerFieldExtension: FieldExtensionComponent< - FieldExtensionComponentProps + string, + {} >; // Warning: (ae-missing-release-tag) "EntityPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) export const EntityPickerFieldExtension: FieldExtensionComponent< - FieldExtensionComponentProps + string, + EntityPickerUiOptions >; // Warning: (ae-missing-release-tag) "EntityPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -76,7 +75,8 @@ export interface EntityPickerUiOptions { // @public export const EntityTagsPickerFieldExtension: FieldExtensionComponent< - FieldExtensionComponentProps + string[], + EntityTagsPickerUiOptions >; // Warning: (ae-missing-release-tag) "EntityTagsPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -93,6 +93,9 @@ export interface EntityTagsPickerUiOptions { // @public export const FavouriteTemplate: (props: Props) => JSX.Element; +// @public +export type FieldExtensionComponent<_TReturnValue, _TInputProps> = () => null; + // @public export interface FieldExtensionComponentProps< TFieldReturnValue, @@ -107,10 +110,12 @@ export interface FieldExtensionComponentProps< // @public export type FieldExtensionOptions< TFieldReturnValue = unknown, - TProps = FieldProps, + TInputProps = unknown, > = { name: string; - component: (props: TProps) => JSX.Element | null; + component: ( + props: FieldExtensionComponentProps, + ) => JSX.Element | null; validation?: CustomFieldValidator; }; @@ -150,7 +155,8 @@ export type LogEvent = { // // @public (undocumented) export const OwnedEntityPickerFieldExtension: FieldExtensionComponent< - FieldExtensionComponentProps + string, + OwnedEntityPickerUiOptions >; // Warning: (ae-missing-release-tag) "OwnedEntityPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -167,7 +173,8 @@ export interface OwnedEntityPickerUiOptions { // // @public (undocumented) export const OwnerPickerFieldExtension: FieldExtensionComponent< - FieldExtensionComponentProps + string, + OwnerPickerUiOptions >; // Warning: (ae-missing-release-tag) "OwnerPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -193,7 +200,8 @@ export const repoPickerValidation: ( // // @public (undocumented) export const RepoUrlPickerFieldExtension: FieldExtensionComponent< - FieldExtensionComponentProps + string, + RepoUrlPickerUiOptions >; // Warning: (ae-missing-release-tag) "RepoUrlPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -291,9 +299,7 @@ export class ScaffolderClient implements ScaffolderApi { streamLogs(options: ScaffolderStreamLogsOptions): Observable; } -// Warning: (ae-missing-release-tag) "ScaffolderFieldExtensions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const ScaffolderFieldExtensions: React_2.ComponentType; // Warning: (ae-missing-release-tag) "ScaffolderGetIntegrationsListOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index e69c4bab5c..5310a4c79f 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -109,7 +109,7 @@ export const createValidator = ( export const TemplatePage = ({ customFieldExtensions = [], }: { - customFieldExtensions?: FieldExtensionOptions[]; + customFieldExtensions?: FieldExtensionOptions[]; }) => { const apiHolder = useApiHolder(); const secretsContext = useContext(SecretsContext); diff --git a/plugins/scaffolder/src/extensions/default.ts b/plugins/scaffolder/src/extensions/default.ts index b3bb37065e..8081dcb0ca 100644 --- a/plugins/scaffolder/src/extensions/default.ts +++ b/plugins/scaffolder/src/extensions/default.ts @@ -20,35 +20,33 @@ import { EntityTagsPicker } from '../components/fields/EntityTagsPicker/EntityTa import { OwnerPicker } from '../components/fields/OwnerPicker/OwnerPicker'; import { RepoUrlPicker } from '../components/fields/RepoUrlPicker/RepoUrlPicker'; import { repoPickerValidation } from '../components/fields/RepoUrlPicker/validation'; -import { FieldExtensionOptions } from './types'; import { OwnedEntityPicker } from '../components/fields/OwnedEntityPicker/OwnedEntityPicker'; -export const DEFAULT_SCAFFOLDER_FIELD_EXTENSIONS: FieldExtensionOptions[] = - [ - { - component: EntityPicker, - name: 'EntityPicker', - }, - { - component: EntityNamePicker, - name: 'EntityNamePicker', - validation: entityNamePickerValidation, - }, - { - component: EntityTagsPicker, - name: 'EntityTagsPicker', - }, - { - component: RepoUrlPicker, - name: 'RepoUrlPicker', - validation: repoPickerValidation, - }, - { - component: OwnerPicker, - name: 'OwnerPicker', - }, - { - component: OwnedEntityPicker, - name: 'OwnedEntityPicker', - }, - ]; +export const DEFAULT_SCAFFOLDER_FIELD_EXTENSIONS = [ + { + component: EntityPicker, + name: 'EntityPicker', + }, + { + component: EntityNamePicker, + name: 'EntityNamePicker', + validation: entityNamePickerValidation, + }, + { + component: EntityTagsPicker, + name: 'EntityTagsPicker', + }, + { + component: RepoUrlPicker, + name: 'RepoUrlPicker', + validation: repoPickerValidation, + }, + { + component: OwnerPicker, + name: 'OwnerPicker', + }, + { + component: OwnedEntityPicker, + name: 'OwnedEntityPicker', + }, +]; diff --git a/plugins/scaffolder/src/extensions/index.tsx b/plugins/scaffolder/src/extensions/index.tsx index 75a166beff..60b9022053 100644 --- a/plugins/scaffolder/src/extensions/index.tsx +++ b/plugins/scaffolder/src/extensions/index.tsx @@ -25,8 +25,18 @@ import { Extension, attachComponentData } from '@backstage/core-plugin-api'; export const FIELD_EXTENSION_WRAPPER_KEY = 'scaffolder.extensions.wrapper.v1'; export const FIELD_EXTENSION_KEY = 'scaffolder.extensions.field.v1'; +/** + * A type used to wrap up the FieldExtension to embed the ReturnValue and the InputProps + * + * @public + */ export type FieldExtensionComponent<_TReturnValue, _TInputProps> = () => null; +/** + * Method for creating field extensions that can be used in the scaffolder + * frontend form. + * @public + */ export function createScaffolderFieldExtension< TReturnValue = unknown, TInputProps = unknown, @@ -48,6 +58,11 @@ export function createScaffolderFieldExtension< }; } +/** + * The Wrapping component for defining fields extensions inside + * + * @public + */ export const ScaffolderFieldExtensions: React.ComponentType = (): JSX.Element | null => null; diff --git a/plugins/scaffolder/src/extensions/types.ts b/plugins/scaffolder/src/extensions/types.ts index 574ce7ea64..be8dc14b0a 100644 --- a/plugins/scaffolder/src/extensions/types.ts +++ b/plugins/scaffolder/src/extensions/types.ts @@ -35,10 +35,12 @@ export type CustomFieldValidator = ( */ export type FieldExtensionOptions< TFieldReturnValue = unknown, - TProps = FieldProps, + TInputProps = unknown, > = { name: string; - component: (props: TProps) => JSX.Element | null; + component: ( + props: FieldExtensionComponentProps, + ) => JSX.Element | null; validation?: CustomFieldValidator; }; diff --git a/plugins/scaffolder/src/index.ts b/plugins/scaffolder/src/index.ts index 9776549091..f33cf289f4 100644 --- a/plugins/scaffolder/src/index.ts +++ b/plugins/scaffolder/src/index.ts @@ -40,7 +40,12 @@ export { createScaffolderFieldExtension, ScaffolderFieldExtensions, } from './extensions'; - +export type { + CustomFieldValidator, + FieldExtensionOptions, + FieldExtensionComponentProps, + FieldExtensionComponent, +} from './extensions'; export { EntityPickerFieldExtension, EntityNamePickerFieldExtension, @@ -52,5 +57,4 @@ export { scaffolderPlugin, } from './plugin'; export * from './components'; -export * from './extensions'; export type { TaskPageProps } from './components/TaskPage'; From 8e1df15a95a0e89246a99c18e059e51fee882842 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Mar 2022 15:32:04 +0100 Subject: [PATCH 062/150] chore: fix up the interface a little bit Signed-off-by: blam --- plugins/catalog-react/src/hooks/useEntity.tsx | 51 ++++++------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/plugins/catalog-react/src/hooks/useEntity.tsx b/plugins/catalog-react/src/hooks/useEntity.tsx index e46ee680eb..8ff5df9349 100644 --- a/plugins/catalog-react/src/hooks/useEntity.tsx +++ b/plugins/catalog-react/src/hooks/useEntity.tsx @@ -27,8 +27,8 @@ import { catalogApiRef } from '../api'; import { useEntityCompoundName } from './useEntityCompoundName'; /** @public */ -export type EntityLoadingStatus = { - entity?: Entity; +export type EntityLoadingStatus = { + entity?: TEntity; loading: boolean; error?: Error; refresh?: VoidFunction; @@ -129,40 +129,21 @@ export const useEntityFromUrl = (): EntityLoadingStatus => { return { entity, loading, error, refresh }; }; -/** - * @public - * - * The response shape for {@link useEntity} - */ -export interface UseEntityResponse { - entity: T; - /** @deprecated use {@link useAsyncEntity} instead */ - loading: boolean; - /** @deprecated use {@link useAsyncEntity} instead */ - error?: Error; - /** @deprecated use {@link useAsyncEntity} instead */ - refresh?: VoidFunction; -} - -/** - * @public - * - * The response shape for {@link useAsyncEntity} - */ -export interface UseAsyncEntityResponse { - entity?: T; - loading: boolean; - error?: Error; - refresh?: VoidFunction; -} - /** * Grab the current entity from the context, throws if the entity has not yet been loaded * or is not available. * * @public */ -export function useEntity(): UseEntityResponse { +export function useEntity(): { + entity: TEntity; + /** @deprecated use {@link useAsyncEntity} instead */ + loading: boolean; + /** @deprecated use {@link useAsyncEntity} instead */ + error?: Error; + /** @deprecated use {@link useAsyncEntity} instead */ + refresh?: VoidFunction; +} { const versionedHolder = useVersionedContext<{ 1: EntityLoadingStatus }>('entity-context'); @@ -178,7 +159,7 @@ export function useEntity(): UseEntityResponse { if (!value.entity) { // Once we have removed the additional fields from being returned we can drop this deprecation // and move to the error instead. - // throw new Error('useEntity hook is being called outside of an EntityPage where the entity has not been loaded. If this is intentional, please use useAsyncEntity instead.'); + // throw new Error('useEntity hook is being called outside of an EntityLayout where the entity has not been loaded. If this is intentional, please use useAsyncEntity instead.'); // eslint-disable-next-line no-console console.warn( @@ -187,7 +168,7 @@ export function useEntity(): UseEntityResponse { } const { entity, loading, error, refresh } = value; - return { entity: entity as T, loading, error, refresh }; + return { entity: entity as TEntity, loading, error, refresh }; } /** @@ -196,8 +177,8 @@ export function useEntity(): UseEntityResponse { * @public */ export function useAsyncEntity< - T extends Entity = Entity, ->(): UseAsyncEntityResponse { + TEntity extends Entity = Entity, +>(): EntityLoadingStatus { const versionedHolder = useVersionedContext<{ 1: EntityLoadingStatus }>('entity-context'); @@ -210,5 +191,5 @@ export function useAsyncEntity< } const { entity, loading, error, refresh } = value; - return { entity: entity as T, loading, error, refresh }; + return { entity: entity as TEntity, loading, error, refresh }; } From 141af65385d3b6c0bea94f6c29ee4ab2f22e7baf Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Mar 2022 15:34:47 +0100 Subject: [PATCH 063/150] chore: fixing the api-report and clean up the types Signed-off-by: blam --- plugins/catalog-react/api-report.md | 35 +++++-------------- plugins/catalog-react/src/hooks/index.ts | 2 -- plugins/catalog-react/src/hooks/useEntity.tsx | 2 +- 3 files changed, 9 insertions(+), 30 deletions(-) diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 0103ac2f0e..1a94af6052 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -225,8 +225,8 @@ export const EntityListProvider: ({ }: PropsWithChildren<{}>) => JSX.Element; // @public (undocumented) -export type EntityLoadingStatus = { - entity?: Entity; +export type EntityLoadingStatus = { + entity?: TEntity; loading: boolean; error?: Error; refresh?: VoidFunction; @@ -525,23 +525,16 @@ export type UnregisterEntityDialogProps = { // @public export function useAsyncEntity< - T extends Entity = Entity, ->(): UseAsyncEntityResponse; + TEntity extends Entity = Entity, +>(): EntityLoadingStatus; // @public -export interface UseAsyncEntityResponse { - // (undocumented) - entity?: T; - // (undocumented) - error?: Error; - // (undocumented) +export function useEntity(): { + entity: TEntity; loading: boolean; - // (undocumented) + error?: Error; refresh?: VoidFunction; -} - -// @public -export function useEntity(): UseEntityResponse; +}; // @public @deprecated export const useEntityCompoundName: () => { @@ -583,18 +576,6 @@ export function useEntityPermission(permission: Permission): { error?: Error; }; -// @public -export interface UseEntityResponse { - // (undocumented) - entity: T; - // @deprecated (undocumented) - error?: Error; - // @deprecated (undocumented) - loading: boolean; - // @deprecated (undocumented) - refresh?: VoidFunction; -} - // @public export function useEntityTypeFilter(): { loading: boolean; diff --git a/plugins/catalog-react/src/hooks/index.ts b/plugins/catalog-react/src/hooks/index.ts index 8e0fc48e72..da23d54ad7 100644 --- a/plugins/catalog-react/src/hooks/index.ts +++ b/plugins/catalog-react/src/hooks/index.ts @@ -24,8 +24,6 @@ export type { EntityLoadingStatus, EntityProviderProps, AsyncEntityProviderProps, - UseEntityResponse, - UseAsyncEntityResponse, } from './useEntity'; export { useEntityCompoundName } from './useEntityCompoundName'; export { diff --git a/plugins/catalog-react/src/hooks/useEntity.tsx b/plugins/catalog-react/src/hooks/useEntity.tsx index 8ff5df9349..f70adef708 100644 --- a/plugins/catalog-react/src/hooks/useEntity.tsx +++ b/plugins/catalog-react/src/hooks/useEntity.tsx @@ -27,7 +27,7 @@ import { catalogApiRef } from '../api'; import { useEntityCompoundName } from './useEntityCompoundName'; /** @public */ -export type EntityLoadingStatus = { +export type EntityLoadingStatus = { entity?: TEntity; loading: boolean; error?: Error; From 81273e95cffc127ff75273c752519fa3247add41 Mon Sep 17 00:00:00 2001 From: Mike Lewis Date: Tue, 1 Mar 2022 13:07:15 +0000 Subject: [PATCH 064/150] catalog-common: mark permission-related exports as alpha Marks all strictly permission-related exports in catalog-common as alpha. Signed-off-by: Mike Lewis --- .changeset/nice-walls-reply.md | 14 ++++++++++++++ plugins/catalog-common/api-report.md | 16 ++++++++-------- plugins/catalog-common/package.json | 8 +++++--- plugins/catalog-common/src/permissions.ts | 18 ++++++++++-------- 4 files changed, 37 insertions(+), 19 deletions(-) create mode 100644 .changeset/nice-walls-reply.md diff --git a/.changeset/nice-walls-reply.md b/.changeset/nice-walls-reply.md new file mode 100644 index 0000000000..72b389c130 --- /dev/null +++ b/.changeset/nice-walls-reply.md @@ -0,0 +1,14 @@ +--- +'@backstage/plugin-catalog-common': minor +--- + +Mark permission-related exports as alpha. This means that the exports below should now be imported from `@backstage/plugin-catalog-common/alpha` instead of `@backstage/plugin-catalog-common`. + +- `RESOURCE_TYPE_CATALOG_ENTITY` +- `catalogEntityReadPermission` +- `catalogEntityCreatePermission` +- `catalogEntityDeletePermission` +- `catalogEntityRefreshPermission` +- `catalogLocationReadPermission` +- `catalogLocationCreatePermission` +- `catalogLocationDeletePermission` diff --git a/plugins/catalog-common/api-report.md b/plugins/catalog-common/api-report.md index 9826a8ddef..57f8c3983e 100644 --- a/plugins/catalog-common/api-report.md +++ b/plugins/catalog-common/api-report.md @@ -5,27 +5,27 @@ ```ts import { Permission } from '@backstage/plugin-permission-common'; -// @public +// @alpha export const catalogEntityCreatePermission: Permission; -// @public +// @alpha export const catalogEntityDeletePermission: Permission; -// @public +// @alpha export const catalogEntityReadPermission: Permission; -// @public +// @alpha export const catalogEntityRefreshPermission: Permission; -// @public +// @alpha export const catalogLocationCreatePermission: Permission; -// @public +// @alpha export const catalogLocationDeletePermission: Permission; -// @public +// @alpha export const catalogLocationReadPermission: Permission; -// @public (undocumented) +// @alpha export const RESOURCE_TYPE_CATALOG_ENTITY = 'catalog-entity'; ``` diff --git a/plugins/catalog-common/package.json b/plugins/catalog-common/package.json index 9a52d22798..acf4ed4aaa 100644 --- a/plugins/catalog-common/package.json +++ b/plugins/catalog-common/package.json @@ -10,7 +10,8 @@ "access": "public", "main": "dist/index.cjs.js", "module": "dist/index.esm.js", - "types": "dist/index.d.ts" + "types": "dist/index.d.ts", + "alphaTypes": "dist/index.alpha.d.ts" }, "backstage": { "role": "common-library" @@ -25,7 +26,7 @@ "backstage" ], "scripts": { - "build": "backstage-cli package build", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", @@ -39,6 +40,7 @@ "@backstage/cli": "^0.14.0" }, "files": [ - "dist" + "dist", + "alpha" ] } diff --git a/plugins/catalog-common/src/permissions.ts b/plugins/catalog-common/src/permissions.ts index 52c225e441..8e1b036470 100644 --- a/plugins/catalog-common/src/permissions.ts +++ b/plugins/catalog-common/src/permissions.ts @@ -17,8 +17,10 @@ import { Permission } from '@backstage/plugin-permission-common'; /** + * Permission resource type which corresponds to catalog entities. + * * {@link https://backstage.io/docs/features/software-catalog/software-catalog-overview} - * @public + * @alpha */ export const RESOURCE_TYPE_CATALOG_ENTITY = 'catalog-entity'; @@ -28,7 +30,7 @@ export const RESOURCE_TYPE_CATALOG_ENTITY = 'catalog-entity'; * * If this permission is not authorized, it will appear that the entity does not * exist in the catalog — both in the frontend and in API responses. - * @public + * @alpha */ export const catalogEntityReadPermission: Permission = { name: 'catalog.entity.read', @@ -42,7 +44,7 @@ export const catalogEntityReadPermission: Permission = { * This permission is used to authorize actions that involve creating a new * catalog entity. This includes registering an existing component into the * catalog. - * @public + * @alpha */ export const catalogEntityCreatePermission: Permission = { name: 'catalog.entity.create', @@ -55,7 +57,7 @@ export const catalogEntityCreatePermission: Permission = { /** * This permission is used to designate actions that involve removing one or * more entities from the catalog. - * @public + * @alpha */ export const catalogEntityDeletePermission: Permission = { name: 'catalog.entity.delete', @@ -68,7 +70,7 @@ export const catalogEntityDeletePermission: Permission = { /** * This permission is used to designate refreshing one or more entities from the * catalog. - * @public + * @alpha */ export const catalogEntityRefreshPermission: Permission = { name: 'catalog.entity.refresh', @@ -84,7 +86,7 @@ export const catalogEntityRefreshPermission: Permission = { * * If this permission is not authorized, it will appear that the location does * not exist in the catalog — both in the frontend and in API responses. - * @public + * @alpha */ export const catalogLocationReadPermission: Permission = { name: 'catalog.location.read', @@ -96,7 +98,7 @@ export const catalogLocationReadPermission: Permission = { /** * This permission is used to designate actions that involve creating catalog * locations. - * @public + * @alpha */ export const catalogLocationCreatePermission: Permission = { name: 'catalog.location.create', @@ -108,7 +110,7 @@ export const catalogLocationCreatePermission: Permission = { /** * This permission is used to designate actions that involve deleting locations * from the catalog. - * @public + * @alpha */ export const catalogLocationDeletePermission: Permission = { name: 'catalog.location.delete', From 617a132871334cd992cb7e597da54822ae655039 Mon Sep 17 00:00:00 2001 From: Mike Lewis Date: Tue, 1 Mar 2022 13:45:13 +0000 Subject: [PATCH 065/150] create-app: import catalogEntityCreatePermission from /alpha Signed-off-by: Mike Lewis --- .changeset/gorgeous-actors-shave.md | 12 ++++++++++++ .../templates/default-app/packages/app/src/App.tsx | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 .changeset/gorgeous-actors-shave.md diff --git a/.changeset/gorgeous-actors-shave.md b/.changeset/gorgeous-actors-shave.md new file mode 100644 index 0000000000..bcf5f43018 --- /dev/null +++ b/.changeset/gorgeous-actors-shave.md @@ -0,0 +1,12 @@ +--- +'@backstage/create-app': patch +--- + +Update import location of catalogEntityCreatePermission. + +To apply this change to an existing app, make the following change to `packages/app/src/App.tsx`: + +```diff +-import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common'; ++import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common/alpha'; +``` diff --git a/packages/create-app/templates/default-app/packages/app/src/App.tsx b/packages/create-app/templates/default-app/packages/app/src/App.tsx index 6f00993273..f4ff424926 100644 --- a/packages/create-app/templates/default-app/packages/app/src/App.tsx +++ b/packages/create-app/templates/default-app/packages/app/src/App.tsx @@ -30,7 +30,7 @@ import { createApp } from '@backstage/app-defaults'; import { FlatRoutes } from '@backstage/core-app-api'; import { CatalogGraphPage } from '@backstage/plugin-catalog-graph'; import { PermissionedRoute } from '@backstage/plugin-permission-react'; -import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common'; +import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common/alpha'; const app = createApp({ apis, From 40559d7a3b952907b837780b538bfa5d2eb1dbbc Mon Sep 17 00:00:00 2001 From: MT Lewis Date: Tue, 1 Mar 2022 14:32:39 +0000 Subject: [PATCH 066/150] Update .changeset/nice-walls-reply.md Co-authored-by: Johan Haals Signed-off-by: Mike Lewis --- .changeset/nice-walls-reply.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/nice-walls-reply.md b/.changeset/nice-walls-reply.md index 72b389c130..f6d1fb7562 100644 --- a/.changeset/nice-walls-reply.md +++ b/.changeset/nice-walls-reply.md @@ -2,7 +2,7 @@ '@backstage/plugin-catalog-common': minor --- -Mark permission-related exports as alpha. This means that the exports below should now be imported from `@backstage/plugin-catalog-common/alpha` instead of `@backstage/plugin-catalog-common`. +**Breaking**: Mark permission-related exports as alpha. This means that the exports below should now be imported from `@backstage/plugin-catalog-common/alpha` instead of `@backstage/plugin-catalog-common`. - `RESOURCE_TYPE_CATALOG_ENTITY` - `catalogEntityReadPermission` From 5c592573007545ce2771e3ff9ec3a05b0ca48b55 Mon Sep 17 00:00:00 2001 From: Mike Lewis Date: Tue, 1 Mar 2022 14:40:08 +0000 Subject: [PATCH 067/150] catalog-backend: add 'breaking' prefix to existing changeset Signed-off-by: Mike Lewis --- .changeset/chilled-dolls-agree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/chilled-dolls-agree.md b/.changeset/chilled-dolls-agree.md index 0468bdfee0..750307bf53 100644 --- a/.changeset/chilled-dolls-agree.md +++ b/.changeset/chilled-dolls-agree.md @@ -2,7 +2,7 @@ '@backstage/plugin-catalog-backend': minor --- -Mark permission-related exports as alpha. This means that the exports below should now be imported from `@backstage/plugin-catalog-backend/alpha` instead of `@backstage/plugin-catalog-backend`. +**Breaking**: Mark permission-related exports as alpha. This means that the exports below should now be imported from `@backstage/plugin-catalog-backend/alpha` instead of `@backstage/plugin-catalog-backend`. - `catalogConditions` - `createCatalogPolicyDecision` From 1f5b25eeabf1ee614bc58231b6c78785ef3110e9 Mon Sep 17 00:00:00 2001 From: Mike Lewis Date: Tue, 1 Mar 2022 15:01:24 +0000 Subject: [PATCH 068/150] permission-common: changeset formatting Signed-off-by: Mike Lewis --- .changeset/nice-dragons-collect.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/nice-dragons-collect.md b/.changeset/nice-dragons-collect.md index 040b030539..b3d9efefd9 100644 --- a/.changeset/nice-dragons-collect.md +++ b/.changeset/nice-dragons-collect.md @@ -2,4 +2,4 @@ '@backstage/plugin-permission-common': patch --- -Add api doc comments to Permission type properties. +Add api doc comments to `Permission` type properties. From 0df6077ab5f6c25baa5ebd151871a01ab0bbe869 Mon Sep 17 00:00:00 2001 From: Francesco Saltori Date: Tue, 1 Mar 2022 17:14:14 +0100 Subject: [PATCH 069/150] Autoremove containers launched by DockerContainerRunner Signed-off-by: Francesco Saltori --- .changeset/grumpy-apes-repeat.md | 5 +++++ .../backend-common/src/util/DockerContainerRunner.test.ts | 2 ++ packages/backend-common/src/util/DockerContainerRunner.ts | 1 + 3 files changed, 8 insertions(+) create mode 100644 .changeset/grumpy-apes-repeat.md diff --git a/.changeset/grumpy-apes-repeat.md b/.changeset/grumpy-apes-repeat.md new file mode 100644 index 0000000000..bc47b0d422 --- /dev/null +++ b/.changeset/grumpy-apes-repeat.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +DockerContainerRunner.runContainer now automatically removes the container when its execution terminates diff --git a/packages/backend-common/src/util/DockerContainerRunner.test.ts b/packages/backend-common/src/util/DockerContainerRunner.test.ts index 8833e4518b..3208679d1f 100644 --- a/packages/backend-common/src/util/DockerContainerRunner.test.ts +++ b/packages/backend-common/src/util/DockerContainerRunner.test.ts @@ -115,6 +115,7 @@ describe('DockerContainerRunner', () => { Env: envVarsArray, WorkingDir: workingDir, HostConfig: { + AutoRemove: true, Binds: expect.arrayContaining([ `${path.join(rootDir, 'input')}:/input`, `${path.join(rootDir, 'output')}:/output`, @@ -207,6 +208,7 @@ describe('DockerContainerRunner', () => { logStream, expect.objectContaining({ HostConfig: { + AutoRemove: true, Binds: [], }, Volumes: {}, diff --git a/packages/backend-common/src/util/DockerContainerRunner.ts b/packages/backend-common/src/util/DockerContainerRunner.ts index 522328c2ec..4d4684c814 100644 --- a/packages/backend-common/src/util/DockerContainerRunner.ts +++ b/packages/backend-common/src/util/DockerContainerRunner.ts @@ -105,6 +105,7 @@ export class DockerContainerRunner implements ContainerRunner { await this.dockerClient.run(imageName, args, logStream, { Volumes, HostConfig: { + AutoRemove: true, Binds, }, ...(workingDir ? { WorkingDir: workingDir } : {}), From 8dc290872dcf820d6ba6e7fe2c779081fbcd8ef3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 1 Mar 2022 17:54:56 +0100 Subject: [PATCH 070/150] docs/local-dev: add docs for the experimental type build Signed-off-by: Patrik Oldsberg --- docs/local-dev/cli-build-system.md | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/local-dev/cli-build-system.md b/docs/local-dev/cli-build-system.md index 75f0606752..f57efc1255 100644 --- a/docs/local-dev/cli-build-system.md +++ b/docs/local-dev/cli-build-system.md @@ -552,3 +552,36 @@ The following is an excerpt of a typical setup of an isomorphic library package: }, "files": ["dist"], ``` + +## Experimental Type Build + +The Backstage CLI has an experimental feature where multiple different type definition files can be generated for different release stages. The release stages are marked in the [TSDoc](https://tsdoc.org/) for each individual export, using either `@public`, `@alpha`, or `@beta`. Rather than just building a single `index.d.ts` file, the build process will instead output `index.d.ts`, `index.beta.d.ts`, and `index.alpha.d.ts`. Each of these files will have exports from more unstable release stages stripped, meaning that `index.d.ts` will omit all exports marked with `@alpha` or `@beta`, while `index.beta.d.ts` will omit all exports marked with `@alpha`. + +This feature is aimed at projects that publish to package registries and wish to maintain different levels of API stability within each package. There is no need to use this within a single monorepo, as it has no effect due to only applying to built and published packages. + +In order for the experimental type build to work, `@microsoft/api-extractor` must be installed in your project, as it is an optional peer dependency of the Backstage CLI. There are then three steps that need to be taken for each package where you want to enable this feature: + +- Add the `--experimental-type-build` flag to the `"build"` script of the package. +- Add either one or both of `"alphaTypes"` and `"betaTypes"` to the `"publishConfig"` of the package: + ```json + "publishConfig": { + ... + "types": "dist/index.d.ts", + "alphaTypes": "dist/index.alpha.d.ts", + "betaTypes": "dist/index.beta.d.ts" + }, + ``` +- Add either one or both of `"alpha"` and `"beta"` to the `"files"` of the package: + ```json + "files": [ + "dist", + "alpha", + "beta" + ] + ``` + +Once this setup is complete, users of the published packages will only be able to access the stable API via the main package entry point, for example `@acme/my-plugin`. Exports marked with `@alpha` or `@beta` will only be available via the `/alpha` entry point, for example `@acme/my-plugin/alpha`, and exports marked with `@beta` will only be available via `/beta`. This does not apply within the monorepo that contains the package. There all exports still have to be imported via the main entry point. + +Note that these different entry points are only separated during type checking. At runtime they all share the same code which contains the exports from all releases stages. + +An example of this setup can be seen in the [`@backstage/catalog-model`](https://github.com/backstage/backstage/blob/da0675bf9f28ed1460f03635a22d3c26abd14707/packages/catalog-model/package.json#L14) package, which has enabled `alpha` type exports. With this setup, exports marked as `@alpha` are only available for import via `@backstage/catalog-model/alpha`. The `@backstage/catalog-model` package currently does not have any exports marked as `@beta`, or a `/beta` entry point. From 1bd6e019abd3e4e7dd845ec575d97b73e9a135b0 Mon Sep 17 00:00:00 2001 From: su-gupta Date: Tue, 1 Mar 2022 12:15:18 -0500 Subject: [PATCH 071/150] change to md files Signed-off-by: su-gupta --- contrib/search/confluence/ConfluenceCollator | 87 ----------------- .../search/confluence/ConfluenceCollator.md | 95 +++++++++++++++++++ ...ltListItem => ConfluenceResultListItem.md} | 19 +++- contrib/search/confluence/README.md | 32 ++++++- 4 files changed, 138 insertions(+), 95 deletions(-) delete mode 100644 contrib/search/confluence/ConfluenceCollator create mode 100644 contrib/search/confluence/ConfluenceCollator.md rename contrib/search/confluence/{ConfluenceResultListItem => ConfluenceResultListItem.md} (72%) diff --git a/contrib/search/confluence/ConfluenceCollator b/contrib/search/confluence/ConfluenceCollator deleted file mode 100644 index 5d6668b953..0000000000 --- a/contrib/search/confluence/ConfluenceCollator +++ /dev/null @@ -1,87 +0,0 @@ -import { DocumentCollator } from '@backstage/search-common'; -import fetch from 'cross-fetch' - -export class ConfluenceCollator implements DocumentCollator { - public readonly type: string = 'confluence'; - - async execute() { - - const ConfluenceUrlBase = 'https://{CONFLUENCE-ORG-NAME}.atlassian.net/wiki/rest/api' - - async function getConfluenceData(requestUrl: string) { - var emptyJson = {} - try { - const res = await fetch(requestUrl, { - method: 'get', - headers: { - 'Authorization': `Basic ${process.env.CONFLUENCE_TOKEN}` - }, - }); - if (res.ok) { - return await res.json(); - } - } catch (err) { - console.error(err); - } - return emptyJson - } - - async function getSpaces(): Promise { - const data = await getConfluenceData(`${ConfluenceUrlBase}/space?&limit=1000&type=global&status=current`); - let spacesList = [] - if (data["results"]) { - const results = data["results"]; - for (const result of results) { - spacesList.push(result["key"]) - } - } - return spacesList - } - - async function getDocumentsFromSpaces(spaces: string[]): Promise { - let documentsList = [] - for (var space of spaces) { - let next = true - let requestUrl = `${ConfluenceUrlBase}/content?limit=1000&status=current&spaceKey=${space}` - while (next) { - const data = await getConfluenceData(requestUrl) - if (data["results"]) { - const results = data["results"] - for (const result of results) { - documentsList.push(result["_links"]["self"]) - } - if (data["_links"]["next"]) { - requestUrl = data["_links"]["base"] + data["_links"]["next"] - } else { - next = false - } - } else { - break - } - } - } - return documentsList - } - - async function getDocumentInfo(documents: string[]) { - let documentInfo = [] - for (var documentUrl of documents) { - const data = await getConfluenceData(documentUrl + '?expand=body.storage') - if (data["status"] && data["status"]=="current") { - const documentMetaData = { - title: data["title"], - text: data["body"]["storage"]["value"], - location: data["_links"]["base"] + data["_links"]["webui"], - } - documentInfo.push(documentMetaData) - } - } - return documentInfo - } - - const spacesList = await getSpaces(); - const documentsList = await getDocumentsFromSpaces(spacesList); - const documentMetaDataList = await getDocumentInfo(documentsList); - return documentMetaDataList - } -} diff --git a/contrib/search/confluence/ConfluenceCollator.md b/contrib/search/confluence/ConfluenceCollator.md new file mode 100644 index 0000000000..f636ae88ea --- /dev/null +++ b/contrib/search/confluence/ConfluenceCollator.md @@ -0,0 +1,95 @@ +ConfluenceCollator.ts reference + +```ts +import { DocumentCollator } from '@backstage/search-common'; +import fetch from 'cross-fetch'; + +export class ConfluenceCollator implements DocumentCollator { + public readonly type: string = 'confluence'; + + async execute() { + const ConfluenceUrlBase = + 'https://{CONFLUENCE-ORG-NAME}.atlassian.net/wiki/rest/api'; + + async function getConfluenceData(requestUrl: string) { + var emptyJson = {}; + try { + const res = await fetch(requestUrl, { + method: 'get', + headers: { + Authorization: `Basic ${process.env.CONFLUENCE_TOKEN}`, + }, + }); + if (res.ok) { + return await res.json(); + } + } catch (err) { + console.error(err); + } + return emptyJson; + } + + async function getSpaces(): Promise { + const data = await getConfluenceData( + `${ConfluenceUrlBase}/space?&limit=1000&type=global&status=current`, + ); + let spacesList = []; + if (data['results']) { + const results = data['results']; + for (const result of results) { + spacesList.push(result['key']); + } + } + return spacesList; + } + + async function getDocumentsFromSpaces(spaces: string[]): Promise { + let documentsList = []; + for (var space of spaces) { + let next = true; + let requestUrl = `${ConfluenceUrlBase}/content?limit=1000&status=current&spaceKey=${space}`; + while (next) { + const data = await getConfluenceData(requestUrl); + if (data['results']) { + const results = data['results']; + for (const result of results) { + documentsList.push(result['_links']['self']); + } + if (data['_links']['next']) { + requestUrl = data['_links']['base'] + data['_links']['next']; + } else { + next = false; + } + } else { + break; + } + } + } + return documentsList; + } + + async function getDocumentInfo(documents: string[]) { + let documentInfo = []; + for (var documentUrl of documents) { + const data = await getConfluenceData( + documentUrl + '?expand=body.storage', + ); + if (data['status'] && data['status'] == 'current') { + const documentMetaData = { + title: data['title'], + text: data['body']['storage']['value'], + location: data['_links']['base'] + data['_links']['webui'], + }; + documentInfo.push(documentMetaData); + } + } + return documentInfo; + } + + const spacesList = await getSpaces(); + const documentsList = await getDocumentsFromSpaces(spacesList); + const documentMetaDataList = await getDocumentInfo(documentsList); + return documentMetaDataList; + } +} +``` diff --git a/contrib/search/confluence/ConfluenceResultListItem b/contrib/search/confluence/ConfluenceResultListItem.md similarity index 72% rename from contrib/search/confluence/ConfluenceResultListItem rename to contrib/search/confluence/ConfluenceResultListItem.md index cd6d13bdea..ba7c4f91a9 100644 --- a/contrib/search/confluence/ConfluenceResultListItem +++ b/contrib/search/confluence/ConfluenceResultListItem.md @@ -1,3 +1,6 @@ +ConfluenceResultListItem.tsx reference + +```tsx import React from 'react'; import { Link } from '@backstage/core-components'; import { IndexableDocument } from '@backstage/search-common'; @@ -17,26 +20,31 @@ export const ConfluenceResultListItem = ({ result }: Props) => { const chars = []; let isTag = false; for (const c of result.text.substring(0, 500)) { - if (c === "<") { + if (c === '<') { isTag = true; continue; } - if (c === ">") { + if (c === '>') { isTag = false; - chars.push(" ") + chars.push(' '); continue; } if (!isTag) { chars.push(c); } } - const excerpt = chars.join("").substring(0, 80) + (result.text.length > 80 ? "..." : ""); + const excerpt = + chars.join('').substring(0, 80) + (result.text.length > 80 ? '...' : ''); return ( - + { ); }; +``` diff --git a/contrib/search/confluence/README.md b/contrib/search/confluence/README.md index 5ebd275560..80beb9c169 100644 --- a/contrib/search/confluence/README.md +++ b/contrib/search/confluence/README.md @@ -1,6 +1,32 @@ # Confluence These files help you add Confluence as a source to the Backstage Search plugin. -To do so, add both files in this directory under the packages/backend/src/plugins/search/ pathway in your Backstage app as TypeScript files. -Then, update your packages/app/src/components/search/SearchPage.tsx and packages/backend/src/plugins/search.ts -to include the new Search source. +To do so, add both files in this directory under the packages/backend/src/plugins/search/ pathway in your Backstage app. +Then, add the following code to your packages/app/src/components/search/SearchPage.tsx: + +```tsx +import { ConfluenceResultListItem } from './ConfluenceResultListItem'; +``` + +```tsx +case 'confluence': + return ( + + ); +``` + +and the following to packages/backend/src/plugins/search.ts: + +```ts +import { ConfluenceCollator } from './search/ConfluenceCollator'; +``` + +```ts +indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, + collator: new ConfluenceCollator(), +}); +``` From 0c8ba31d72670d9ff5eda3ca726e061fff088115 Mon Sep 17 00:00:00 2001 From: Harry Hogg Date: Tue, 1 Mar 2022 17:46:39 +0000 Subject: [PATCH 072/150] plugin-auth-backend: Added validation to ensure any custom auth resolvers are using EntityRefs for subject claims Signed-off-by: Harry Hogg --- .changeset/tasty-poems-raise.md | 5 +++ .../src/identity/TokenFactory.test.ts | 39 ++++++++++++++++--- .../auth-backend/src/identity/TokenFactory.ts | 10 +++++ 3 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 .changeset/tasty-poems-raise.md diff --git a/.changeset/tasty-poems-raise.md b/.changeset/tasty-poems-raise.md new file mode 100644 index 0000000000..47556cb3ad --- /dev/null +++ b/.changeset/tasty-poems-raise.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': minor +--- + +Added validation to TokenFactory.issueToken that ensure any sub claim given is a valid entityRef. This will affect any custom resolver functions given to auth providers. diff --git a/plugins/auth-backend/src/identity/TokenFactory.test.ts b/plugins/auth-backend/src/identity/TokenFactory.test.ts index f5f06f1209..4b1c1804df 100644 --- a/plugins/auth-backend/src/identity/TokenFactory.test.ts +++ b/plugins/auth-backend/src/identity/TokenFactory.test.ts @@ -18,6 +18,7 @@ import { MemoryKeyStore } from './MemoryKeyStore'; import { TokenFactory } from './TokenFactory'; import { getVoidLogger } from '@backstage/backend-common'; import { JWKS, JSONWebKey, JWT } from 'jose'; +import { stringifyEntityRef } from '@backstage/catalog-model'; const logger = getVoidLogger(); @@ -28,6 +29,12 @@ function jwtKid(jwt: string): string { return header.kid; } +const entityRef = stringifyEntityRef({ + kind: 'User', + namespace: 'default', + name: 'JackFrost', +}); + describe('TokenFactory', () => { it('should issue valid tokens signed by a listed key', async () => { const keyDurationSeconds = 5; @@ -39,7 +46,7 @@ describe('TokenFactory', () => { }); await expect(factory.listPublicKeys()).resolves.toEqual({ keys: [] }); - const token = await factory.issueToken({ claims: { sub: 'foo' } }); + const token = await factory.issueToken({ claims: { sub: entityRef } }); const { keys } = await factory.listPublicKeys(); const keyStore = JWKS.asKeyStore({ @@ -53,7 +60,7 @@ describe('TokenFactory', () => { expect(payload).toEqual({ iss: 'my-issuer', aud: 'backstage', - sub: 'foo', + sub: entityRef, iat: expect.any(Number), exp: expect.any(Number), }); @@ -71,8 +78,12 @@ describe('TokenFactory', () => { logger, }); - const token1 = await factory.issueToken({ claims: { sub: 'foo' } }); - const token2 = await factory.issueToken({ claims: { sub: 'foo' } }); + const token1 = await factory.issueToken({ + claims: { sub: entityRef }, + }); + const token2 = await factory.issueToken({ + claims: { sub: entityRef }, + }); expect(jwtKid(token1)).toBe(jwtKid(token2)); await expect(factory.listPublicKeys()).resolves.toEqual({ @@ -89,7 +100,9 @@ describe('TokenFactory', () => { keys: [], }); - const token3 = await factory.issueToken({ claims: { sub: 'foo' } }); + const token3 = await factory.issueToken({ + claims: { sub: entityRef }, + }); expect(jwtKid(token3)).not.toBe(jwtKid(token2)); await expect(factory.listPublicKeys()).resolves.toEqual({ @@ -100,4 +113,20 @@ describe('TokenFactory', () => { ], }); }); + + it('should throw an error with a non entityRef sub claim', async () => { + const keyDurationSeconds = 5; + const factory = new TokenFactory({ + issuer: 'my-issuer', + keyStore: new MemoryKeyStore(), + keyDurationSeconds, + logger, + }); + + await expect(() => { + return factory.issueToken({ + claims: { sub: 'UserId' }, + }); + }).rejects.toThrowError(); + }); }); diff --git a/plugins/auth-backend/src/identity/TokenFactory.ts b/plugins/auth-backend/src/identity/TokenFactory.ts index 041a292d44..cb2a8b453d 100644 --- a/plugins/auth-backend/src/identity/TokenFactory.ts +++ b/plugins/auth-backend/src/identity/TokenFactory.ts @@ -19,6 +19,7 @@ import { JSONWebKey, JWK, JWS } from 'jose'; import { Logger } from 'winston'; import { v4 as uuid } from 'uuid'; import { DateTime } from 'luxon'; +import { parseEntityRef } from '@backstage/catalog-model'; const MS_IN_S = 1000; @@ -72,6 +73,15 @@ export class TokenFactory implements TokenIssuer { const iat = Math.floor(Date.now() / MS_IN_S); const exp = iat + this.keyDurationSeconds; + // Validate that the subject claim is a valid EntityRef + try { + parseEntityRef(sub); + } catch (error) { + throw new Error( + '"sub" claim provided by the auth resolver is not a valid EntityRef.', + ); + } + this.logger.info(`Issuing token for ${sub}, with entities ${ent ?? []}`); return JWS.sign({ iss, sub, aud, iat, exp, ent }, key, { From 4368196fba0d145fdb7bb478301fb5f8e92a89b0 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Wed, 2 Mar 2022 10:06:23 +1100 Subject: [PATCH 073/150] Accept object in shouldBuild for future extensibility Signed-off-by: Nikolas Skoufis --- plugins/techdocs-backend/api-report.md | 7 ++++++- plugins/techdocs-backend/src/index.ts | 1 + .../src/service/DocsBuildStrategy.test.ts | 4 ++-- .../src/service/DocsBuildStrategy.ts | 13 +++++++++++-- plugins/techdocs-backend/src/service/index.ts | 5 ++++- plugins/techdocs-backend/src/service/router.ts | 2 +- 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index 3ce3a2504b..2bf4664f77 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -45,7 +45,7 @@ export class DefaultTechDocsCollator implements DocumentCollator { // @public export interface DocsBuildStrategy { // (undocumented) - shouldBuild(entity: Entity): Promise; + shouldBuild(params: ShouldBuildParameters): Promise; } // @public @@ -76,6 +76,11 @@ export type RouterOptions = | RecommendedDeploymentOptions | OutOfTheBoxDeploymentOptions; +// @public +export type ShouldBuildParameters = { + entity: Entity; +}; + // @public export type TechDocsCollatorOptions = { discovery: PluginEndpointDiscovery; diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts index 2a17bf2736..01acbea2cc 100644 --- a/plugins/techdocs-backend/src/index.ts +++ b/plugins/techdocs-backend/src/index.ts @@ -26,6 +26,7 @@ export type { RecommendedDeploymentOptions, OutOfTheBoxDeploymentOptions, DocsBuildStrategy, + ShouldBuildParameters, } from './service'; export { DefaultTechDocsCollator } from './search'; diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts index 76b70e67e9..84bd960f8f 100644 --- a/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts @@ -46,7 +46,7 @@ describe('DefaultDocsBuildStrategy', () => { MockedConfigReader.prototype.getString.mockReturnValue('local'); - const result = await defaultDocsBuildStrategy.shouldBuild(entity); + const result = await defaultDocsBuildStrategy.shouldBuild({ entity }); expect(result).toBe(true); }); @@ -57,7 +57,7 @@ describe('DefaultDocsBuildStrategy', () => { MockedConfigReader.prototype.getString.mockReturnValue('external'); - const result = await defaultDocsBuildStrategy.shouldBuild(entity); + const result = await defaultDocsBuildStrategy.shouldBuild({ entity }); expect(result).toBe(false); }); diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts index e0220d7a32..42a16234f9 100644 --- a/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts @@ -16,13 +16,22 @@ import { Entity } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; +/** + * Parameters passed to the shouldBuild method on the DocsBuildStrategy interface + * + * @public + */ +export type ShouldBuildParameters = { + entity: Entity; +}; + /** * A strategy for when to build TechDocs locally, and when to skip building TechDocs (allowing for an external build) * * @public */ export interface DocsBuildStrategy { - shouldBuild(entity: Entity): Promise; + shouldBuild(params: ShouldBuildParameters): Promise; } export class DefaultDocsBuildStrategy { @@ -36,7 +45,7 @@ export class DefaultDocsBuildStrategy { return new DefaultDocsBuildStrategy(config); } - async shouldBuild(_: Entity): Promise { + async shouldBuild(_: ShouldBuildParameters): Promise { return this.config.getString('techdocs.builder') === 'local'; } } diff --git a/plugins/techdocs-backend/src/service/index.ts b/plugins/techdocs-backend/src/service/index.ts index 29db2d2038..7355a34e32 100644 --- a/plugins/techdocs-backend/src/service/index.ts +++ b/plugins/techdocs-backend/src/service/index.ts @@ -20,4 +20,7 @@ export type { RecommendedDeploymentOptions, OutOfTheBoxDeploymentOptions, } from './router'; -export type { DocsBuildStrategy } from './DocsBuildStrategy'; +export type { + DocsBuildStrategy, + ShouldBuildParameters, +} from './DocsBuildStrategy'; diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 4d51ebd485..abe83b3d47 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -221,7 +221,7 @@ export async function createRouter( // techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local' // If set to 'external', it will assume that an external process (e.g. CI/CD pipeline // of the repository) is responsible for building and publishing documentation to the storage provider - const shouldBuild = await docsBuildStrategy.shouldBuild(entity); + const shouldBuild = await docsBuildStrategy.shouldBuild({ entity }); if (!shouldBuild) { // However, if caching is enabled, take the opportunity to check and // invalidate stale cache entries. From 63bb0a96449fd3ff9b2448596027c26446fd03d9 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Wed, 2 Mar 2022 10:09:28 +1100 Subject: [PATCH 074/150] Update comments and logs for shouldBuild Signed-off-by: Nikolas Skoufis --- plugins/techdocs-backend/src/service/router.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index abe83b3d47..183e26c95d 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -218,9 +218,11 @@ export async function createRouter( responseHandler = createEventStream(res); } - // techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local' - // If set to 'external', it will assume that an external process (e.g. CI/CD pipeline - // of the repository) is responsible for building and publishing documentation to the storage provider + // By default, techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to + // 'local'. If set to 'external', it will assume that an external process (e.g. CI/CD pipeline + // of the repository) is responsible for building and publishing documentation to the storage provider. + // Altering the implementation of the injected docsBuildStrategy allows for more complex behaviours, based on + // either config or the properties of the entity (e.g. annotations, labels, spec fields etc.). const shouldBuild = await docsBuildStrategy.shouldBuild({ entity }); if (!shouldBuild) { // However, if caching is enabled, take the opportunity to check and @@ -253,7 +255,7 @@ export async function createRouter( responseHandler.error( new Error( - "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'preparer' was provided to the router initialization.", + "Invalid configuration. 'docsBuildStrategy.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization.", ), ); }); From 9a0510144f31ae354bc9906283f3614ac708c3a9 Mon Sep 17 00:00:00 2001 From: Phil Kuang Date: Tue, 1 Mar 2022 18:21:32 -0500 Subject: [PATCH 075/150] refactor(redis): require protocol in connection string Signed-off-by: Phil Kuang --- .changeset/chilled-items-trade.md | 13 +++++++++++++ docs/overview/architecture-overview.md | 2 +- packages/backend-common/config.d.ts | 2 +- .../backend-common/src/cache/CacheManager.test.ts | 7 +++---- packages/backend-common/src/cache/CacheManager.ts | 2 +- 5 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 .changeset/chilled-items-trade.md diff --git a/.changeset/chilled-items-trade.md b/.changeset/chilled-items-trade.md new file mode 100644 index 0000000000..b069885498 --- /dev/null +++ b/.changeset/chilled-items-trade.md @@ -0,0 +1,13 @@ +--- +'@backstage/backend-common': minor +--- + +**BREAKING**: The connection string for `redis` cache store now requires a protocol prefix. + +```diff +backend: + cache: + store: redis +- connection: user:pass@cache.example.com:6379 ++ connection: redis://user:pass@cache.example.com:6379 +``` diff --git a/docs/overview/architecture-overview.md b/docs/overview/architecture-overview.md index 358dc53852..5b6b81cf10 100644 --- a/docs/overview/architecture-overview.md +++ b/docs/overview/architecture-overview.md @@ -310,7 +310,7 @@ backend: backend: cache: store: redis - connection: user:pass@cache.example.com:6379 + connection: redis://user:pass@cache.example.com:6379 ``` Contributions supporting other cache stores are welcome! diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 57645d9fda..0c3e3132c0 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -137,7 +137,7 @@ export interface Config { | { store: 'redis'; /** - * A redis connection string in the form `user:pass@host:port`. + * A redis connection string in the form `redis://user:pass@host:port`. * @secret */ connection: string; diff --git a/packages/backend-common/src/cache/CacheManager.test.ts b/packages/backend-common/src/cache/CacheManager.test.ts index 660aa82bb0..c6cad2e49b 100644 --- a/packages/backend-common/src/cache/CacheManager.test.ts +++ b/packages/backend-common/src/cache/CacheManager.test.ts @@ -195,14 +195,13 @@ describe('CacheManager', () => { }); it('returns a Redis client when configured', () => { - const redisHostAndPort = '127.0.0.1:6379'; - const expectedHost = `redis://${redisHostAndPort}`; + const redisConnection = 'redis://127.0.0.1:6379'; const manager = CacheManager.fromConfig( new ConfigReader({ backend: { cache: { store: 'redis', - connection: redisHostAndPort, + connection: redisConnection, }, }, }), @@ -218,7 +217,7 @@ describe('CacheManager', () => { expect(mockCacheCalls[0][0].store).toBeInstanceOf(KeyvRedis); const redis = KeyvRedis as jest.Mock; const mockRedisCalls = redis.mock.calls.splice(-1); - expect(mockRedisCalls[0][0]).toEqual(expectedHost); + expect(mockRedisCalls[0][0]).toEqual(redisConnection); }); describe('connection errors', () => { diff --git a/packages/backend-common/src/cache/CacheManager.ts b/packages/backend-common/src/cache/CacheManager.ts index d63ac6d940..d8652e42d8 100644 --- a/packages/backend-common/src/cache/CacheManager.ts +++ b/packages/backend-common/src/cache/CacheManager.ts @@ -133,7 +133,7 @@ export class CacheManager { return new Keyv({ namespace: pluginId, ttl: defaultTtl, - store: new KeyvRedis(`redis://${this.connection}`), + store: new KeyvRedis(this.connection), }); } From eed6b57cf84dd6fe896684a8693ba2a2c71c9cec Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Wed, 2 Mar 2022 11:09:48 +1100 Subject: [PATCH 076/150] Update error message in tests Signed-off-by: Nikolas Skoufis --- plugins/techdocs-backend/src/service/router.test.ts | 4 ++-- plugins/techdocs-backend/src/service/router.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/techdocs-backend/src/service/router.test.ts b/plugins/techdocs-backend/src/service/router.test.ts index 521024252c..c7cda203bb 100644 --- a/plugins/techdocs-backend/src/service/router.test.ts +++ b/plugins/techdocs-backend/src/service/router.test.ts @@ -216,7 +216,7 @@ describe('createRouter', () => { expect(response.status).toBe(500); expect(response.text).toMatch( - /Invalid configuration\. 'techdocs\.builder' was set to 'local' but no 'preparer' was provided to the router initialization/, + /Invalid configuration\. docsBuildStrategy\.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization./, ); expect(MockDocsSynchronizer.prototype.doSync).toBeCalledTimes(0); @@ -343,7 +343,7 @@ data: {"updated":false} expect(response.get('content-type')).toBe('text/event-stream'); expect(response.text).toEqual( `event: error -data: "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'preparer' was provided to the router initialization." +data: "Invalid configuration. docsBuildStrategy.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization." `, ); diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 183e26c95d..73f212d049 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -255,7 +255,7 @@ export async function createRouter( responseHandler.error( new Error( - "Invalid configuration. 'docsBuildStrategy.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization.", + "Invalid configuration. docsBuildStrategy.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization.", ), ); }); From 0cd42252c98dffb9329498508cb9b712e5fc2be6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Mar 2022 04:08:55 +0000 Subject: [PATCH 077/150] chore(deps): bump @typescript-eslint/parser from 5.9.1 to 5.13.0 Bumps [@typescript-eslint/parser](https://github.com/typescript-eslint/typescript-eslint/tree/HEAD/packages/parser) from 5.9.1 to 5.13.0. - [Release notes](https://github.com/typescript-eslint/typescript-eslint/releases) - [Changelog](https://github.com/typescript-eslint/typescript-eslint/blob/main/packages/parser/CHANGELOG.md) - [Commits](https://github.com/typescript-eslint/typescript-eslint/commits/v5.13.0/packages/parser) --- updated-dependencies: - dependency-name: "@typescript-eslint/parser" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- yarn.lock | 74 +++++++++++++++++++++++++++---------------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/yarn.lock b/yarn.lock index 69a359daa7..bc2dad5f55 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6710,15 +6710,23 @@ eslint-utils "^3.0.0" "@typescript-eslint/parser@^5.9.0": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.9.1.tgz#b114011010a87e17b3265ca715e16c76a9834cef" - integrity sha512-PLYO0AmwD6s6n0ZQB5kqPgfvh73p0+VqopQQLuNfi7Lm0EpfKyDalchpVwkE+81k5HeiRrTV/9w1aNHzjD7C4g== + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.13.0.tgz#0394ed8f2f849273c0bf4b811994d177112ced5c" + integrity sha512-GdrU4GvBE29tm2RqWOM0P5QfCtgCyN4hXICj/X9ibKED16136l9ZpoJvCL5pSKtmJzA+NRDzQ312wWMejCVVfg== dependencies: - "@typescript-eslint/scope-manager" "5.9.1" - "@typescript-eslint/types" "5.9.1" - "@typescript-eslint/typescript-estree" "5.9.1" + "@typescript-eslint/scope-manager" "5.13.0" + "@typescript-eslint/types" "5.13.0" + "@typescript-eslint/typescript-estree" "5.13.0" debug "^4.3.2" +"@typescript-eslint/scope-manager@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.13.0.tgz#cf6aff61ca497cb19f0397eea8444a58f46156b6" + integrity sha512-T4N8UvKYDSfVYdmJq7g2IPJYCRzwtp74KyDZytkR4OL3NRupvswvmJQJ4CX5tDSurW2cvCc1Ia1qM7d0jpa7IA== + dependencies: + "@typescript-eslint/types" "5.13.0" + "@typescript-eslint/visitor-keys" "5.13.0" + "@typescript-eslint/scope-manager@5.9.0": version "5.9.0" resolved "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.9.0.tgz#02dfef920290c1dcd7b1999455a3eaae7a1a3117" @@ -6727,14 +6735,6 @@ "@typescript-eslint/types" "5.9.0" "@typescript-eslint/visitor-keys" "5.9.0" -"@typescript-eslint/scope-manager@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.9.1.tgz#6c27be89f1a9409f284d95dfa08ee3400166fe69" - integrity sha512-8BwvWkho3B/UOtzRyW07ffJXPaLSUKFBjpq8aqsRvu6HdEuzCY57+ffT7QoV4QXJXWSU1+7g3wE4AlgImmQ9pQ== - dependencies: - "@typescript-eslint/types" "5.9.1" - "@typescript-eslint/visitor-keys" "5.9.1" - "@typescript-eslint/type-utils@5.9.0": version "5.9.0" resolved "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.9.0.tgz#fd5963ead04bc9b7af9c3a8e534d8d39f1ce5f93" @@ -6744,15 +6744,28 @@ debug "^4.3.2" tsutils "^3.21.0" +"@typescript-eslint/types@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.13.0.tgz#da1de4ae905b1b9ff682cab0bed6b2e3be9c04e5" + integrity sha512-LmE/KO6DUy0nFY/OoQU0XelnmDt+V8lPQhh8MOVa7Y5k2gGRd6U9Kp3wAjhB4OHg57tUO0nOnwYQhRRyEAyOyg== + "@typescript-eslint/types@5.9.0": version "5.9.0" resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.9.0.tgz#e5619803e39d24a03b3369506df196355736e1a3" integrity sha512-mWp6/b56Umo1rwyGCk8fPIzb9Migo8YOniBGPAQDNC6C52SeyNGN4gsVwQTAR+RS2L5xyajON4hOLwAGwPtUwg== -"@typescript-eslint/types@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.9.1.tgz#1bef8f238a2fb32ebc6ff6d75020d9f47a1593c6" - integrity sha512-SsWegWudWpkZCwwYcKoDwuAjoZXnM1y2EbEerTHho19Hmm+bQ56QG4L4jrtCu0bI5STaRTvRTZmjprWlTw/5NQ== +"@typescript-eslint/typescript-estree@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.13.0.tgz#b37c07b748ff030a3e93d87c842714e020b78141" + integrity sha512-Q9cQow0DeLjnp5DuEDjLZ6JIkwGx3oYZe+BfcNuw/POhtpcxMTy18Icl6BJqTSd+3ftsrfuVb7mNHRZf7xiaNA== + dependencies: + "@typescript-eslint/types" "5.13.0" + "@typescript-eslint/visitor-keys" "5.13.0" + debug "^4.3.2" + globby "^11.0.4" + is-glob "^4.0.3" + semver "^7.3.5" + tsutils "^3.21.0" "@typescript-eslint/typescript-estree@5.9.0": version "5.9.0" @@ -6767,18 +6780,13 @@ semver "^7.3.5" tsutils "^3.21.0" -"@typescript-eslint/typescript-estree@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.9.1.tgz#d5b996f49476495070d2b8dd354861cf33c005d6" - integrity sha512-gL1sP6A/KG0HwrahVXI9fZyeVTxEYV//6PmcOn1tD0rw8VhUWYeZeuWHwwhnewnvEMcHjhnJLOBhA9rK4vmb8A== +"@typescript-eslint/visitor-keys@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.13.0.tgz#f45ff55bcce16403b221ac9240fbeeae4764f0fd" + integrity sha512-HLKEAS/qA1V7d9EzcpLFykTePmOQqOFim8oCvhY3pZgQ8Hi38hYpHd9e5GN6nQBFQNecNhws5wkS9Y5XIO0s/g== dependencies: - "@typescript-eslint/types" "5.9.1" - "@typescript-eslint/visitor-keys" "5.9.1" - debug "^4.3.2" - globby "^11.0.4" - is-glob "^4.0.3" - semver "^7.3.5" - tsutils "^3.21.0" + "@typescript-eslint/types" "5.13.0" + eslint-visitor-keys "^3.0.0" "@typescript-eslint/visitor-keys@5.9.0": version "5.9.0" @@ -6788,14 +6796,6 @@ "@typescript-eslint/types" "5.9.0" eslint-visitor-keys "^3.0.0" -"@typescript-eslint/visitor-keys@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.9.1.tgz#f52206f38128dd4f675cf28070a41596eee985b7" - integrity sha512-Xh37pNz9e9ryW4TVdwiFzmr4hloty8cFj8GTWMXh3Z8swGwyQWeCcNgF0hm6t09iZd6eiZmIf4zHedQVP6TVtg== - dependencies: - "@typescript-eslint/types" "5.9.1" - eslint-visitor-keys "^3.0.0" - "@vscode/sqlite3@^5.0.7": version "5.0.7" resolved "https://registry.npmjs.org/@vscode/sqlite3/-/sqlite3-5.0.7.tgz#358df36bb0e9e735c54785e3e4b9b2dce1d32895" From b2ccb676c2881c7ea9ba2a888c8f68a4f9a01fcf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Mar 2022 04:09:23 +0000 Subject: [PATCH 078/150] chore(deps): bump raw-body from 2.5.0 to 2.5.1 Bumps [raw-body](https://github.com/stream-utils/raw-body) from 2.5.0 to 2.5.1. - [Release notes](https://github.com/stream-utils/raw-body/releases) - [Changelog](https://github.com/stream-utils/raw-body/blob/master/HISTORY.md) - [Commits](https://github.com/stream-utils/raw-body/compare/2.5.0...2.5.1) --- updated-dependencies: - dependency-name: raw-body dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 69a359daa7..427eb00081 100644 --- a/yarn.lock +++ b/yarn.lock @@ -20645,9 +20645,9 @@ raw-body@2.4.3: unpipe "1.0.0" raw-body@^2.4.1: - version "2.5.0" - resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.0.tgz#865890d9435243e9fe6141feb4decf929a6e1525" - integrity sha512-XpyZ6O7PVu3ItMQl0LslfsRoKxMOxi3SzDkrOtxMES5AqLFpYjQCryxI4LGygUN2jL+RgFsPkMPPlG7cg/47+A== + version "2.5.1" + resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" + integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== dependencies: bytes "3.1.2" http-errors "2.0.0" From ac7b1161a6c403388bc1cbabe3947e0b7c78bd5a Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 1 Mar 2022 15:50:03 +0100 Subject: [PATCH 079/150] catalog-model: Remove deprecations Signed-off-by: Johan Haals --- .changeset/weak-news-reply.md | 23 ++ packages/catalog-model/api-report.md | 86 ----- .../catalog-model/src/entity/constants.ts | 36 -- packages/catalog-model/src/entity/index.ts | 12 - packages/catalog-model/src/entity/ref.test.ts | 319 +----------------- packages/catalog-model/src/entity/ref.ts | 77 ----- .../catalog-model/src/entity/util.test.ts | 198 ----------- packages/catalog-model/src/entity/util.ts | 133 -------- packages/catalog-model/src/index.ts | 2 +- .../catalog-model/src/location/annotation.ts | 24 -- .../catalog-model/src/location/helpers.ts | 27 -- packages/catalog-model/src/location/index.ts | 7 +- packages/catalog-model/src/location/types.ts | 10 - packages/catalog-model/src/types.ts | 11 - 14 files changed, 26 insertions(+), 939 deletions(-) create mode 100644 .changeset/weak-news-reply.md delete mode 100644 packages/catalog-model/src/entity/util.test.ts delete mode 100644 packages/catalog-model/src/entity/util.ts diff --git a/.changeset/weak-news-reply.md b/.changeset/weak-news-reply.md new file mode 100644 index 0000000000..3becabc2e6 --- /dev/null +++ b/.changeset/weak-news-reply.md @@ -0,0 +1,23 @@ +--- +'@backstage/catalog-model': minor +--- + +**Breaking**: The following changes are all breaking changes. + +Removed `EDIT_URL_ANNOTATION` and `VIEW_URL_ANNOTATION`, `LOCATION_ANNOTATION`, `ORIGIN_LOCATION_ANNOTATION`, `LOCATION_ANNOTATION`, `SOURCE_LOCATION_ANNOTATION`. All of these constants have been prefixed with ANNOTATION to be easier to find meaning `SOURCE_LOCATION_ANNOTATION` is available as `ANNOTATION_SOURCE_LOCATION`. + +Removed `parseLocationReference`, replaced by `parseLocationRef`. + +Removed `stringifyLocationReference`, replaced by `stringifyLocationRef`. + +Removed `Location` type which has been moved to `catalog-client`. + +Removed `ENTITY_DEFAULT_NAMESPACE`, replaced by `DEFAULT_NAMESPACE`. + +Removed `compareEntityToRef` compare using `stringifyEntityRef` instead. + +Removed `JSONSchema` type which should be imported from `json-schema` package instead. + +Removed utility methods: `entityHasChanges`, `generateEntityEtag`, `generateEntityUid`, `generateUpdatedEntity`. + +Removed `ENTITY_META_GENERATED_FIELDS` and `EntityRefContext`. diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 210d0187e2..4574129d6f 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -4,8 +4,6 @@ ```ts import { JsonObject } from '@backstage/types'; -import { JSONSchema7 } from 'json-schema'; -import { JsonValue } from '@backstage/types'; import { SerializedError } from '@backstage/errors'; // @alpha @@ -66,23 +64,6 @@ export class CommonValidatorFunctions { static isValidUrl(value: unknown): boolean; } -// @public @deprecated -export function compareEntityToRef( - entity: Entity, - ref: - | string - | { - kind?: string; - namespace?: string; - name: string; - } - | EntityName, - context?: { - defaultKind?: string; - defaultNamespace?: string; - }, -): boolean; - // @public interface ComponentEntityV1alpha1 extends Entity { // (undocumented) @@ -134,9 +115,6 @@ export { DomainEntityV1alpha1 }; // @public export const domainEntityV1alpha1Validator: KindValidator; -// @public @deprecated -export const EDIT_URL_ANNOTATION = 'backstage.io/edit-url'; - // @public export type Entity = { apiVersion: string; @@ -146,16 +124,6 @@ export type Entity = { relations?: EntityRelation[]; }; -// @public @deprecated -export const ENTITY_DEFAULT_NAMESPACE = 'default'; - -// @public @deprecated -export const ENTITY_META_GENERATED_FIELDS: readonly [ - 'uid', - 'etag', - 'generation', -]; - // @public export type EntityEnvelope = { apiVersion: string; @@ -171,9 +139,6 @@ export function entityEnvelopeSchemaValidator< T extends EntityEnvelope = EntityEnvelope, >(schema?: unknown): (data: unknown) => T; -// @public @deprecated -export function entityHasChanges(previous: Entity, next: Entity): boolean; - // @public export function entityKindSchemaValidator( schema: unknown, @@ -228,12 +193,6 @@ export type EntityRef = name: string; }; -// @public @deprecated -export type EntityRefContext = { - defaultKind?: string; - defaultNamespace?: string; -}; - // @public export type EntityRelation = { type: string; @@ -269,15 +228,6 @@ export class FieldFormatEntityPolicy implements EntityPolicy { enforce(entity: Entity): Promise; } -// @public @deprecated -export function generateEntityEtag(): string; - -// @public @deprecated -export function generateEntityUid(): string; - -// @public @deprecated -export function generateUpdatedEntity(previous: Entity, next: Entity): Entity; - // @public export function getEntityName(entity: Entity): EntityName; @@ -312,11 +262,6 @@ export { GroupEntityV1alpha1 }; // @public export const groupEntityV1alpha1Validator: KindValidator; -// @public @deprecated -export type JSONSchema = JSONSchema7 & { - [key in string]?: JsonValue; -}; - // @public export type KindValidator = { check(entity: Entity): Promise; @@ -342,15 +287,6 @@ export class KubernetesValidatorFunctions { static isValidObjectName(value: unknown): boolean; } -// @public @deprecated -type Location_2 = { - id: string; -} & LocationSpec; -export { Location_2 as Location }; - -// @public @deprecated -export const LOCATION_ANNOTATION = 'backstage.io/managed-by-location'; - // @public interface LocationEntityV1alpha1 extends Entity { // (undocumented) @@ -388,10 +324,6 @@ export class NoForeignRootFieldsEntityPolicy implements EntityPolicy { enforce(entity: Entity): Promise; } -// @public @deprecated -export const ORIGIN_LOCATION_ANNOTATION = - 'backstage.io/managed-by-origin-location'; - // @public @deprecated export function parseEntityName( ref: @@ -428,12 +360,6 @@ export function parseLocationRef(ref: string): { target: string; }; -// @public @deprecated -export function parseLocationReference(ref: string): { - type: string; - target: string; -}; - // @public export const RELATION_API_CONSUMED_BY = 'apiConsumedBy'; @@ -503,9 +429,6 @@ export class SchemaValidEntityPolicy implements EntityPolicy { enforce(entity: Entity): Promise; } -// @public @deprecated -export const SOURCE_LOCATION_ANNOTATION = 'backstage.io/source-location'; - // @public export function stringifyEntityRef( ref: @@ -523,12 +446,6 @@ export function stringifyLocationRef(ref: { target: string; }): string; -// @public @deprecated -export function stringifyLocationReference(ref: { - type: string; - target: string; -}): string; - // @public interface SystemEntityV1alpha1 extends Entity { // (undocumented) @@ -581,7 +498,4 @@ export type Validators = { isValidAnnotationValue(value: unknown): boolean; isValidTag(value: unknown): boolean; }; - -// @public @deprecated -export const VIEW_URL_ANNOTATION = 'backstage.io/view-url'; ``` diff --git a/packages/catalog-model/src/entity/constants.ts b/packages/catalog-model/src/entity/constants.ts index 6cef19cac9..5dc21683bd 100644 --- a/packages/catalog-model/src/entity/constants.ts +++ b/packages/catalog-model/src/entity/constants.ts @@ -14,14 +14,6 @@ * limitations under the License. */ -/** - * The namespace that entities without an explicit namespace fall into. - * - * @public - * @deprecated use {@link DEFAULT_NAMESPACE} instead. - */ -export const ENTITY_DEFAULT_NAMESPACE = 'default'; - /** * The namespace that entities without an explicit namespace fall into. * @@ -29,34 +21,6 @@ export const ENTITY_DEFAULT_NAMESPACE = 'default'; */ export const DEFAULT_NAMESPACE = 'default'; -/** - * The keys of EntityMeta that are auto-generated. - * - * @public - * @deprecated will be removed in a future release. - */ -export const ENTITY_META_GENERATED_FIELDS = [ - 'uid', - 'etag', - 'generation', -] as const; - -/** - * Annotation for linking to entity page from catalog pages. - * - * @public - * @deprecated use {@link ANNOTATION_VIEW_URL} instead. - */ -export const VIEW_URL_ANNOTATION = 'backstage.io/view-url'; - -/** - * Annotation for linking to entity edit page from catalog pages. - * - * @public - * @deprecated use {@link ANNOTATION_EDIT_URL} instead. - */ -export const EDIT_URL_ANNOTATION = 'backstage.io/edit-url'; - /** * Annotation for linking to entity page from catalog pages. * diff --git a/packages/catalog-model/src/entity/index.ts b/packages/catalog-model/src/entity/index.ts index f619d0f39e..3a346b217c 100644 --- a/packages/catalog-model/src/entity/index.ts +++ b/packages/catalog-model/src/entity/index.ts @@ -15,11 +15,7 @@ */ export { - EDIT_URL_ANNOTATION, - ENTITY_DEFAULT_NAMESPACE, DEFAULT_NAMESPACE, - ENTITY_META_GENERATED_FIELDS, - VIEW_URL_ANNOTATION, ANNOTATION_EDIT_URL, ANNOTATION_VIEW_URL, } from './constants'; @@ -38,16 +34,8 @@ export type { } from './EntityStatus'; export * from './policies'; export { - compareEntityToRef, getEntityName, parseEntityName, parseEntityRef, stringifyEntityRef, } from './ref'; -export type { EntityRefContext } from './ref'; -export { - entityHasChanges, - generateEntityEtag, - generateEntityUid, - generateUpdatedEntity, -} from './util'; diff --git a/packages/catalog-model/src/entity/ref.test.ts b/packages/catalog-model/src/entity/ref.test.ts index b0c7068fae..045c4710dc 100644 --- a/packages/catalog-model/src/entity/ref.test.ts +++ b/packages/catalog-model/src/entity/ref.test.ts @@ -15,8 +15,7 @@ */ import { DEFAULT_NAMESPACE } from './constants'; -import { Entity } from './Entity'; -import { compareEntityToRef, parseEntityName, parseEntityRef } from './ref'; +import { parseEntityName, parseEntityRef } from './ref'; describe('ref', () => { describe('parseEntityName', () => { @@ -313,320 +312,4 @@ describe('ref', () => { ).toThrow(/namespace/); }); }); - - describe('compareEntityToRef', () => { - const entityWithNamespace: Entity = { - apiVersion: 'a', - kind: 'K', - metadata: { - name: 'n', - namespace: 'ns', - }, - }; - const entityWithoutNamespace: Entity = { - apiVersion: 'a', - kind: 'K', - metadata: { - name: 'n', - }, - }; - - it('handles matching string refs', () => { - expect(compareEntityToRef(entityWithNamespace, 'K:ns/n')).toBe(true); - expect(compareEntityToRef(entityWithNamespace, 'k:nS/N')).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'K:n', { - defaultNamespace: 'ns', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'K:n', { - defaultNamespace: 'Ns', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'ns/n', { defaultKind: 'K' }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'n', { - defaultKind: 'K', - defaultNamespace: 'ns', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'N', { - defaultKind: 'k', - defaultNamespace: 'nS', - }), - ).toBe(true); - - expect(compareEntityToRef(entityWithoutNamespace, 'K:default/n')).toBe( - true, - ); - expect(compareEntityToRef(entityWithoutNamespace, 'K:deFault/n')).toBe( - true, - ); - expect( - compareEntityToRef(entityWithoutNamespace, 'K:n', { - defaultNamespace: 'default', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, 'K:n', { - defaultNamespace: 'deFault', - }), - ).toBe(true); - expect(compareEntityToRef(entityWithoutNamespace, 'K:default/n')).toBe( - true, - ); - expect(compareEntityToRef(entityWithoutNamespace, 'K:n')).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, 'default/n', { - defaultKind: 'K', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, 'n', { - defaultKind: 'K', - defaultNamespace: 'default', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, 'n', { - defaultKind: 'K', - }), - ).toBe(true); - }); - - it('handles mismatching string refs', () => { - expect(compareEntityToRef(entityWithNamespace, 'X:ns/n')).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, 'ns/n', { - defaultKind: 'X', - }), - ).toBe(false); - - expect(compareEntityToRef(entityWithNamespace, 'K:xx/n')).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, 'K:n', { - defaultNamespace: 'xx', - }), - ).toBe(false); - - expect(compareEntityToRef(entityWithNamespace, 'K:ns/x')).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, 'x', { - defaultKind: 'K', - defaultNamespace: 'ns', - }), - ).toBe(false); - }); - - it('handles matching compound refs', () => { - expect( - compareEntityToRef(entityWithNamespace, { - kind: 'K', - namespace: 'ns', - name: 'n', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, { - kind: 'k', - namespace: 'Ns', - name: 'N', - }), - ).toBe(true); - expect( - compareEntityToRef( - entityWithNamespace, - { kind: 'K', name: 'n' }, - { - defaultNamespace: 'ns', - }, - ), - ).toBe(true); - expect( - compareEntityToRef( - entityWithNamespace, - { namespace: 'ns', name: 'n' }, - { defaultKind: 'K' }, - ), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'n', { - defaultKind: 'K', - defaultNamespace: 'ns', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'N', { - defaultKind: 'k', - defaultNamespace: 'nS', - }), - ).toBe(true); - - expect( - compareEntityToRef(entityWithoutNamespace, { - kind: 'K', - namespace: 'default', - name: 'n', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, { - kind: 'k', - namespace: 'deFault', - name: 'N', - }), - ).toBe(true); - expect( - compareEntityToRef( - entityWithoutNamespace, - { kind: 'K', name: 'n' }, - { - defaultNamespace: 'default', - }, - ), - ).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, { kind: 'K', name: 'n' }), - ).toBe(true); - expect( - compareEntityToRef( - entityWithoutNamespace, - { namespace: 'default', name: 'n' }, - { - defaultKind: 'K', - }, - ), - ).toBe(true); - expect( - compareEntityToRef( - entityWithoutNamespace, - { name: 'n' }, - { - defaultKind: 'K', - defaultNamespace: 'default', - }, - ), - ).toBe(true); - expect( - compareEntityToRef( - entityWithoutNamespace, - { name: 'N' }, - { - defaultKind: 'k', - defaultNamespace: 'defAult', - }, - ), - ).toBe(true); - expect( - compareEntityToRef( - entityWithoutNamespace, - { name: 'n' }, - { - defaultKind: 'K', - }, - ), - ).toBe(true); - }); - - it('handles mismatching compound refs', () => { - expect( - compareEntityToRef(entityWithNamespace, { - kind: 'X', - namespace: 'ns', - name: 'n', - }), - ).toBe(false); - expect( - compareEntityToRef( - entityWithNamespace, - { - namespace: 'ns', - name: 'n', - }, - { defaultKind: 'X' }, - ), - ).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, { - kind: 'X', - namespace: 'default', - name: 'n', - }), - ).toBe(false); - expect( - compareEntityToRef( - entityWithoutNamespace, - { - namespace: 'default', - name: 'n', - }, - { defaultKind: 'X' }, - ), - ).toBe(false); - - expect( - compareEntityToRef(entityWithNamespace, { - kind: 'K', - namespace: 'xx', - name: 'n', - }), - ).toBe(false); - expect( - compareEntityToRef( - entityWithNamespace, - { - kind: 'K', - name: 'n', - }, - { defaultNamespace: 'xx' }, - ), - ).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, { - kind: 'K', - namespace: 'xx', - name: 'n', - }), - ).toBe(false); - expect( - compareEntityToRef( - entityWithoutNamespace, - { - kind: 'K', - name: 'n', - }, - { defaultNamespace: 'xx' }, - ), - ).toBe(false); - - expect( - compareEntityToRef(entityWithNamespace, { - kind: 'K', - namespace: 'ns', - name: 'x', - }), - ).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, { - kind: 'K', - namespace: 'default', - name: 'x', - }), - ).toBe(false); - expect( - compareEntityToRef( - entityWithoutNamespace, - { - kind: 'K', - name: 'x', - }, - { defaultNamespace: 'default' }, - ), - ).toBe(false); - }); - }); }); diff --git a/packages/catalog-model/src/entity/ref.ts b/packages/catalog-model/src/entity/ref.ts index 66974f5a01..f9bfcf38cd 100644 --- a/packages/catalog-model/src/entity/ref.ts +++ b/packages/catalog-model/src/entity/ref.ts @@ -53,19 +53,6 @@ export function getEntityName(entity: Entity): EntityName { }; } -/** - * The context of defaults that entity reference parsing happens within. - * - * @public - * @deprecated type inlined, will be removed in a future release. - */ -export type EntityRefContext = { - /** The default kind, if none is given in the reference */ - defaultKind?: string; - /** The default namespace, if none is given in the reference */ - defaultNamespace?: string; -}; - /** * Parses an entity reference, either on string or compound form, and always * returns a complete entity name including kind, namespace and name. @@ -203,67 +190,3 @@ export function stringifyEntityRef( 'en-US', )}/${name.toLocaleLowerCase('en-US')}`; } - -/** - * Compares an entity to either a string reference or a compound reference. - * - * @remarks - * - * The comparison is case insensitive, and all of kind, namespace, and name - * must match (after applying the optional context to the ref). - * - * @public - * @param entity - The entity to match - * @param ref - A string or compound entity ref - * @param context - An optional context of default kind and namespace, that apply - * to the ref if given - * @returns True if matching, false otherwise - * @deprecated compare using stringifyEntityRef instead. - */ -export function compareEntityToRef( - entity: Entity, - ref: - | string - | { kind?: string; namespace?: string; name: string } - | EntityName, - context?: { - /** The default kind, if none is given in the reference */ - defaultKind?: string; - /** The default namespace, if none is given in the reference */ - defaultNamespace?: string; - }, -): boolean { - const entityKind = entity.kind; - const entityNamespace = entity.metadata.namespace || DEFAULT_NAMESPACE; - const entityName = entity.metadata.name; - - let refKind: string | undefined; - let refNamespace: string | undefined; - let refName: string; - if (typeof ref === 'string') { - const parsed = parseRefString(ref); - refKind = parsed.kind || context?.defaultKind; - refNamespace = - parsed.namespace || context?.defaultNamespace || DEFAULT_NAMESPACE; - refName = parsed.name; - } else { - refKind = ref.kind || context?.defaultKind; - refNamespace = - ref.namespace || context?.defaultNamespace || DEFAULT_NAMESPACE; - refName = ref.name; - } - - if (!refKind || !refNamespace) { - throw new Error( - `Entity reference or context did not contain kind and namespace`, - ); - } - - return ( - entityKind.toLocaleLowerCase('en-US') === - refKind.toLocaleLowerCase('en-US') && - entityNamespace.toLocaleLowerCase('en-US') === - refNamespace.toLocaleLowerCase('en-US') && - entityName.toLocaleLowerCase('en-US') === refName.toLocaleLowerCase('en-US') - ); -} diff --git a/packages/catalog-model/src/entity/util.test.ts b/packages/catalog-model/src/entity/util.test.ts deleted file mode 100644 index 1c39961c0f..0000000000 --- a/packages/catalog-model/src/entity/util.test.ts +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import lodash from 'lodash'; -import { - generateEntityEtag, - generateEntityUid, - entityHasChanges, - generateUpdatedEntity, -} from './util'; -import { Entity } from './Entity'; - -describe('util', () => { - describe('generateEntityUid', () => { - it('generates randomness', () => { - expect(generateEntityUid()).not.toEqual(''); - expect(generateEntityUid()).not.toEqual(generateEntityUid()); - }); - }); - - describe('generateEntityEtag', () => { - it('generates randomness', () => { - expect(generateEntityEtag()).not.toEqual(''); - expect(generateEntityEtag()).not.toEqual(generateEntityEtag()); - }); - }); - - describe('entityHasChanges', () => { - let a: Entity; - beforeEach(() => { - a = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'name', - custom: 'custom', - labels: { - labelKey: 'labelValue', - }, - annotations: { - annotationKey: 'annotationValue', - }, - }, - spec: { - a: 'a', - }, - }; - }); - - it('happy path: clone has no changes', () => { - const b = lodash.cloneDeep(a); - expect(entityHasChanges(a, b)).toBe(false); - }); - - it('detects root field changes', () => { - let b: any = lodash.cloneDeep(a); - b.apiVersion += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.apiVersion; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - b.kind += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.kind; - expect(entityHasChanges(a, b)).toBe(true); - }); - - it('detects metadata changes', () => { - let b: any = lodash.cloneDeep(a); - b.metadata.name += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.metadata.custom; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.metadata.custom; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - b.metadata.labels.n = 'n'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - b.metadata.labels.labelKey += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - b.metadata.annotations.annotationKey += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.metadata.annotations.annotationKey; - expect(entityHasChanges(a, b)).toBe(true); - }); - - it('detects spec changes', () => { - let b: any = lodash.cloneDeep(a); - b.spec.a += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.spec.a; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - b.spec.n = 'n'; - expect(entityHasChanges(a, b)).toBe(true); - }); - }); - - describe('generateUpdatedEntity', () => { - let a: Entity; - let b: any; - beforeEach(() => { - a = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - uid: 'da921f56-f655-4e6e-9b8b-bb19a57818d8', - etag: 'NzY5NDA5NzQtYmEwNC00MDY0LWFiYmItNTYxYzQxM2JhZDcx', - generation: 2, - name: 'name', - custom: 'custom', - labels: { - labelKey: 'labelValue', - }, - annotations: { - annotationKey: 'annotationValue', - }, - }, - spec: { - a: 'a', - }, - }; - b = lodash.cloneDeep(a); - delete b.metadata.uid; - delete b.metadata.etag; - delete b.metadata.generation; - }); - - it('happy path: running on itself leaves it unchanged', () => { - const result = generateUpdatedEntity(a, b); - expect(result).toEqual(a); - }); - - it('bumps etag and generation when spec is changed', () => { - b.spec.a += 'a'; - const result = generateUpdatedEntity(a, b); - expect(result.metadata.uid).toEqual(a.metadata.uid); - expect(result.metadata.etag).not.toEqual(a.metadata.etag); - expect(result.metadata.generation).toEqual(a.metadata.generation! + 1); - expect(result.spec).toEqual({ a: 'aa' }); - }); - - it('bumps only etag when other things than spec are changed', () => { - b.metadata.n = 'n'; - const result = generateUpdatedEntity(a, b); - expect(result.metadata.uid).toEqual(a.metadata.uid); - expect(result.metadata.etag).not.toEqual(a.metadata.etag); - expect(result.metadata.generation).toEqual(a.metadata.generation); - expect(result.metadata.n).toEqual('n'); - }); - - it('retains new annotations', () => { - b.metadata.annotations.annotationKey = 'changedValue'; - b.metadata.annotations.newKey = 'newValue'; - const result = generateUpdatedEntity(a, b); - expect(result.metadata.uid).toEqual(a.metadata.uid); - expect(result.metadata.etag).not.toEqual(a.metadata.etag); - expect(result.metadata.generation).toEqual(a.metadata.generation); - expect(result.metadata.annotations).toEqual({ - annotationKey: 'changedValue', - newKey: 'newValue', - }); - }); - - it('retains old annotations', () => { - b.metadata.annotations.newKey = 'newValue'; - const result = generateUpdatedEntity(a, b); - expect(result.metadata.uid).toEqual(a.metadata.uid); - expect(result.metadata.etag).not.toEqual(a.metadata.etag); - expect(result.metadata.generation).toEqual(a.metadata.generation); - expect(result.metadata.annotations).toEqual({ - annotationKey: 'annotationValue', - newKey: 'newValue', - }); - }); - }); -}); diff --git a/packages/catalog-model/src/entity/util.ts b/packages/catalog-model/src/entity/util.ts deleted file mode 100644 index 78359b45f4..0000000000 --- a/packages/catalog-model/src/entity/util.ts +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import lodash from 'lodash'; -import { v4 as uuidv4 } from 'uuid'; -import { Entity, AlphaEntity } from './Entity'; - -/** - * Generates a new random UID for an entity. - * - * @public - * @returns A string with enough randomness to uniquely identify an entity - * @deprecated use `uuidv4()` instead. - */ -export function generateEntityUid(): string { - return uuidv4(); -} - -/** - * Generates a new random Etag for an entity. - * - * @public - * @returns A string with enough randomness to uniquely identify an entity - * revision - * @deprecated will be removed in a future release. - */ -export function generateEntityEtag(): string { - return Buffer.from(uuidv4(), 'utf8').toString('base64').replace(/[^\w]/g, ''); -} - -/** - * Checks whether there are any significant changes going from the previous to - * the next version of this entity. - * - * @remarks - * - * Significance, in this case, means that we do not compare generated fields - * such as uid, etag and generation. - * - * Note that this comparison does NOT take status, relations or similar into - * account. It only compares the actual input entity data, i.e. metadata and - * spec. - * - * @public - * @param previous - The old state of the entity - * @param next - The new state of the entity - * @deprecated will be removed in a future release. - */ -export function entityHasChanges(previous: Entity, next: Entity): boolean { - const e1 = lodash.cloneDeep(previous); - const e2 = lodash.cloneDeep(next); - - if (!e1.metadata.labels) { - e1.metadata.labels = {}; - } - if (!e2.metadata.labels) { - e2.metadata.labels = {}; - } - if (!e1.metadata.annotations) { - e1.metadata.annotations = {}; - } - if (!e2.metadata.annotations) { - e2.metadata.annotations = {}; - } - if (!e1.metadata.tags) { - e1.metadata.tags = []; - } - if (!e2.metadata.tags) { - e2.metadata.tags = []; - } - - // Remove generated fields - delete e1.metadata.uid; - delete e1.metadata.etag; - delete e1.metadata.generation; - delete e2.metadata.uid; - delete e2.metadata.etag; - delete e2.metadata.generation; - - // Remove things that we explicitly do not compare - delete e1.relations; - delete (e1 as AlphaEntity).status; - delete e2.relations; - delete (e2 as AlphaEntity).status; - - return !lodash.isEqual(e1, e2); -} - -/** - * Takes an old revision of an entity and a new desired state, and merges - * them into a complete new state. - * - * @remarks - * - * The previous revision is expected to be a complete model loaded from the - * catalog, including the uid, etag and generation fields. - * - * @public - * @param previous - The old state of the entity - * @param next - The new state of the entity - * @returns An entity with the merged state of both - * @deprecated will be removed in a future release. - */ -export function generateUpdatedEntity(previous: Entity, next: Entity): Entity { - const { uid, etag, generation } = previous.metadata; - if (!uid || !etag || !generation) { - throw new Error('Previous entity must have uid, etag and generation'); - } - - const result = lodash.cloneDeep(next); - - // Generated fields are copied and updated - const bumpEtag = entityHasChanges(previous, result); - const bumpGeneration = !lodash.isEqual(previous.spec, result.spec); - result.metadata.uid = uid; - result.metadata.etag = bumpEtag ? generateEntityEtag() : etag; - result.metadata.generation = bumpGeneration ? generation + 1 : generation; - - return result; -} diff --git a/packages/catalog-model/src/index.ts b/packages/catalog-model/src/index.ts index ef55337ae2..986341bd33 100644 --- a/packages/catalog-model/src/index.ts +++ b/packages/catalog-model/src/index.ts @@ -24,5 +24,5 @@ export * from './entity'; export { EntityPolicies } from './EntityPolicies'; export * from './kinds'; export * from './location'; -export type { EntityName, EntityRef, JSONSchema } from './types'; +export type { EntityName, EntityRef } from './types'; export * from './validation'; diff --git a/packages/catalog-model/src/location/annotation.ts b/packages/catalog-model/src/location/annotation.ts index 3ff624360e..61dbc0b305 100644 --- a/packages/catalog-model/src/location/annotation.ts +++ b/packages/catalog-model/src/location/annotation.ts @@ -14,30 +14,6 @@ * limitations under the License. */ -/** - * Constant storing location annotation. - * - * @public - * @deprecated use {@link ANNOTATION_LOCATION} instead. - * */ -export const LOCATION_ANNOTATION = 'backstage.io/managed-by-location'; -/** - * Constant storing origin location annotation - * - * @public - * @deprecated use {@link ANNOTATION_ORIGIN_LOCATION} instead. - */ -export const ORIGIN_LOCATION_ANNOTATION = - 'backstage.io/managed-by-origin-location'; - -/** - * Contant storing source location annotation - * - * @public - * @deprecated use {@link ANNOTATION_SOURCE_LOCATION} instead. - * */ -export const SOURCE_LOCATION_ANNOTATION = 'backstage.io/source-location'; - /** * Constant storing location annotation. * diff --git a/packages/catalog-model/src/location/helpers.ts b/packages/catalog-model/src/location/helpers.ts index 7c73908157..8f8acc3b1d 100644 --- a/packages/catalog-model/src/location/helpers.ts +++ b/packages/catalog-model/src/location/helpers.ts @@ -18,18 +18,6 @@ import { ANNOTATION_SOURCE_LOCATION } from '.'; import { Entity, stringifyEntityRef } from '../entity'; import { ANNOTATION_LOCATION } from './annotation'; -/** - * Parses a string form location reference. - * - * @public - * @param ref - A string-form location reference, e.g. `'url:https://host'` - * @returns A location reference, e.g. `{ type: 'url', target: 'https://host' }` - * @deprecated use {@link parseLocationRef} instead - */ -export function parseLocationReference(ref: string) { - return parseLocationRef(ref); -} - /** * Parses a string form location reference. * @@ -72,21 +60,6 @@ export function parseLocationRef(ref: string): { return { type, target }; } -/** - * Turns a location reference into its string form. - * - * @public - * @param ref - A location reference, e.g. `{ type: 'url', target: 'https://host' }` - * @returns A string-form location reference, e.g. `'url:https://host'` - * @deprecated use {@link stringifyLocationRef} instead - */ -export function stringifyLocationReference(ref: { - type: string; - target: string; -}): string { - return stringifyLocationRef(ref); -} - /** * Turns a location ref into its string form. * diff --git a/packages/catalog-model/src/location/index.ts b/packages/catalog-model/src/location/index.ts index 8302ec202f..dc709144b2 100644 --- a/packages/catalog-model/src/location/index.ts +++ b/packages/catalog-model/src/location/index.ts @@ -18,15 +18,10 @@ export { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, ANNOTATION_SOURCE_LOCATION, - LOCATION_ANNOTATION, - ORIGIN_LOCATION_ANNOTATION, - SOURCE_LOCATION_ANNOTATION, } from './annotation'; export { getEntitySourceLocation, parseLocationRef, - parseLocationReference, stringifyLocationRef, - stringifyLocationReference, } from './helpers'; -export type { Location, LocationSpec } from './types'; +export type { LocationSpec } from './types'; diff --git a/packages/catalog-model/src/location/types.ts b/packages/catalog-model/src/location/types.ts index 9b87e4919d..9e2c9339e6 100644 --- a/packages/catalog-model/src/location/types.ts +++ b/packages/catalog-model/src/location/types.ts @@ -31,13 +31,3 @@ export type LocationSpec = { target: string; presence?: 'optional' | 'required'; }; - -/** - * Entity location for a specific entity. - * - * @public - * @deprecated import from {@link @backstage/catalog-client#Location} instead. - */ -export type Location = { - id: string; -} & LocationSpec; diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts index 13550968be..5b5f063ca3 100644 --- a/packages/catalog-model/src/types.ts +++ b/packages/catalog-model/src/types.ts @@ -14,17 +14,6 @@ * limitations under the License. */ -import { JsonValue } from '@backstage/types'; -import { JSONSchema7 } from 'json-schema'; - -/** - * JSONSchema extendable by arbitrary JSON attributes - * - * @public - * @deprecated use JSONSchema7 from the json-schema package instead. - */ -export type JSONSchema = JSONSchema7 & { [key in string]?: JsonValue }; - /** * A complete entity name, with the full kind-namespace-name triplet. * From c81f9d6b9e247e5435c37512d20e2b529a57b473 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 2 Mar 2022 09:22:28 +0100 Subject: [PATCH 080/150] update changeset Signed-off-by: Johan Haals Co-authored-by: Patrik Oldsberg --- .changeset/weak-news-reply.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/weak-news-reply.md b/.changeset/weak-news-reply.md index 3becabc2e6..4a7a38cef5 100644 --- a/.changeset/weak-news-reply.md +++ b/.changeset/weak-news-reply.md @@ -2,7 +2,7 @@ '@backstage/catalog-model': minor --- -**Breaking**: The following changes are all breaking changes. +**BREAKING**: The following changes are all breaking changes. Removed `EDIT_URL_ANNOTATION` and `VIEW_URL_ANNOTATION`, `LOCATION_ANNOTATION`, `ORIGIN_LOCATION_ANNOTATION`, `LOCATION_ANNOTATION`, `SOURCE_LOCATION_ANNOTATION`. All of these constants have been prefixed with ANNOTATION to be easier to find meaning `SOURCE_LOCATION_ANNOTATION` is available as `ANNOTATION_SOURCE_LOCATION`. From 078319592a7152b60167f6e2d4495ec85077b301 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 2 Mar 2022 10:06:12 +0100 Subject: [PATCH 081/150] remove roadie resolutions for plugin-catalog & plugin-model Signed-off-by: Johan Haals --- package.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/package.json b/package.json index e7cb42c1c6..1882d400b4 100644 --- a/package.json +++ b/package.json @@ -45,9 +45,7 @@ ] }, "resolutions": { - "**/@graphql-codegen/cli/**/ws": "^7.4.6", - "**/@roadiehq/**/@backstage/plugin-catalog": "*", - "**/@roadiehq/**/@backstage/catalog-model": "*" + "**/@graphql-codegen/cli/**/ws": "^7.4.6" }, "version": "0.69.0", "dependencies": { From 06cd854a0583959c746527d5bdb72fbf27aede63 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 2 Mar 2022 10:20:31 +0100 Subject: [PATCH 082/150] update yarn.lock Signed-off-by: Johan Haals --- yarn.lock | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index 9eef93080e..90dd0207c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1359,7 +1359,9 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.10.0": - version "0.11.0" + version "0.10.1" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.10.1.tgz#dcc3415eb4d4ee3d437355c477e85c7479626b3b" + integrity sha512-c004aQeO9cxtSZZc2iBcE6eoqurQLdj7YUm8mHWs8hEaPTA2UPVHawt+wlt89VywkI89X0wF7BuXV2LKVUfXvw== dependencies: "@backstage/config" "^0.1.15" "@backstage/errors" "^0.2.2" @@ -1371,16 +1373,20 @@ uuid "^8.0.0" "@backstage/catalog-model@^0.9.7": - version "0.11.0" + version "0.9.10" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.9.10.tgz#bd5662e1ad7bd7c9604f3f45d055c99b5b2bb87f" + integrity sha512-KhCjbZKhS5zZhHiGHmBMq6hDGDshMSZOPGXehtdhr6/oW7Ee5fDcOnhMqreCi1Ebm4RIWJhZcRrxO6X1TTi4TQ== dependencies: - "@backstage/config" "^0.1.15" - "@backstage/errors" "^0.2.2" - "@backstage/types" "^0.1.3" + "@backstage/config" "^0.1.13" + "@backstage/errors" "^0.2.0" + "@backstage/types" "^0.1.1" "@types/json-schema" "^7.0.5" + "@types/yup" "^0.29.13" ajv "^7.0.3" json-schema "^0.4.0" lodash "^4.17.21" uuid "^8.0.0" + yup "^0.32.9" "@backstage/core-plugin-api@^0.6.0", "@backstage/core-plugin-api@^0.6.1": version "0.6.1" From 2de1d82bd1b8f4f23e0117a7530cba9da10c18eb Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 2 Mar 2022 10:23:30 +0100 Subject: [PATCH 083/150] chore: added changeset Signed-off-by: blam --- .changeset/strong-suns-hope.md | 5 +++++ plugins/catalog-react/api-report.md | 2 +- .../catalog-react/src/hooks/useEntityOwnership.ts | 13 +++++-------- 3 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 .changeset/strong-suns-hope.md diff --git a/.changeset/strong-suns-hope.md b/.changeset/strong-suns-hope.md new file mode 100644 index 0000000000..ca59e43deb --- /dev/null +++ b/.changeset/strong-suns-hope.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +Removing the `EntityName` path for the `useEntityOwnership` as it has never worked correctly. Please pass in an entire `Entity` instead. diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 2d2b56b634..c5d102cc29 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -544,7 +544,7 @@ export function useEntityListProvider< // @public export function useEntityOwnership(): { loading: boolean; - isOwnedEntity: (entity: Entity | EntityName) => boolean; + isOwnedEntity: (entity: Entity) => boolean; }; // @alpha diff --git a/plugins/catalog-react/src/hooks/useEntityOwnership.ts b/plugins/catalog-react/src/hooks/useEntityOwnership.ts index 77f6c4bf24..de3eca9dab 100644 --- a/plugins/catalog-react/src/hooks/useEntityOwnership.ts +++ b/plugins/catalog-react/src/hooks/useEntityOwnership.ts @@ -17,7 +17,6 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Entity, - EntityName, parseEntityRef, RELATION_MEMBER_OF, RELATION_OWNED_BY, @@ -77,7 +76,7 @@ export async function loadCatalogOwnerRefs( */ export function useEntityOwnership(): { loading: boolean; - isOwnedEntity: (entity: Entity | EntityName) => boolean; + isOwnedEntity: (entity: Entity) => boolean; } { const identityApi = useApi(identityApiRef); const catalogApi = useApi(catalogApiRef); @@ -94,12 +93,10 @@ export function useEntityOwnership(): { const isOwnedEntity = useMemo(() => { const myOwnerRefs = new Set(refs ?? []); - return (entity: Entity | EntityName) => { - const entityOwnerRefs = ( - 'metadata' in entity - ? getEntityRelations(entity, RELATION_OWNED_BY) - : [entity] - ).map(stringifyEntityRef); + return (entity: Entity) => { + const entityOwnerRefs = getEntityRelations(entity, RELATION_OWNED_BY).map( + stringifyEntityRef, + ); for (const ref of entityOwnerRefs) { if (myOwnerRefs.has(ref)) { return true; From fd061e47ee35fe54000d7e6b31d5fa47f76ec034 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 2 Mar 2022 10:38:10 +0100 Subject: [PATCH 084/150] chore: code review comments Signed-off-by: blam --- .changeset/tall-pillows-smash.md | 6 ------ plugins/catalog-react/src/hooks/useEntity.tsx | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 .changeset/tall-pillows-smash.md diff --git a/.changeset/tall-pillows-smash.md b/.changeset/tall-pillows-smash.md deleted file mode 100644 index 3c1d18cbe3..0000000000 --- a/.changeset/tall-pillows-smash.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@backstage/plugin-allure': patch -'@backstage/plugin-todo': patch ---- - -Fixing broken tests for the packages with the new `useEntity` change diff --git a/plugins/catalog-react/src/hooks/useEntity.tsx b/plugins/catalog-react/src/hooks/useEntity.tsx index f70adef708..da3fb8f3a5 100644 --- a/plugins/catalog-react/src/hooks/useEntity.tsx +++ b/plugins/catalog-react/src/hooks/useEntity.tsx @@ -178,7 +178,7 @@ export function useEntity(): { */ export function useAsyncEntity< TEntity extends Entity = Entity, ->(): EntityLoadingStatus { +>(): EntityLoadingStatus { const versionedHolder = useVersionedContext<{ 1: EntityLoadingStatus }>('entity-context'); From baca071d05bf087a2527b6651bcdc24a5f91fb40 Mon Sep 17 00:00:00 2001 From: Ben Lambert Date: Wed, 2 Mar 2022 11:26:50 +0100 Subject: [PATCH 085/150] Updating changeset Minor change but no deprecation period as it doesn't work Signed-off-by: Ben Lambert --- .changeset/strong-suns-hope.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/strong-suns-hope.md b/.changeset/strong-suns-hope.md index ca59e43deb..876f5f1618 100644 --- a/.changeset/strong-suns-hope.md +++ b/.changeset/strong-suns-hope.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-react': patch +'@backstage/plugin-catalog-react': minor --- Removing the `EntityName` path for the `useEntityOwnership` as it has never worked correctly. Please pass in an entire `Entity` instead. From 6ed51204ff27d95cdbbac9862ad5a959aa2a1207 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 2 Mar 2022 11:32:45 +0100 Subject: [PATCH 086/150] chore: woops - api-report needed update Signed-off-by: blam --- plugins/catalog-react/api-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 1a94af6052..b660ceadb5 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -526,7 +526,7 @@ export type UnregisterEntityDialogProps = { // @public export function useAsyncEntity< TEntity extends Entity = Entity, ->(): EntityLoadingStatus; +>(): EntityLoadingStatus; // @public export function useEntity(): { From b838717e9235e6d02c47f7b98e660de0c4cf51f8 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 2 Mar 2022 12:00:44 +0000 Subject: [PATCH 087/150] exort FetchUrlReader Signed-off-by: Brian Fletcher --- .changeset/olive-glasses-approve.md | 5 +++++ packages/backend-common/src/reading/index.ts | 1 + 2 files changed, 6 insertions(+) create mode 100644 .changeset/olive-glasses-approve.md diff --git a/.changeset/olive-glasses-approve.md b/.changeset/olive-glasses-approve.md new file mode 100644 index 0000000000..e74b7ef485 --- /dev/null +++ b/.changeset/olive-glasses-approve.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Export FetchUrlReader to facilitate more flexible configuration of the backend. diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 2c3394e4fc..9c29bc93fe 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -19,6 +19,7 @@ export { BitbucketUrlReader } from './BitbucketUrlReader'; export { GithubUrlReader } from './GithubUrlReader'; export { GitlabUrlReader } from './GitlabUrlReader'; export { AwsS3UrlReader } from './AwsS3UrlReader'; +export { FetchUrlReader } from './FetchUrlReader'; export type { FromReadableArrayOptions, ReaderFactory, From fb01d26fd71802671926d354227f86b305b2fe5a Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 2 Mar 2022 12:27:49 +0000 Subject: [PATCH 088/150] adds api reports Signed-off-by: Brian Fletcher --- packages/backend-common/api-report.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 9b963f3fc9..4ed9bb1514 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -234,6 +234,21 @@ export type ErrorHandlerOptions = { logClientErrors?: boolean; }; +// @public +export class FetchUrlReader implements UrlReader { + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(): Promise; + // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) + search(): Promise; + // (undocumented) + toString(): string; +} + // @public export type FromReadableArrayOptions = Array<{ data: Readable; From efa549c4abcc8703184718acfa1c8019504de473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 2 Mar 2022 13:47:03 +0100 Subject: [PATCH 089/150] form refs properly in resolver docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- docs/auth/identity-resolver.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/auth/identity-resolver.md b/docs/auth/identity-resolver.md index a33351e938..3b355470d1 100644 --- a/docs/auth/identity-resolver.md +++ b/docs/auth/identity-resolver.md @@ -27,6 +27,8 @@ sign-in resolvers and set them for any of the Authentication providers inside `@backstage/plugin-auth-backend` plugin. ```ts +import { DEFAULT_NAMESPACE, stringifyEntityRef } from '@backstage/catalog-model'; + export default async function createPlugin({ ... }: PluginEnvironment): Promise { @@ -38,22 +40,31 @@ export default async function createPlugin({ resolver: async ({ profile: { email } }, ctx) => { // Call a custom validator function that checks that the email is // valid and on our own company's domain, and throws an Error if it - // isn't + // isn't. + // TODO: Implement this function validateEmail(email); // List of entity references that denote the identity and // membership of the user - const ent = []; + const ent: string[] = []; // Let's use the username in the email ID as the user's default // unique identifier inside Backstage. const [id] = email.split('@'); - ent.push(`User:default/${id}`) + ent.push(stringifyEntityRef({ + kind: 'User', + namespace: DEFAULT_NAMESPACE, + name: id, + })); // Let's call the internal LDAP provider to get a list of groups // that the user belongs to, and add those to the list as well const ldapGroups = await getLdapGroups(email); - ldapGroups.forEach(group => ent.push(`Group:default/${group}`)) + ldapGroups.forEach(group => ent.push(stringifyEntityRef({ + kind: 'Group', + namespace: DEFAULT_NAMESPACE, + name: group, + }))); // Issue the token containing the entity claims const token = await ctx.tokenIssuer.issueToken({ From f3cce3dcf79b2597a5b77086b46ad3529324889b Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 2 Mar 2022 13:50:18 +0100 Subject: [PATCH 090/150] core-app-api: Remove GithubSession and SamlSession types Signed-off-by: Johan Haals --- .changeset/olive-lobsters-rescue.md | 5 ++ packages/core-app-api/api-report.md | 18 ------ .../apis/implementations/auth/github/index.ts | 1 - .../apis/implementations/auth/github/types.ts | 62 ------------------- .../apis/implementations/auth/saml/index.ts | 2 +- .../apis/implementations/auth/saml/types.ts | 12 ---- 6 files changed, 6 insertions(+), 94 deletions(-) create mode 100644 .changeset/olive-lobsters-rescue.md delete mode 100644 packages/core-app-api/src/apis/implementations/auth/github/types.ts diff --git a/.changeset/olive-lobsters-rescue.md b/.changeset/olive-lobsters-rescue.md new file mode 100644 index 0000000000..37a7712782 --- /dev/null +++ b/.changeset/olive-lobsters-rescue.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-app-api': minor +--- + +**BREAKING**: Removed export of `GithubSession` and `SamlSession` which are only used internally. diff --git a/packages/core-app-api/api-report.md b/packages/core-app-api/api-report.md index 17a9592523..c52b953465 100644 --- a/packages/core-app-api/api-report.md +++ b/packages/core-app-api/api-report.md @@ -380,17 +380,6 @@ export class GithubAuth { static normalizeScope(scope?: string): Set; } -// @public @deprecated -export type GithubSession = { - providerInfo: { - accessToken: string; - scopes: Set; - expiresAt?: Date; - }; - profile: ProfileInfo; - backstageIdentity: BackstageIdentityResponse; -}; - // @public export class GitlabAuth { // (undocumented) @@ -532,13 +521,6 @@ export class SamlAuth signOut(): Promise; } -// @public @deprecated -export type SamlSession = { - userId: string; - profile: ProfileInfo; - backstageIdentity: BackstageIdentityResponse; -}; - // @public export type SignInPageProps = { onSignInSuccess(identityApi: IdentityApi): void; diff --git a/packages/core-app-api/src/apis/implementations/auth/github/index.ts b/packages/core-app-api/src/apis/implementations/auth/github/index.ts index b5aa1a0a25..5e53bacfc1 100644 --- a/packages/core-app-api/src/apis/implementations/auth/github/index.ts +++ b/packages/core-app-api/src/apis/implementations/auth/github/index.ts @@ -14,5 +14,4 @@ * limitations under the License. */ -export type { GithubSession } from './types'; export { default as GithubAuth } from './GithubAuth'; diff --git a/packages/core-app-api/src/apis/implementations/auth/github/types.ts b/packages/core-app-api/src/apis/implementations/auth/github/types.ts deleted file mode 100644 index 0ef662905f..0000000000 --- a/packages/core-app-api/src/apis/implementations/auth/github/types.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - ProfileInfo, - BackstageIdentityResponse, -} from '@backstage/core-plugin-api'; -import { z } from 'zod'; - -// TODO(Rugvip): Make GithubSession internal - -/** - * Session information for GitHub auth. - * - * @public - * @deprecated This type is internal and will be removed - */ -export type GithubSession = { - providerInfo: { - accessToken: string; - scopes: Set; - expiresAt?: Date; - }; - profile: ProfileInfo; - // TODO(Rugvip): This should be made optional once the type is no longer public - backstageIdentity: BackstageIdentityResponse; -}; - -export const githubSessionSchema: z.ZodSchema = z.object({ - providerInfo: z.object({ - accessToken: z.string(), - scopes: z.set(z.string()), - expiresAt: z.date().optional(), - }), - profile: z.object({ - email: z.string().optional(), - displayName: z.string().optional(), - picture: z.string().optional(), - }), - backstageIdentity: z.object({ - id: z.string(), - token: z.string(), - identity: z.object({ - type: z.literal('user'), - userEntityRef: z.string(), - ownershipEntityRefs: z.array(z.string()), - }), - }), -}); diff --git a/packages/core-app-api/src/apis/implementations/auth/saml/index.ts b/packages/core-app-api/src/apis/implementations/auth/saml/index.ts index 2e749a0648..1f5f8fbbf8 100644 --- a/packages/core-app-api/src/apis/implementations/auth/saml/index.ts +++ b/packages/core-app-api/src/apis/implementations/auth/saml/index.ts @@ -14,4 +14,4 @@ * limitations under the License. */ export { default as SamlAuth } from './SamlAuth'; -export type { ExportedSamlSession as SamlSession } from './types'; +// export type { ExportedSamlSession as SamlSession } from './types'; diff --git a/packages/core-app-api/src/apis/implementations/auth/saml/types.ts b/packages/core-app-api/src/apis/implementations/auth/saml/types.ts index f1345fd154..55b278b429 100644 --- a/packages/core-app-api/src/apis/implementations/auth/saml/types.ts +++ b/packages/core-app-api/src/apis/implementations/auth/saml/types.ts @@ -20,18 +20,6 @@ import { } from '@backstage/core-plugin-api'; import { z } from 'zod'; -/** - * Session information for SAML auth. - * - * @public - * @deprecated This type is internal and will be removed - */ -export type ExportedSamlSession = { - userId: string; - profile: ProfileInfo; - backstageIdentity: BackstageIdentityResponse; -}; - /** @internal */ export type SamlSession = { profile: ProfileInfo; From 70fdad285ed589ea2a2d881608d0518e92315b41 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 2 Mar 2022 13:50:56 +0100 Subject: [PATCH 091/150] chore: remove comment Signed-off-by: Johan Haals --- .../core-app-api/src/apis/implementations/auth/saml/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core-app-api/src/apis/implementations/auth/saml/index.ts b/packages/core-app-api/src/apis/implementations/auth/saml/index.ts index 1f5f8fbbf8..930e6cb115 100644 --- a/packages/core-app-api/src/apis/implementations/auth/saml/index.ts +++ b/packages/core-app-api/src/apis/implementations/auth/saml/index.ts @@ -14,4 +14,3 @@ * limitations under the License. */ export { default as SamlAuth } from './SamlAuth'; -// export type { ExportedSamlSession as SamlSession } from './types'; From 3c1d3cb07e360caa1d697f3d6db5d3d33589d631 Mon Sep 17 00:00:00 2001 From: Karan Shah Date: Wed, 2 Mar 2022 13:41:02 +0000 Subject: [PATCH 092/150] Add a changeset Signed-off-by: Karan Shah --- .changeset/red-chefs-beam.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/red-chefs-beam.md diff --git a/.changeset/red-chefs-beam.md b/.changeset/red-chefs-beam.md new file mode 100644 index 0000000000..5cadccb0aa --- /dev/null +++ b/.changeset/red-chefs-beam.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-airbrake': patch +'@backstage/plugin-airbrake-backend': patch +--- + +The Airbrake plugin installation instructions have been updated to work better and conform to how the frontend and backend plugins are supposed to be integrated into a Backstage instance. From dbf84eee55e10cab80f658dca5f68f2ebf7bb54e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 2 Mar 2022 12:26:49 +0100 Subject: [PATCH 093/150] core-app-api: removed deprecated GithubAuth.normalizeScopes Signed-off-by: Patrik Oldsberg --- .changeset/tame-lions-know.md | 5 +++++ packages/core-app-api/api-report.md | 2 -- .../implementations/auth/github/GithubAuth.ts | 15 --------------- 3 files changed, 5 insertions(+), 17 deletions(-) create mode 100644 .changeset/tame-lions-know.md diff --git a/.changeset/tame-lions-know.md b/.changeset/tame-lions-know.md new file mode 100644 index 0000000000..7f304b22ef --- /dev/null +++ b/.changeset/tame-lions-know.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-app-api': minor +--- + +**BREAKING**: Removed the deprecated `GithubAuth.normalizeScopes` method. diff --git a/packages/core-app-api/api-report.md b/packages/core-app-api/api-report.md index c52b953465..116b33c598 100644 --- a/packages/core-app-api/api-report.md +++ b/packages/core-app-api/api-report.md @@ -376,8 +376,6 @@ export type FlatRoutesProps = { export class GithubAuth { // (undocumented) static create(options: OAuthApiCreateOptions): typeof githubAuthApiRef.T; - // @deprecated (undocumented) - static normalizeScope(scope?: string): Set; } // @public diff --git a/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts b/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts index b0af4c7ade..7efe4e95c6 100644 --- a/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts +++ b/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts @@ -47,19 +47,4 @@ export default class GithubAuth { defaultScopes, }); } - - /** - * @deprecated This method is deprecated and will be removed in a future release. - */ - static normalizeScope(scope?: string): Set { - if (!scope) { - return new Set(); - } - - const scopeList = Array.isArray(scope) - ? scope - : scope.split(/[\s|,]/).filter(Boolean); - - return new Set(scopeList); - } } From 5fba4c0304b1a4f27aa296820d2d38343a8af38e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 2 Mar 2022 12:27:15 +0100 Subject: [PATCH 094/150] config-loader: tweak a comment so that it does not show up in deprecation list Signed-off-by: Patrik Oldsberg --- packages/config-loader/src/lib/schema/collect.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index a05796139b..3ff161cec4 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -182,7 +182,7 @@ function compileTsSchemas(paths: string[]) { program, // All schemas should export a `Config` symbol 'Config', - // This enables usage of @visibility and @deprecated in doc comments + // This enables the use of these tags in TSDoc comments { required: true, validationKeywords: ['visibility', 'deprecated'], From 8c3f30cb28196312f4e2bd79f690e7243e392c13 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 2 Mar 2022 12:29:24 +0100 Subject: [PATCH 095/150] cli: remove deprecated index.html templating variables Signed-off-by: Patrik Oldsberg --- .changeset/orange-crews-explain.md | 5 ++++ packages/cli/src/lib/bundler/config.ts | 33 -------------------------- 2 files changed, 5 insertions(+), 33 deletions(-) create mode 100644 .changeset/orange-crews-explain.md diff --git a/.changeset/orange-crews-explain.md b/.changeset/orange-crews-explain.md new file mode 100644 index 0000000000..4b831392eb --- /dev/null +++ b/.changeset/orange-crews-explain.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': minor +--- + +**BREAKING**: Removed the deprecated `app.` template variables from the `index.html` templating. These should be replaced by using `config.getString("app.")` instead. diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts index fe64f23209..bfd05807ad 100644 --- a/packages/cli/src/lib/bundler/config.ts +++ b/packages/cli/src/lib/bundler/config.ts @@ -15,7 +15,6 @@ */ import fs from 'fs-extra'; -import chalk from 'chalk'; import { resolve as resolvePath } from 'path'; import ForkTsCheckerWebpackPlugin from 'fork-ts-checker-webpack-plugin'; import HtmlWebpackPlugin from 'html-webpack-plugin'; @@ -118,43 +117,11 @@ export async function createConfig( }), ); - const appParamDeprecationMsg = chalk.red( - 'DEPRECATION WARNING: using `app.` in the index.html template is deprecated, use `config.getString("app.")` instead.', - ); plugins.push( new HtmlWebpackPlugin({ template: paths.targetHtml, templateParameters: { publicPath: validBaseUrl.pathname.replace(/\/$/, ''), - app: { - get title() { - console.warn(appParamDeprecationMsg); - return frontendConfig.getString('app.title'); - }, - get baseUrl() { - console.warn(appParamDeprecationMsg); - return validBaseUrl.href; - }, - get googleAnalyticsTrackingId() { - console.warn(appParamDeprecationMsg); - return frontendConfig.getOptionalString( - 'app.googleAnalyticsTrackingId', - ); - }, - get datadogRum() { - console.warn(appParamDeprecationMsg); - return { - env: frontendConfig.getOptionalString('app.datadogRum.env'), - clientToken: frontendConfig.getOptionalString( - 'app.datadogRum.clientToken', - ), - applicationId: frontendConfig.getOptionalString( - 'app.datadogRum.applicationId', - ), - site: frontendConfig.getOptionalString('app.datadogRum.site'), - }; - }, - }, config: frontendConfig, }, }), From f590d1681b26ba77ca8f50c267107e0f4e1490df Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 2 Mar 2022 15:48:17 +0100 Subject: [PATCH 096/150] core-plugin-api: Deprecate `favoriteEntityTooltip` and `favoriteEntityIcon` Signed-off-by: Johan Haals --- .changeset/metal-months-tie.md | 5 +++++ .changeset/small-brooms-retire.md | 6 ++++++ plugins/catalog-react/api-report.md | 4 ++-- .../FavoriteEntity/FavoriteEntity.tsx | 16 ++++++++++++---- .../components/CatalogTable/CatalogTable.tsx | 15 +++++++++++---- .../src/home/components/Tables/actions.tsx | 17 +++++++++++------ 6 files changed, 47 insertions(+), 16 deletions(-) create mode 100644 .changeset/metal-months-tie.md create mode 100644 .changeset/small-brooms-retire.md diff --git a/.changeset/metal-months-tie.md b/.changeset/metal-months-tie.md new file mode 100644 index 0000000000..890c7e8b5d --- /dev/null +++ b/.changeset/metal-months-tie.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +Deprecated `favoriteEntityTooltip` and `favoriteEntityIcon` since the utility value is very low. diff --git a/.changeset/small-brooms-retire.md b/.changeset/small-brooms-retire.md new file mode 100644 index 0000000000..2aa722a666 --- /dev/null +++ b/.changeset/small-brooms-retire.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog': patch +'@backstage/plugin-techdocs': patch +--- + +Removed usage of deprecated favorite utility methods. diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 78ad5d4b97..3f9a2c47e1 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -400,7 +400,7 @@ export type EntityTypeReturn = { // @public export const FavoriteEntity: (props: FavoriteEntityProps) => JSX.Element; -// @public (undocumented) +// @public @deprecated (undocumented) export const favoriteEntityIcon: (isStarred: boolean) => JSX.Element; // @public (undocumented) @@ -408,7 +408,7 @@ export type FavoriteEntityProps = ComponentProps & { entity: Entity; }; -// @public (undocumented) +// @public @deprecated (undocumented) export const favoriteEntityTooltip: ( isStarred: boolean, ) => 'Remove from favorites' | 'Add to favorites'; diff --git a/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx b/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx index 3832a82877..eaa2821448 100644 --- a/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx +++ b/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx @@ -32,11 +32,17 @@ const YellowStar = withStyles({ }, })(Star); -/** @public */ +/** + * @public + * @deprecated due to low utility value. + */ export const favoriteEntityTooltip = (isStarred: boolean) => isStarred ? 'Remove from favorites' : 'Add to favorites'; -/** @public */ +/** + * @public + * @deprecated due to low utility value. + */ export const favoriteEntityIcon = (isStarred: boolean) => isStarred ? : ; @@ -55,8 +61,10 @@ export const FavoriteEntity = (props: FavoriteEntityProps) => { {...props} onClick={() => toggleStarredEntity()} > - - {favoriteEntityIcon(isStarredEntity)} + + {isStarredEntity ? : } ); diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx index a9a6f26228..06a2395ab6 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx @@ -20,8 +20,6 @@ import { RELATION_PART_OF, } from '@backstage/catalog-model'; import { - favoriteEntityIcon, - favoriteEntityTooltip, formatEntityRefTitle, getEntityRelations, useEntityList, @@ -40,6 +38,9 @@ import { TableProps, WarningPanel, } from '@backstage/core-components'; +import StarBorder from '@material-ui/icons/StarBorder'; +import { withStyles } from '@material-ui/core/styles'; +import Star from '@material-ui/icons/Star'; /** * Props for {@link CatalogTable}. @@ -51,6 +52,12 @@ export interface CatalogTableProps { actions?: TableProps['actions']; } +const YellowStar = withStyles({ + root: { + color: '#f3ba37', + }, +})(Star); + /** @public */ export const CatalogTable = (props: CatalogTableProps) => { const { columns, actions } = props; @@ -116,8 +123,8 @@ export const CatalogTable = (props: CatalogTableProps) => { const isStarred = isStarredEntity(entity); return { cellStyle: { paddingLeft: '1em' }, - icon: () => favoriteEntityIcon(isStarred), - tooltip: favoriteEntityTooltip(isStarred), + icon: () => (isStarred ? : ), + tooltip: isStarred ? 'Remove from favorites' : 'Add to favorites', onClick: () => toggleStarredEntity(entity), }; }, diff --git a/plugins/techdocs/src/home/components/Tables/actions.tsx b/plugins/techdocs/src/home/components/Tables/actions.tsx index d0dbb8bf47..3fd8881718 100644 --- a/plugins/techdocs/src/home/components/Tables/actions.tsx +++ b/plugins/techdocs/src/home/components/Tables/actions.tsx @@ -16,11 +16,16 @@ import React from 'react'; import ShareIcon from '@material-ui/icons/Share'; -import { - favoriteEntityIcon, - favoriteEntityTooltip, -} from '@backstage/plugin-catalog-react'; import { DocsTableRow } from './types'; +import { withStyles } from '@material-ui/styles'; +import Star from '@material-ui/icons/Star'; +import StarBorder from '@material-ui/icons/StarBorder'; + +const YellowStar = withStyles({ + root: { + color: '#f3ba37', + }, +})(Star); /** * Not directly exported, but through DocsTable.actions and EntityListDocsTable.actions @@ -46,8 +51,8 @@ export const actionFactories = { const isStarred = isStarredEntity(entity); return { cellStyle: { paddingLeft: '1em' }, - icon: () => favoriteEntityIcon(isStarred), - tooltip: favoriteEntityTooltip(isStarred), + icon: () => (isStarred ? : ), + tooltip: isStarred ? 'Remove from favorites' : 'Add to favorites', onClick: () => toggleStarredEntity(entity), }; }; From 9562b13913f36ce7e845e94bd69aa910bf39a549 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 2 Mar 2022 17:01:36 +0100 Subject: [PATCH 097/150] chore: removing FavouriteTemplate in favor of FavoriteEntity Signed-off-by: blam --- .../FavouriteTemplate/FavouriteTemplate.tsx | 74 ------------------- .../src/components/FavouriteTemplate/index.ts | 17 ----- .../components/TemplateCard/TemplateCard.tsx | 10 ++- plugins/scaffolder/src/components/index.ts | 1 - 4 files changed, 8 insertions(+), 94 deletions(-) delete mode 100644 plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx delete mode 100644 plugins/scaffolder/src/components/FavouriteTemplate/index.ts diff --git a/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx b/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx deleted file mode 100644 index 6e8fe45b2f..0000000000 --- a/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Entity } from '@backstage/catalog-model'; -import { useStarredEntity } from '@backstage/plugin-catalog-react'; -import { IconButton, makeStyles, Tooltip, withStyles } from '@material-ui/core'; -import Star from '@material-ui/icons/Star'; -import StarBorder from '@material-ui/icons/StarBorder'; -import React, { ComponentProps } from 'react'; - -type Props = ComponentProps & { entity: Entity }; - -const YellowStar = withStyles({ - root: { - color: '#f3ba37', - }, -})(Star); - -const WhiteBorderStar = withStyles({ - root: { - color: '#ffffff', - }, -})(StarBorder); - -const useStyles = makeStyles(theme => ({ - starButton: { - position: 'absolute', - top: theme.spacing(0.5), - right: theme.spacing(0.5), - padding: '0.25rem', - }, -})); - -export const favouriteTemplateTooltip = (isStarred: boolean) => - isStarred ? 'Remove from favorites' : 'Add to favorites'; - -export const favouriteTemplateIcon = (isStarred: boolean) => - isStarred ? : ; - -/** - * IconButton for showing if a current entity is starred and adding/removing it from the favourite entities - * @param props - MaterialUI IconButton props extended by required `entity` prop - */ -export const FavouriteTemplate = (props: Props) => { - const classes = useStyles(); - const { toggleStarredEntity, isStarredEntity } = useStarredEntity( - props.entity, - ); - return ( - toggleStarredEntity()} - > - - {favouriteTemplateIcon(isStarredEntity)} - - - ); -}; diff --git a/plugins/scaffolder/src/components/FavouriteTemplate/index.ts b/plugins/scaffolder/src/components/FavouriteTemplate/index.ts deleted file mode 100644 index 5955e88c01..0000000000 --- a/plugins/scaffolder/src/components/FavouriteTemplate/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export * from './FavouriteTemplate'; diff --git a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx index f4710427c2..3a7691dd5a 100644 --- a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx +++ b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx @@ -21,6 +21,7 @@ import { } from '@backstage/integration-react'; import { EntityRefLinks, + FavoriteEntity, getEntityRelations, getEntitySourceLocation, } from '@backstage/plugin-catalog-react'; @@ -42,7 +43,6 @@ import { import WarningIcon from '@material-ui/icons/Warning'; import React from 'react'; import { selectedTemplateRouteRef } from '../../routes'; -import { FavouriteTemplate } from '../FavouriteTemplate/FavouriteTemplate'; import { Button, ItemCardHeader } from '@backstage/core-components'; import { useApi, useRouteRef } from '@backstage/core-plugin-api'; @@ -74,6 +74,12 @@ const useStyles = makeStyles(theme => ({ leftButton: { marginRight: 'auto', }, + starButton: { + position: 'absolute', + top: theme.spacing(0.5), + right: theme.spacing(0.5), + padding: '0.25rem', + }, })); const useDeprecationStyles = makeStyles(theme => ({ @@ -159,7 +165,7 @@ export const TemplateCard = ({ template, deprecated }: TemplateCardProps) => { return ( - + {deprecated && } Date: Wed, 2 Mar 2022 17:03:25 +0100 Subject: [PATCH 098/150] chore: update api-report Signed-off-by: blam --- plugins/scaffolder/api-report.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index a385e13534..b51f18fcc9 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -8,7 +8,6 @@ import { ApiHolder } from '@backstage/core-plugin-api'; import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; -import { ComponentProps } from 'react'; import { ComponentType } from 'react'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; @@ -17,7 +16,6 @@ import { ExternalRouteRef } from '@backstage/core-plugin-api'; import { FetchApi } from '@backstage/core-plugin-api'; import { FieldProps } from '@rjsf/core'; import { FieldValidation } from '@rjsf/core'; -import { IconButton } from '@material-ui/core'; import { JsonObject } from '@backstage/types'; import { JSONSchema7 } from 'json-schema'; import { JsonValue } from '@backstage/types'; @@ -87,12 +85,6 @@ export interface EntityTagsPickerUiOptions { kinds?: string[]; } -// Warning: (ae-forgotten-export) The symbol "Props" needs to be exported by the entry point index.d.ts -// Warning: (ae-missing-release-tag) "FavouriteTemplate" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export const FavouriteTemplate: (props: Props) => JSX.Element; - // @public export type FieldExtensionComponent<_TReturnValue, _TInputProps> = () => null; From 1c2755991dc56abf8a52b1153e3e2c7d6c067006 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 2 Mar 2022 17:05:26 +0100 Subject: [PATCH 099/150] chore: added changeset Signed-off-by: blam --- .changeset/large-dancers-learn.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/large-dancers-learn.md diff --git a/.changeset/large-dancers-learn.md b/.changeset/large-dancers-learn.md new file mode 100644 index 0000000000..43f9d27b11 --- /dev/null +++ b/.changeset/large-dancers-learn.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': minor +--- + +- **BREAKING**: Removed the `FavouriteEntity` export in favor of the `FavoriteEntity` from `@backstage/plguin-catalog-react`. Please migrate any usages to that component instead if you are creating your own `TemplateCard` page. From c8222292bfec538787f8c6c488103d0a414bce75 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 2 Mar 2022 17:08:25 +0100 Subject: [PATCH 100/150] chore: renamed formatEntityRefTitle to humanizeEnityRef Signed-off-by: blam --- .../CatalogGraphCard/CatalogGraphCard.tsx | 4 ++-- .../CatalogGraphPage/CatalogGraphPage.tsx | 8 ++++---- .../components/EntityRelationsGraph/CustomNode.tsx | 4 ++-- .../EntityListComponent/EntityListComponent.tsx | 8 ++++---- .../StepPrepareCreatePullRequest.tsx | 4 ++-- .../EntityOwnerPicker/EntityOwnerPicker.tsx | 4 ++-- .../src/components/EntityRefLink/EntityRefLink.tsx | 4 ++-- .../{format.test.ts => humanize.test.ts} | 14 +++++++------- .../EntityRefLink/{format.ts => humanize.ts} | 2 +- .../src/components/EntityRefLink/index.ts | 2 +- .../src/components/EntityTable/columns.tsx | 6 +++--- .../components/AncestryPage.tsx | 4 ++-- plugins/catalog-react/src/filters.ts | 4 ++-- .../src/components/CatalogTable/CatalogTable.tsx | 8 ++++---- .../src/components/CatalogTable/columns.tsx | 4 ++-- .../GroupsExplorerContent/GroupsDiagram.tsx | 4 ++-- .../fossa/src/components/FossaPage/FossaPage.tsx | 6 +++--- .../Cards/OwnershipCard/OwnershipCard.tsx | 4 ++-- .../fields/EntityPicker/EntityPicker.tsx | 4 ++-- .../fields/OwnedEntityPicker/OwnedEntityPicker.tsx | 4 ++-- .../src/home/components/Tables/DocsTable.tsx | 4 ++-- 21 files changed, 53 insertions(+), 53 deletions(-) rename plugins/catalog-react/src/components/EntityRefLink/{format.test.ts => humanize.test.ts} (86%) rename plugins/catalog-react/src/components/EntityRefLink/{format.ts => humanize.ts} (97%) diff --git a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx index b2dfd9f9b2..e6811aad62 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx @@ -21,7 +21,7 @@ import { import { InfoCard, InfoCardVariants } from '@backstage/core-components'; import { useAnalytics, useRouteRef } from '@backstage/core-plugin-api'; import { - formatEntityRefTitle, + humanizeEntityRef, useEntity, entityRouteRef, } from '@backstage/plugin-catalog-react'; @@ -94,7 +94,7 @@ export const CatalogGraphCard = ({ }); analytics.captureEvent( 'click', - node.title ?? formatEntityRefTitle(nodeEntityName), + node.title ?? humanizeEntityRef(nodeEntityName), { attributes: { to: path } }, ); navigate(path); diff --git a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx index 2ae7a99dd8..15d0cd139b 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx @@ -24,7 +24,7 @@ import { import { useAnalytics, useRouteRef } from '@backstage/core-plugin-api'; import { entityRouteRef, - formatEntityRefTitle, + humanizeEntityRef, } from '@backstage/plugin-catalog-react'; import { Grid, makeStyles, Paper, Typography } from '@material-ui/core'; import FilterListIcon from '@material-ui/icons/FilterList'; @@ -149,14 +149,14 @@ export const CatalogGraphPage = ({ analytics.captureEvent( 'click', - node.title ?? formatEntityRefTitle(nodeEntityName), + node.title ?? humanizeEntityRef(nodeEntityName), { attributes: { to: path } }, ); navigate(path); } else { analytics.captureEvent( 'click', - node.title ?? formatEntityRefTitle(nodeEntityName), + node.title ?? humanizeEntityRef(nodeEntityName), ); setRootEntityNames([nodeEntityName]); } @@ -168,7 +168,7 @@ export const CatalogGraphPage = ({

formatEntityRefTitle(e)).join(', ')} + subtitle={rootEntityNames.map(e => humanizeEntityRef(e)).join(', ')} /> ({ function sortEntities(entities: Array) { return entities.sort((a, b) => - formatEntityRefTitle(a).localeCompare(formatEntityRefTitle(b)), + humanizeEntityRef(a).localeCompare(humanizeEntityRef(b)), ); } @@ -130,7 +130,7 @@ export const EntityListComponent = (props: EntityListComponentProps) => { ) ?? WorkIcon; return ( { - + ); })} diff --git a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx index 36e8c33a5c..1cbdfce355 100644 --- a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx +++ b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx @@ -19,7 +19,7 @@ import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { assertError } from '@backstage/errors'; import { catalogApiRef, - formatEntityRefTitle, + humanizeEntityRef, } from '@backstage/plugin-catalog-react'; import { Box, FormHelperText, Grid, Typography } from '@material-ui/core'; import { makeStyles } from '@material-ui/core/styles'; @@ -139,7 +139,7 @@ export const StepPrepareCreatePullRequest = ( }); return groupEntities.items - .map(e => formatEntityRefTitle(e, { defaultKind: 'group' })) + .map(e => humanizeEntityRef(e, { defaultKind: 'group' })) .sort(); }); diff --git a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx index d025d08012..ec2c8126e5 100644 --- a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx +++ b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx @@ -31,7 +31,7 @@ import React, { useEffect, useMemo, useState } from 'react'; import { useEntityList } from '../../hooks/useEntityListProvider'; import { EntityOwnerFilter } from '../../filters'; import { getEntityRelations } from '../../utils'; -import { formatEntityRefTitle } from '../EntityRefLink'; +import { humanizeEntityRef } from '../EntityRefLink'; /** @public */ export type CatalogReactEntityOwnerPickerClassKey = 'input'; @@ -86,7 +86,7 @@ export const EntityOwnerPicker = () => { backendEntities .flatMap((e: Entity) => getEntityRelations(e, RELATION_OWNED_BY).map(o => - formatEntityRefTitle(o, { defaultKind: 'group' }), + humanizeEntityRef(o, { defaultKind: 'group' }), ), ) .filter(Boolean) as string[], diff --git a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx index 1e01402ddf..da2db24341 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx +++ b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx @@ -22,7 +22,7 @@ import { } from '@backstage/catalog-model'; import React, { forwardRef } from 'react'; import { entityRouteRef } from '../../routes'; -import { formatEntityRefTitle } from './format'; +import { humanizeEntityRef } from './format'; import { Link, LinkProps } from '@backstage/core-components'; import { useRouteRef } from '@backstage/core-plugin-api'; import { Tooltip } from '@material-ui/core'; @@ -72,7 +72,7 @@ export const EntityRefLink = forwardRef( namespace = namespace?.toLocaleLowerCase('en-US') ?? DEFAULT_NAMESPACE; const routeParams = { kind, namespace, name }; - const formattedEntityRefTitle = formatEntityRefTitle( + const formattedEntityRefTitle = humanizeEntityRef( { kind, namespace, name }, { defaultKind }, ); diff --git a/plugins/catalog-react/src/components/EntityRefLink/format.test.ts b/plugins/catalog-react/src/components/EntityRefLink/humanize.test.ts similarity index 86% rename from plugins/catalog-react/src/components/EntityRefLink/format.test.ts rename to plugins/catalog-react/src/components/EntityRefLink/humanize.test.ts index b489f088fd..b541946ece 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/format.test.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/humanize.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { formatEntityRefTitle } from './format'; +import { humanizeEntityRef } from './humanize'; describe('formatEntityRefTitle', () => { it('formats entity in default namespace', () => { @@ -30,7 +30,7 @@ describe('formatEntityRefTitle', () => { lifecycle: 'production', }, }; - const title = formatEntityRefTitle(entity); + const title = humanizeEntityRef(entity); expect(title).toEqual('component:software'); }); @@ -48,7 +48,7 @@ describe('formatEntityRefTitle', () => { lifecycle: 'production', }, }; - const title = formatEntityRefTitle(entity); + const title = humanizeEntityRef(entity); expect(title).toEqual('component:test/software'); }); @@ -66,7 +66,7 @@ describe('formatEntityRefTitle', () => { lifecycle: 'production', }, }; - const title = formatEntityRefTitle(entity, { defaultKind: 'Component' }); + const title = humanizeEntityRef(entity, { defaultKind: 'Component' }); expect(title).toEqual('test/software'); }); @@ -76,7 +76,7 @@ describe('formatEntityRefTitle', () => { namespace: 'default', name: 'software', }; - const title = formatEntityRefTitle(entityName); + const title = humanizeEntityRef(entityName); expect(title).toEqual('component:software'); }); @@ -87,7 +87,7 @@ describe('formatEntityRefTitle', () => { name: 'software', }; - const title = formatEntityRefTitle(entityName); + const title = humanizeEntityRef(entityName); expect(title).toEqual('component:test/software'); }); @@ -98,7 +98,7 @@ describe('formatEntityRefTitle', () => { name: 'software', }; - const title = formatEntityRefTitle(entityName, { + const title = humanizeEntityRef(entityName, { defaultKind: 'component', }); expect(title).toEqual('test/software'); diff --git a/plugins/catalog-react/src/components/EntityRefLink/format.ts b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts similarity index 97% rename from plugins/catalog-react/src/components/EntityRefLink/format.ts rename to plugins/catalog-react/src/components/EntityRefLink/humanize.ts index 90100245a1..35782c21b0 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/format.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts @@ -21,7 +21,7 @@ import { } from '@backstage/catalog-model'; /** @public */ -export function formatEntityRefTitle( +export function humanizeEntityRef( entityRef: Entity | EntityName, opts?: { defaultKind?: string }, ) { diff --git a/plugins/catalog-react/src/components/EntityRefLink/index.ts b/plugins/catalog-react/src/components/EntityRefLink/index.ts index d49993feb6..19e185459f 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/index.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/index.ts @@ -18,4 +18,4 @@ export { EntityRefLink } from './EntityRefLink'; export type { EntityRefLinkProps } from './EntityRefLink'; export { EntityRefLinks } from './EntityRefLinks'; export type { EntityRefLinksProps } from './EntityRefLinks'; -export { formatEntityRefTitle } from './format'; +export { humanizeEntityRef } from './format'; diff --git a/plugins/catalog-react/src/components/EntityTable/columns.tsx b/plugins/catalog-react/src/components/EntityTable/columns.tsx index d6c3582af6..d5ef791831 100644 --- a/plugins/catalog-react/src/components/EntityTable/columns.tsx +++ b/plugins/catalog-react/src/components/EntityTable/columns.tsx @@ -26,7 +26,7 @@ import { getEntityRelations } from '../../utils'; import { EntityRefLink, EntityRefLinks, - formatEntityRefTitle, + humanizeEntityRef, } from '../EntityRefLink'; /** @public */ @@ -38,7 +38,7 @@ export const columnFactories = Object.freeze({ function formatContent(entity: T): string { return ( entity.metadata?.title || - formatEntityRefTitle(entity, { + humanizeEntityRef(entity, { defaultKind, }) ); @@ -87,7 +87,7 @@ export const columnFactories = Object.freeze({ function formatContent(entity: T): string { return getRelations(entity) - .map(r => formatEntityRefTitle(r, { defaultKind })) + .map(r => humanizeEntityRef(r, { defaultKind })) .join(', '); } diff --git a/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx b/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx index bf6c8971b0..1f37ee41b1 100644 --- a/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx +++ b/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx @@ -33,7 +33,7 @@ import React, { useLayoutEffect, useRef, useState } from 'react'; import { useNavigate } from 'react-router'; import useAsync from 'react-use/lib/useAsync'; import { catalogApiRef } from '../../../api'; -import { formatEntityRefTitle } from '../../../components/EntityRefLink/format'; +import { humanizeEntityRef } from '../../../components/EntityRefLink/format'; import { entityRouteRef } from '../../../routes'; import { EntityKindIcon } from './EntityKindIcon'; @@ -132,7 +132,7 @@ function CustomNode({ node }: DependencyGraphTypes.RenderNodeProps) { const displayTitle = node.metadata.title || (node.kind && node.metadata.name && node.metadata.namespace - ? formatEntityRefTitle({ + ? humanizeEntityRef({ kind: node.kind, name: node.metadata.name, namespace: node.metadata.namespace || '', diff --git a/plugins/catalog-react/src/filters.ts b/plugins/catalog-react/src/filters.ts index 5d2c2e531b..9fb057a01c 100644 --- a/plugins/catalog-react/src/filters.ts +++ b/plugins/catalog-react/src/filters.ts @@ -15,7 +15,7 @@ */ import { Entity, RELATION_OWNED_BY } from '@backstage/catalog-model'; -import { formatEntityRefTitle } from './components/EntityRefLink'; +import { humanizeEntityRef } from './components/EntityRefLink'; import { EntityFilter, UserListFilterKind } from './types'; import { getEntityRelations } from './utils'; @@ -107,7 +107,7 @@ export class EntityOwnerFilter implements EntityFilter { filterEntity(entity: Entity): boolean { return this.values.some(v => getEntityRelations(entity, RELATION_OWNED_BY).some( - o => formatEntityRefTitle(o, { defaultKind: 'group' }) === v, + o => humanizeEntityRef(o, { defaultKind: 'group' }) === v, ), ); } diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx index a9a6f26228..4ef51d9945 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx @@ -22,7 +22,7 @@ import { import { favoriteEntityIcon, favoriteEntityTooltip, - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, useEntityList, useStarredEntities, @@ -132,16 +132,16 @@ export const CatalogTable = (props: CatalogTableProps) => { return { entity, resolved: { - name: formatEntityRefTitle(entity, { + name: humanizeEntityRef(entity, { defaultKind: 'Component', }), ownedByRelationsTitle: ownedByRelations - .map(r => formatEntityRefTitle(r, { defaultKind: 'group' })) + .map(r => humanizeEntityRef(r, { defaultKind: 'group' })) .join(', '), ownedByRelations, partOfSystemRelationTitle: partOfSystemRelations .map(r => - formatEntityRefTitle(r, { + humanizeEntityRef(r, { defaultKind: 'system', }), ) diff --git a/plugins/catalog/src/components/CatalogTable/columns.tsx b/plugins/catalog/src/components/CatalogTable/columns.tsx index d243be7c6f..0fb91d4f3f 100644 --- a/plugins/catalog/src/components/CatalogTable/columns.tsx +++ b/plugins/catalog/src/components/CatalogTable/columns.tsx @@ -15,7 +15,7 @@ */ import React from 'react'; import { - formatEntityRefTitle, + humanizeEntityRef, EntityRefLink, EntityRefLinks, } from '@backstage/plugin-catalog-react'; @@ -34,7 +34,7 @@ export const columnFactories = Object.freeze({ function formatContent(entity: Entity): string { return ( entity.metadata?.title || - formatEntityRefTitle(entity, { + humanizeEntityRef(entity, { defaultKind: options?.defaultKind, }) ); diff --git a/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx b/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx index 36788ed1b6..23553b8b76 100644 --- a/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx +++ b/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx @@ -31,7 +31,7 @@ import { configApiRef, useApi, useRouteRef } from '@backstage/core-plugin-api'; import { catalogApiRef, entityRouteRef, - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, } from '@backstage/plugin-catalog-react'; import { BackstageTheme } from '@backstage/theme'; @@ -193,7 +193,7 @@ export function GroupsDiagram() { kind: catalogItem.kind, name: (catalogItem as GroupEntity).spec?.profile?.displayName || - formatEntityRefTitle(catalogItem, { defaultKind: 'Group' }), + humanizeEntityRef(catalogItem, { defaultKind: 'Group' }), }); // Edge to parent diff --git a/plugins/fossa/src/components/FossaPage/FossaPage.tsx b/plugins/fossa/src/components/FossaPage/FossaPage.tsx index 1b53546ccd..90a36a6177 100644 --- a/plugins/fossa/src/components/FossaPage/FossaPage.tsx +++ b/plugins/fossa/src/components/FossaPage/FossaPage.tsx @@ -23,7 +23,7 @@ import { catalogApiRef, EntityRefLink, EntityRefLinks, - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, } from '@backstage/plugin-catalog-react'; import { Tooltip } from '@material-ui/core'; @@ -222,10 +222,10 @@ export const FossaPage = ({ return { entity, resolved: { - name: formatEntityRefTitle(entity), + name: humanizeEntityRef(entity), ownedByRelations, ownedByRelationsTitle: ownedByRelations - .map(r => formatEntityRefTitle(r, { defaultKind: 'group' })) + .map(r => humanizeEntityRef(r, { defaultKind: 'group' })) .join(', '), loading: summariesLoading, details: summary, diff --git a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx index 2b292cbe8b..cbf517bdca 100644 --- a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx +++ b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx @@ -25,7 +25,7 @@ import { import { useApi, useRouteRef } from '@backstage/core-plugin-api'; import { catalogApiRef, - formatEntityRefTitle, + humanizeEntityRef, isOwnerOf, useEntity, } from '@backstage/plugin-catalog-react'; @@ -108,7 +108,7 @@ const getQueryParams = ( owner: Entity, selectedEntity: EntityTypeProps, ): string => { - const ownerName = formatEntityRefTitle(owner, { defaultKind: 'group' }); + const ownerName = humanizeEntityRef(owner, { defaultKind: 'group' }); const { kind, type } = selectedEntity; const filters = { kind, diff --git a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx index 31c44bfe97..4a2f212fee 100644 --- a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx +++ b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx @@ -16,7 +16,7 @@ import { useApi } from '@backstage/core-plugin-api'; import { catalogApiRef, - formatEntityRefTitle, + humanizeEntityRef, } from '@backstage/plugin-catalog-react'; import { TextField } from '@material-ui/core'; import FormControl from '@material-ui/core/FormControl'; @@ -58,7 +58,7 @@ export const EntityPicker = ( ); const entityRefs = entities?.items.map(e => - formatEntityRefTitle(e, { defaultKind }), + humanizeEntityRef(e, { defaultKind }), ); const onSelect = useCallback( diff --git a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx index 80f63efaf9..aded9acffa 100644 --- a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx +++ b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ import { - formatEntityRefTitle, + humanizeEntityRef, useOwnedEntities, } from '@backstage/plugin-catalog-react'; import { TextField } from '@material-ui/core'; @@ -50,7 +50,7 @@ export const OwnedEntityPicker = ( const { ownedEntities, loading } = useOwnedEntities(allowedKinds); const entityRefs = ownedEntities?.items - .map(e => formatEntityRefTitle(e, { defaultKind })) + .map(e => humanizeEntityRef(e, { defaultKind })) .filter(n => n); const onSelect = (_: any, value: string | null) => { diff --git a/plugins/techdocs/src/home/components/Tables/DocsTable.tsx b/plugins/techdocs/src/home/components/Tables/DocsTable.tsx index b875ba4e98..883c496191 100644 --- a/plugins/techdocs/src/home/components/Tables/DocsTable.tsx +++ b/plugins/techdocs/src/home/components/Tables/DocsTable.tsx @@ -20,7 +20,7 @@ import useCopyToClipboard from 'react-use/lib/useCopyToClipboard'; import { useRouteRef, useApi, configApiRef } from '@backstage/core-plugin-api'; import { Entity, RELATION_OWNED_BY } from '@backstage/catalog-model'; import { - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, } from '@backstage/plugin-catalog-react'; import { rootDocsRouteRef } from '../../../routes'; @@ -76,7 +76,7 @@ export const DocsTable = (props: DocsTableProps) => { }), ownedByRelations, ownedByRelationsTitle: ownedByRelations - .map(r => formatEntityRefTitle(r, { defaultKind: 'group' })) + .map(r => humanizeEntityRef(r, { defaultKind: 'group' })) .join(', '), }, }; From f5a279aa8904a1846d4c692256b777a8e7c86048 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 2 Mar 2022 17:13:17 +0100 Subject: [PATCH 101/150] chore: added backwards compatible method too Signed-off-by: blam --- .../src/components/EntityRefLink/EntityRefLink.tsx | 2 +- plugins/catalog-react/src/components/EntityRefLink/humanize.ts | 3 +++ plugins/catalog-react/src/components/EntityRefLink/index.ts | 2 +- .../components/InspectEntityDialog/components/AncestryPage.tsx | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx index da2db24341..4db48351a9 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx +++ b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx @@ -22,7 +22,7 @@ import { } from '@backstage/catalog-model'; import React, { forwardRef } from 'react'; import { entityRouteRef } from '../../routes'; -import { humanizeEntityRef } from './format'; +import { humanizeEntityRef } from './humanize'; import { Link, LinkProps } from '@backstage/core-components'; import { useRouteRef } from '@backstage/core-plugin-api'; import { Tooltip } from '@material-ui/core'; diff --git a/plugins/catalog-react/src/components/EntityRefLink/humanize.ts b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts index 35782c21b0..af70ae71f4 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/humanize.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts @@ -20,6 +20,9 @@ import { DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; +/** @deprecated please use {@link humanizeEntityRef} instead */ +export const formatEntityRefTitle = humanizeEntityRef; + /** @public */ export function humanizeEntityRef( entityRef: Entity | EntityName, diff --git a/plugins/catalog-react/src/components/EntityRefLink/index.ts b/plugins/catalog-react/src/components/EntityRefLink/index.ts index 19e185459f..50394547a0 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/index.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/index.ts @@ -18,4 +18,4 @@ export { EntityRefLink } from './EntityRefLink'; export type { EntityRefLinkProps } from './EntityRefLink'; export { EntityRefLinks } from './EntityRefLinks'; export type { EntityRefLinksProps } from './EntityRefLinks'; -export { humanizeEntityRef } from './format'; +export { humanizeEntityRef, formatEntityRefTitle } from './humanize'; diff --git a/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx b/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx index 1f37ee41b1..0423c0da44 100644 --- a/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx +++ b/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx @@ -33,7 +33,7 @@ import React, { useLayoutEffect, useRef, useState } from 'react'; import { useNavigate } from 'react-router'; import useAsync from 'react-use/lib/useAsync'; import { catalogApiRef } from '../../../api'; -import { humanizeEntityRef } from '../../../components/EntityRefLink/format'; +import { humanizeEntityRef } from '../../EntityRefLink'; import { entityRouteRef } from '../../../routes'; import { EntityKindIcon } from './EntityKindIcon'; From f41a293231ceac36003bbafbd0062ba477476469 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 2 Mar 2022 17:15:06 +0100 Subject: [PATCH 102/150] chore: added a changeset for the new entitRef method to make humanz Signed-off-by: blam --- .changeset/gorgeous-boats-hide.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .changeset/gorgeous-boats-hide.md diff --git a/.changeset/gorgeous-boats-hide.md b/.changeset/gorgeous-boats-hide.md new file mode 100644 index 0000000000..5c60240433 --- /dev/null +++ b/.changeset/gorgeous-boats-hide.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-graph': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-explore': patch +'@backstage/plugin-fossa': patch +'@backstage/plugin-org': patch +'@backstage/plugin-scaffolder': patch +'@backstage/plugin-techdocs': patch +--- + +- **DEPRECATION**: Deprecated `formatEntityRefTitle` in favor of the new `humanizeEntityRef` method instead. Please migrate to using the new method instead. From 804c7906cfd345b77a16b00eda2a9f66c452ffdc Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 18:21:52 +0100 Subject: [PATCH 103/150] Update core search types to be stream-based Signed-off-by: Eric Peterson --- packages/search-common/api-report.md | 21 ++++++------ packages/search-common/src/types.ts | 32 +++++++++++++------ plugins/search-backend-node/src/types.ts | 13 +++++--- .../search-backend/src/service/router.test.ts | 2 +- 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/packages/search-common/api-report.md b/packages/search-common/api-report.md index 65f9f25180..e47140d274 100644 --- a/packages/search-common/api-report.md +++ b/packages/search-common/api-report.md @@ -3,25 +3,28 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { JsonObject } from '@backstage/types'; import { Permission } from '@backstage/plugin-permission-common'; +import { Readable } from 'stream'; +import { Transform } from 'stream'; +import { Writable } from 'stream'; -// Warning: (ae-missing-release-tag) "DocumentCollator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "DocumentCollatorFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public -export interface DocumentCollator { - // (undocumented) - execute(): Promise; +export interface DocumentCollatorFactory { + getCollator(): Promise; readonly type: string; readonly visibilityPermission?: Permission; } -// Warning: (ae-missing-release-tag) "DocumentDecorator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "DocumentDecoratorFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public -export interface DocumentDecorator { - // (undocumented) - execute(documents: IndexableDocument[]): Promise; +export interface DocumentDecoratorFactory { + getDecorator(): Promise; readonly types?: string[]; } @@ -60,7 +63,7 @@ export type QueryTranslator = (query: SearchQuery) => unknown; // // @public export interface SearchEngine { - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; query( query: SearchQuery, options?: QueryRequestOptions, diff --git a/packages/search-common/src/types.ts b/packages/search-common/src/types.ts index 4e61767ea0..6e106d3899 100644 --- a/packages/search-common/src/types.ts +++ b/packages/search-common/src/types.ts @@ -16,6 +16,7 @@ import { Permission } from '@backstage/plugin-permission-common'; import { JsonObject } from '@backstage/types'; +import { Readable, Transform, Writable } from 'stream'; export interface SearchQuery { term: string; @@ -82,10 +83,9 @@ export type DocumentTypeInfo = { }; /** - * Interface that must be implemented in order to expose new documents to - * search. + * Factory class for instantiating collators. */ -export interface DocumentCollator { +export interface DocumentCollatorFactory { /** * The type or name of the document set returned by this collator. Used as an * index name by Search Engines. @@ -98,21 +98,27 @@ export interface DocumentCollator { */ readonly visibilityPermission?: Permission; - execute(): Promise; + /** + * Instantiates and resolves a document collator. + */ + getCollator(): Promise; } /** - * Interface that must be implemented in order to decorate existing documents with - * additional metadata. + * Factory class for instantiating decorators. */ -export interface DocumentDecorator { +export interface DocumentDecoratorFactory { /** * An optional array of document/index types on which this decorator should * be applied. If no types are provided, this decorator will be applied to * all document/index types. */ readonly types?: string[]; - execute(documents: IndexableDocument[]): Promise; + + /** + * Instantiates and resolves a document decorator. + */ + getDecorator(): Promise; } /** @@ -137,9 +143,15 @@ export interface SearchEngine { setTranslator(translator: QueryTranslator): void; /** - * Add the given documents to the SearchEngine index of the given type. + * Factory method for getting a search engine indexer for a given document + * type. + * + * @param type - The type or name of the document set for which an indexer + * should be retrieved. This corresponds to the `type` property on the + * document collator/decorator factories and will most often be used to + * identify an index or group to which documents should be written. */ - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; /** * Perform a search query against the SearchEngine. diff --git a/plugins/search-backend-node/src/types.ts b/plugins/search-backend-node/src/types.ts index df83357f6c..4bcc8ec114 100644 --- a/plugins/search-backend-node/src/types.ts +++ b/plugins/search-backend-node/src/types.ts @@ -14,7 +14,10 @@ * limitations under the License. */ -import { DocumentCollator, DocumentDecorator } from '@backstage/search-common'; +import { + DocumentCollatorFactory, + DocumentDecoratorFactory, +} from '@backstage/search-common'; /** * Parameters required to register a collator. @@ -26,9 +29,9 @@ export interface RegisterCollatorParameters { defaultRefreshIntervalSeconds: number; /** - * The collator class responsible for returning all documents of the given type. + * The class responsible for returning the document collator of the given type. */ - collator: DocumentCollator; + factory: DocumentCollatorFactory; } /** @@ -36,7 +39,7 @@ export interface RegisterCollatorParameters { */ export interface RegisterDecoratorParameters { /** - * The decorator class responsible for appending or modifying documents of the given type(s). + * The class responsible for returning the decorator which appends, modifies, or filters documents. */ - decorator: DocumentDecorator; + factory: DocumentDecoratorFactory; } diff --git a/plugins/search-backend/src/service/router.test.ts b/plugins/search-backend/src/service/router.test.ts index 0a94250aeb..bdf46a240b 100644 --- a/plugins/search-backend/src/service/router.test.ts +++ b/plugins/search-backend/src/service/router.test.ts @@ -105,7 +105,7 @@ describe('createRouter', () => { beforeAll(async () => { const logger = getVoidLogger(); mockSearchEngine = { - index: jest.fn(), + getIndexer: jest.fn(), setTranslator: jest.fn(), query: jest.fn(), }; From 2eae26293e3ecc1adc30a8fa870b5c260284a3b1 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 18:23:13 +0100 Subject: [PATCH 104/150] Update IndexBuilder to be a stream pipeline Signed-off-by: Eric Peterson --- .../src/IndexBuilder.test.ts | 82 +++++++-------- .../search-backend-node/src/IndexBuilder.ts | 99 +++++++++---------- 2 files changed, 83 insertions(+), 98 deletions(-) diff --git a/plugins/search-backend-node/src/IndexBuilder.test.ts b/plugins/search-backend-node/src/IndexBuilder.test.ts index 0465f2701c..95a2e1d439 100644 --- a/plugins/search-backend-node/src/IndexBuilder.test.ts +++ b/plugins/search-backend-node/src/IndexBuilder.test.ts @@ -16,35 +16,37 @@ import { getVoidLogger } from '@backstage/backend-common'; import { - DocumentCollator, - DocumentDecorator, - IndexableDocument, + DocumentCollatorFactory, + DocumentDecoratorFactory, } from '@backstage/search-common'; +import { Readable, Transform } from 'stream'; import { IndexBuilder } from './IndexBuilder'; import { LunrSearchEngine, SearchEngine } from './index'; -class TestDocumentCollator implements DocumentCollator { +class TestDocumentCollatorFactory implements DocumentCollatorFactory { readonly type: string = 'anything'; - async execute(): Promise { - return []; + async getCollator(): Promise { + const collator = new Readable({ objectMode: true }); + collator._read = () => {}; + return collator; } } -class TypedDocumentCollator extends TestDocumentCollator { +class TypedDocumentCollatorFactory extends TestDocumentCollatorFactory { readonly type = 'an-expected-type'; } -class TestDocumentDecorator implements DocumentDecorator { - async execute(documents: IndexableDocument[]) { - return documents; +class TestDocumentDecoratorFactory implements DocumentDecoratorFactory { + async getDecorator(): Promise { + return new Transform(); } } -class TypedDocumentDecorator extends TestDocumentDecorator { +class TypedDocumentDecoratorFactory extends TestDocumentDecoratorFactory { readonly types = ['an-expected-type']; } -class DifferentlyTypedDocumentDecorator extends TestDocumentDecorator { +class DifferentlyTypedDocumentDecoratorFactory extends TestDocumentDecoratorFactory { readonly types = ['not-the-expected-type']; } @@ -64,13 +66,13 @@ describe('IndexBuilder', () => { describe('addCollator', () => { it('adds a collator', async () => { jest.useFakeTimers(); - const testCollator = new TestDocumentCollator(); - const collatorSpy = jest.spyOn(testCollator, 'execute'); + const testCollatorFactory = new TestDocumentCollatorFactory(); + const collatorSpy = jest.spyOn(testCollatorFactory, 'getCollator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Build the index and ensure the collator was invoked. @@ -84,19 +86,19 @@ describe('IndexBuilder', () => { describe('addDecorator', () => { it('adds a decorator', async () => { jest.useFakeTimers(); - const testCollator = new TestDocumentCollator(); - const testDecorator = new TestDocumentDecorator(); - const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + const testCollatorFactory = new TestDocumentCollatorFactory(); + const testDecoratorFactory = new TestDocumentDecoratorFactory(); + const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Add a decorator. testIndexBuilder.addDecorator({ - decorator: testDecorator, + factory: testDecoratorFactory, }); // Build the index and ensure the decorator was invoked. @@ -110,27 +112,20 @@ describe('IndexBuilder', () => { it('adds a type-specific decorator', async () => { jest.useFakeTimers(); - const testCollator = new TypedDocumentCollator(); - const testDecorator = new TypedDocumentDecorator(); - const docFixture = { - title: 'Test', - text: 'Test text.', - location: '/test/location', - }; - jest - .spyOn(testCollator, 'execute') - .mockImplementation(async () => [docFixture]); - const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + const testCollatorFactory = new TypedDocumentCollatorFactory(); + const testDecoratorFactory = new TypedDocumentDecoratorFactory(); + jest.spyOn(testCollatorFactory, 'getCollator'); + const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Add a decorator for the same type. testIndexBuilder.addDecorator({ - decorator: testDecorator, + factory: testDecoratorFactory, }); // Build the index and ensure the decorator was invoked. @@ -140,31 +135,24 @@ describe('IndexBuilder', () => { // wait for async decorator execution await Promise.resolve(); expect(decoratorSpy).toHaveBeenCalled(); - expect(decoratorSpy).toHaveBeenCalledWith([docFixture]); }); it('adds a type-specific decorator that should not be called', async () => { - const docFixture = { - title: 'Test', - text: 'Test text.', - location: '/test/location', - }; - const testCollator = new TestDocumentCollator(); - const testDecorator = new DifferentlyTypedDocumentDecorator(); - const collatorSpy = jest - .spyOn(testCollator, 'execute') - .mockImplementation(async () => [docFixture]); - const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + const testCollatorFactory = new TestDocumentCollatorFactory(); + const testDecoratorFactory = + new DifferentlyTypedDocumentDecoratorFactory(); + const collatorSpy = jest.spyOn(testCollatorFactory, 'getCollator'); + const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Add a decorator for a different type. testIndexBuilder.addDecorator({ - decorator: testDecorator, + factory: testDecoratorFactory, }); // Build the index and ensure the decorator was not invoked. diff --git a/plugins/search-backend-node/src/IndexBuilder.ts b/plugins/search-backend-node/src/IndexBuilder.ts index 92adb03d7d..5f39c2fa33 100644 --- a/plugins/search-backend-node/src/IndexBuilder.ts +++ b/plugins/search-backend-node/src/IndexBuilder.ts @@ -15,12 +15,12 @@ */ import { - DocumentCollator, - DocumentDecorator, + DocumentCollatorFactory, + DocumentDecoratorFactory, DocumentTypeInfo, - IndexableDocument, SearchEngine, } from '@backstage/search-common'; +import { Transform, pipeline } from 'stream'; import { Logger } from 'winston'; import { Scheduler } from './index'; import { @@ -29,7 +29,7 @@ import { } from './types'; interface CollatorEnvelope { - collate: DocumentCollator; + factory: DocumentCollatorFactory; refreshInterval: number; } @@ -40,7 +40,7 @@ type IndexBuilderOptions = { export class IndexBuilder { private collators: Record; - private decorators: Record; + private decorators: Record; private documentTypes: Record; private searchEngine: SearchEngine; private logger: Logger; @@ -66,18 +66,18 @@ export class IndexBuilder { * given refresh interval. */ addCollator({ - collator, + factory, defaultRefreshIntervalSeconds, }: RegisterCollatorParameters): void { this.logger.info( - `Added ${collator.constructor.name} collator for type ${collator.type}`, + `Added ${factory.constructor.name} collator factory for type ${factory.type}`, ); - this.collators[collator.type] = { + this.collators[factory.type] = { refreshInterval: defaultRefreshIntervalSeconds, - collate: collator, + factory, }; - this.documentTypes[collator.type] = { - visibilityPermission: collator.visibilityPermission, + this.documentTypes[factory.type] = { + visibilityPermission: factory.visibilityPermission, }; } @@ -86,18 +86,18 @@ export class IndexBuilder { * the decorator, it will be applied to documents from all known collators, * otherwise it will only be applied to documents of the given types. */ - addDecorator({ decorator }: RegisterDecoratorParameters): void { - const types = decorator.types || ['*']; + addDecorator({ factory }: RegisterDecoratorParameters): void { + const types = factory.types || ['*']; this.logger.info( - `Added decorator ${decorator.constructor.name} to types ${types.join( + `Added decorator ${factory.constructor.name} to types ${types.join( ', ', )}`, ); types.forEach(type => { if (this.decorators.hasOwnProperty(type)) { - this.decorators[type].push(decorator); + this.decorators[type].push(factory); } else { - this.decorators[type] = [decorator]; + this.decorators[type] = [factory]; } }); } @@ -111,46 +111,43 @@ export class IndexBuilder { Object.keys(this.collators).forEach(type => { scheduler.addToSchedule(async () => { - // Collate, Decorate, Index. - const decorators: DocumentDecorator[] = ( - this.decorators['*'] || [] - ).concat(this.decorators[type] || []); - - this.logger.debug( - `Collating documents for ${type} via ${this.collators[type].collate.constructor.name}`, + // Instantiate the collator. + const collator = await this.collators[type].factory.getCollator(); + this.logger.info( + `Collating documents for ${type} via ${this.collators[type].factory.constructor.name}`, ); - let documents: IndexableDocument[]; - try { - documents = await this.collators[type].collate.execute(); - } catch (e) { - this.logger.error( - `Collating documents for ${type} via ${this.collators[type].collate.constructor.name} failed: ${e}`, - ); - return; - } + // Instantiate all relevant decorators. + const decorators: Transform[] = await Promise.all( + (this.decorators['*'] || []) + .concat(this.decorators[type] || []) + .map(async factory => { + const decorator = await factory.getDecorator(); + this.logger.info( + `Attached decorator via ${factory.constructor.name} to ${type} index pipeline.`, + ); + return decorator; + }), + ); - for (let i = 0; i < decorators.length; i++) { - this.logger.debug( - `Decorating ${type} documents via ${decorators[i].constructor.name}`, - ); - try { - documents = await decorators[i].execute(documents); - } catch (e) { - this.logger.error( - `Decorating ${type} documents via ${decorators[i].constructor.name} failed: ${e}`, - ); - return; - } - } + // Instantiate the indexer. + const indexer = await this.searchEngine.getIndexer(type); - if (!documents || documents.length === 0) { - this.logger.debug(`No documents for type "${type}" to index`); - return; - } + // Compose collator/decorators/indexer into a pipeline + return new Promise(done => { + pipeline([collator, ...decorators, indexer], error => { + if (error) { + this.logger.error( + `Collating documents for ${type} failed: ${error}`, + ); + } else { + this.logger.info(`Collating documents for ${type} succeeded`); + } - // pushing documents to index to a configured search engine. - await this.searchEngine.index(type, documents); + // Signal index pipeline completion! + done(); + }); + }); }, this.collators[type].refreshInterval * 1000); }); From ce3f566e9c37bace23d002d5962b4a94ac544979 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 18:23:49 +0100 Subject: [PATCH 105/150] Introduce test utilities for stream-based search implementations Signed-off-by: Eric Peterson --- .../src/test-utils/TestPipeline.ts | 138 ++++++++++++++++++ .../src/test-utils/index.ts | 18 +++ 2 files changed, 156 insertions(+) create mode 100644 plugins/search-backend-node/src/test-utils/TestPipeline.ts create mode 100644 plugins/search-backend-node/src/test-utils/index.ts diff --git a/plugins/search-backend-node/src/test-utils/TestPipeline.ts b/plugins/search-backend-node/src/test-utils/TestPipeline.ts new file mode 100644 index 0000000000..2dbdeb85ad --- /dev/null +++ b/plugins/search-backend-node/src/test-utils/TestPipeline.ts @@ -0,0 +1,138 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import { pipeline, Readable, Transform, Writable } from 'stream'; + +/** + * Object resolved after a test pipeline is executed. + */ +export type TestPipelineResult = { + /** + * If an error was emitted by the pipeline, it will be set here. + */ + error: unknown; + + /** + * A list of documents collected at the end of the pipeline. If the subject + * under test is an indexer, this will be an empty array (because your + * indexer should have received the documents instead). + */ + documents: IndexableDocument[]; +}; + +/** + * Test utility for Backstage Search collators, decorators, and indexers. + */ +export class TestPipeline { + private collator?: Readable; + private decorator?: Transform; + private indexer?: Writable; + + private constructor({ + collator, + decorator, + indexer, + }: { + collator?: Readable; + decorator?: Transform; + indexer?: Writable; + }) { + this.collator = collator; + this.decorator = decorator; + this.indexer = indexer; + } + + /** + * Provide the collator, decorator, or indexer to be tested. + */ + static withSubject(subject: Readable | Transform | Writable) { + if (subject instanceof Transform) { + return new TestPipeline({ decorator: subject }); + } + + if (subject instanceof Readable) { + return new TestPipeline({ collator: subject }); + } + + if (subject instanceof Writable) { + return new TestPipeline({ indexer: subject }); + } + + throw new Error( + 'Unknown test subject: are you passing a readable, writable, or transform stream?', + ); + } + + /** + * Provide documents for testing decorators and indexers. + */ + withDocuments(documents: IndexableDocument[]): TestPipeline { + if (this.collator) { + throw new Error('Cannot provide documents when testing a collator.'); + } + + // Set a naive readable stream that just pushes all given documents. + this.collator = new Readable({ objectMode: true }); + this.collator._read = () => {}; + process.nextTick(() => { + documents.forEach(document => { + this.collator!.push(document); + }); + this.collator!.push(null); + }); + + return this; + } + + /** + * Execute the test pipeline so that you can make assertions about the result + * or behavior of the given test subject. + */ + async execute(): Promise { + const documents: IndexableDocument[] = []; + if (!this.collator) { + throw new Error( + 'Cannot execute pipeline without a collator or documents', + ); + } + + // If we are here and there is no indexer, we are testing a collator or a + // decorator. Set up a naive writable that captures documents in memory. + if (!this.indexer) { + this.indexer = new Writable({ objectMode: true }); + this.indexer._write = (document: IndexableDocument, _, done) => { + documents.push(document); + done(); + }; + } + + return new Promise(done => { + const pipes: (Readable | Transform | Writable)[] = [this.collator!]; + if (this.decorator) { + pipes.push(this.decorator); + } + pipes.push(this.indexer!); + + pipeline(pipes, error => { + done({ + error, + documents, + }); + }); + }); + } +} diff --git a/plugins/search-backend-node/src/test-utils/index.ts b/plugins/search-backend-node/src/test-utils/index.ts new file mode 100644 index 0000000000..185d25433b --- /dev/null +++ b/plugins/search-backend-node/src/test-utils/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { TestPipeline } from './TestPipeline'; +export type { TestPipelineResult } from './TestPipeline'; From cef19ee9662d0746ccac7b047a6ce9bf94af2781 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 18:24:38 +0100 Subject: [PATCH 106/150] Introduce a base classes to simplify stream-based implementations Signed-off-by: Eric Peterson --- .../indexing/BatchSearchEngineIndexer.test.ts | 156 +++++++++++++++++ .../src/indexing/BatchSearchEngineIndexer.ts | 121 ++++++++++++++ .../src/indexing/DecoratorBase.test.ts | 157 ++++++++++++++++++ .../src/indexing/DecoratorBase.ts | 126 ++++++++++++++ .../search-backend-node/src/indexing/index.ts | 19 +++ 5 files changed, 579 insertions(+) create mode 100644 plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.test.ts create mode 100644 plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts create mode 100644 plugins/search-backend-node/src/indexing/DecoratorBase.test.ts create mode 100644 plugins/search-backend-node/src/indexing/DecoratorBase.ts create mode 100644 plugins/search-backend-node/src/indexing/index.ts diff --git a/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.test.ts b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..b692ce3aff --- /dev/null +++ b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.test.ts @@ -0,0 +1,156 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import { BatchSearchEngineIndexer } from './BatchSearchEngineIndexer'; +import { TestPipeline } from '../test-utils'; + +const indexSpy = jest.fn().mockResolvedValue(undefined); +const initializeSpy = jest.fn().mockResolvedValue(undefined); +const finalizeSpy = jest.fn().mockResolvedValue(undefined); + +class ConcreteBatchIndexer extends BatchSearchEngineIndexer { + async index(documents: IndexableDocument[]): Promise { + return indexSpy(documents); + } + async initialize(): Promise { + return initializeSpy(); + } + async finalize(): Promise { + return finalizeSpy(); + } +} + +describe('BatchSearchEngineIndexer', () => { + const document = { + title: 'Some Document', + text: 'Some document text.', + location: '/some/location', + }; + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should work end-to-end', async () => { + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + await TestPipeline.withSubject(indexer) + .withDocuments([document, document, document]) + .execute(); + expect(indexSpy).toHaveBeenCalledTimes(3); + }); + + it('should call initialize at construction', () => { + // @ts-expect-error + const _indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Allow initialization to complete. + setImmediate(() => { + expect(initializeSpy).toHaveBeenCalled(); + done(); + }); + }); + }); + + it('should emit error if initialization throws', () => { + // Cause the initializer to throw. + const expectedError = new Error('some error'); + initializeSpy.mockRejectedValue(expectedError); + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + indexer.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + // Write a document to force the error state to become known. + indexer.write(document); + }); + }); + + it('should call index according to batchSize', () => { + const indexer = new ConcreteBatchIndexer({ batchSize: 2 }); + + return new Promise(done => { + // Listen for it to finish and assert the batches. + indexer.on('finish', () => { + expect(indexSpy).toHaveBeenCalledTimes(2); + expect(indexSpy).toHaveBeenNthCalledWith(1, [document, document]); + expect(indexSpy).toHaveBeenNthCalledWith(2, [document]); + done(); + }); + + // Write batchSize + 1 documents and end the stream. + indexer.write(document); + indexer.write(document); + indexer.write(document); + indexer.end(); + }); + }); + + it('should call index without exceeding batchSize', () => { + const indexer = new ConcreteBatchIndexer({ batchSize: 2 }); + + return new Promise(done => { + // Listen for it to finish and assert that it still wrote. + indexer.on('finish', () => { + expect(indexSpy).toHaveBeenCalledTimes(1); + expect(indexSpy).toHaveBeenNthCalledWith(1, [document]); + done(); + }); + + // Write batchSize - 1 documents and end the stream. + indexer.write(document); + indexer.end(); + }); + }); + + it('should emit error if index throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('index error'); + indexSpy.mockRejectedValue(expectedError); + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + indexer.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + indexer.write(document); + }); + }); + + it('should emit error if finalize throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('finalize error'); + finalizeSpy.mockRejectedValue(expectedError); + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + indexer.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + indexer.end(); + }); + }); +}); diff --git a/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts new file mode 100644 index 0000000000..4c29d0b573 --- /dev/null +++ b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts @@ -0,0 +1,121 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { assertError } from '@backstage/errors'; +import { IndexableDocument } from '@backstage/search-common'; +import { Writable } from 'stream'; + +export type BatchSearchEngineOptions = { + batchSize: number; +}; + +/** + * Base class encapsulating batch-based stream processing. Useful as a base + * class for search engine indexers. + */ +export abstract class BatchSearchEngineIndexer extends Writable { + private batchSize: number; + private currentBatch: IndexableDocument[] = []; + private initialized: Promise; + + constructor(options: BatchSearchEngineOptions) { + super({ objectMode: true }); + this.batchSize = options.batchSize; + + // @todo Once node v15 is minimum, convert to _construct implementation. + this.initialized = new Promise(done => { + // Necessary to allow concrete implementation classes to construct + // themselves before calling their initialize() methods. + setImmediate(async () => { + try { + await this.initialize(); + done(undefined); + } catch (e) { + assertError(e); + done(e); + } + }); + }); + } + + /** + * Receives an array of indexable documents (of size this.batchSize) which + * should be written to the search engine. This method won't be called again + * at least until it resolves. + */ + public abstract index(documents: IndexableDocument[]): Promise; + + /** + * Any asynchronous setup tasks can be performed here. + */ + public abstract initialize(): Promise; + + /** + * Any asynchronous teardown tasks can be performed here. + */ + public abstract finalize(): Promise; + + /** + * Encapsulates batch stream write logic. + * @internal + */ + async _write( + doc: IndexableDocument, + _e: any, + done: (error?: Error | null) => void, + ) { + // Wait for init before proceeding. Throw error if initialization failed. + const maybeError = await this.initialized; + if (maybeError) { + done(maybeError); + return; + } + + this.currentBatch.push(doc); + if (this.currentBatch.length < this.batchSize) { + done(); + return; + } + + try { + await this.index(this.currentBatch); + this.currentBatch = []; + done(); + } catch (e) { + assertError(e); + done(e); + } + } + + /** + * Encapsulates finalization and final error handling logic. + * @internal + */ + async _final(done: (error?: Error | null) => void) { + try { + // Index any remaining documents. + if (this.currentBatch.length) { + await this.index(this.currentBatch); + this.currentBatch = []; + } + await this.finalize(); + done(); + } catch (e) { + assertError(e); + done(e); + } + } +} diff --git a/plugins/search-backend-node/src/indexing/DecoratorBase.test.ts b/plugins/search-backend-node/src/indexing/DecoratorBase.test.ts new file mode 100644 index 0000000000..3b045dcddf --- /dev/null +++ b/plugins/search-backend-node/src/indexing/DecoratorBase.test.ts @@ -0,0 +1,157 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import { DecoratorBase } from './DecoratorBase'; +import { TestPipeline } from '../test-utils'; + +const decorateSpy = jest.fn().mockResolvedValue(undefined); +const initializeSpy = jest.fn().mockResolvedValue(undefined); +const finalizeSpy = jest.fn().mockResolvedValue(undefined); + +class ConcreteDecorator extends DecoratorBase { + public initialize(): Promise { + return initializeSpy(); + } + public decorate( + document: IndexableDocument, + ): Promise { + return decorateSpy(document); + } + public finalize(): Promise { + return finalizeSpy(); + } +} + +describe('DecoratorBase', () => { + const document = { + title: 'Some Document', + text: 'Some document text.', + location: '/some/location', + }; + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should work end-to-end', async () => { + decorateSpy.mockImplementation(doc => ({ + ...doc, + transformed: true, + })); + + const decorator = new ConcreteDecorator(); + const { documents } = await TestPipeline.withSubject(decorator) + .withDocuments([document, document, document]) + .execute(); + + expect(documents.length).toBe(3); + expect((documents[0] as unknown as any).transformed).toBe(true); + expect((documents[1] as unknown as any).transformed).toBe(true); + expect((documents[2] as unknown as any).transformed).toBe(true); + }); + + it('should allow filtering', async () => { + decorateSpy.mockResolvedValue(undefined); + + const decorator = new ConcreteDecorator(); + const { documents } = await TestPipeline.withSubject(decorator) + .withDocuments([document, document, document]) + .execute(); + + expect(decorateSpy).toHaveBeenCalledTimes(3); + expect(documents.length).toBe(0); + }); + + it('should allow fanning', async () => { + decorateSpy.mockImplementation(doc => { + return [doc, doc]; + }); + + const decorator = new ConcreteDecorator(); + const { documents } = await TestPipeline.withSubject(decorator) + .withDocuments([document, document, document]) + .execute(); + + expect(decorateSpy).toHaveBeenCalledTimes(3); + expect(documents.length).toBe(6); + }); + + it('should call initialize at construction', () => { + // @ts-expect-error + const _indexer = new ConcreteDecorator(); + + return new Promise(done => { + // Allow initialization to complete. + setImmediate(() => { + expect(initializeSpy).toHaveBeenCalled(); + done(); + }); + }); + }); + + it('should emit error if initialization throws', () => { + // Cause the initializer to throw. + const expectedError = new Error('some error'); + initializeSpy.mockRejectedValue(expectedError); + const decorator = new ConcreteDecorator(); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + decorator.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + // Write a document to force the error state to become known. + decorator.write(document); + }); + }); + + it('should emit error if index throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('decorate error'); + decorateSpy.mockRejectedValue(expectedError); + const decorator = new ConcreteDecorator(); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + decorator.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + decorator.write(document); + }); + }); + + it('should emit error if finalize throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('finalize error'); + finalizeSpy.mockRejectedValue(expectedError); + const decorator = new ConcreteDecorator(); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + decorator.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + decorator.end(); + }); + }); +}); diff --git a/plugins/search-backend-node/src/indexing/DecoratorBase.ts b/plugins/search-backend-node/src/indexing/DecoratorBase.ts new file mode 100644 index 0000000000..a28d652fd7 --- /dev/null +++ b/plugins/search-backend-node/src/indexing/DecoratorBase.ts @@ -0,0 +1,126 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { assertError } from '@backstage/errors'; +import { IndexableDocument } from '@backstage/search-common'; +import { Transform } from 'stream'; + +/** + * Base class encapsulating simple async transformations. Useful as a base + * class for Backstage search decorators. + */ +export abstract class DecoratorBase extends Transform { + private initialized: Promise; + + constructor() { + super({ objectMode: true }); + + // @todo Once node v15 is minimum, convert to _construct implementation. + this.initialized = new Promise(done => { + // Necessary to allow concrete implementation classes to construct + // themselves before calling their initialize() methods. + setImmediate(async () => { + try { + await this.initialize(); + done(undefined); + } catch (e) { + assertError(e); + done(e); + } + }); + }); + } + + /** + * Any asynchronous setup tasks can be performed here. + */ + public abstract initialize(): Promise; + + /** + * Receives a single indexable document. In your decorate method, you can: + * + * - Resolve `undefined` to indicate the record should be omitted. + * - Resolve a single modified document, which could contain new fields, + * edited fields, or removed fields. + * - Resolve an array of indexable documents, if the purpose if the decorator + * is to convert one document into multiple derivative documents. + */ + public abstract decorate( + document: IndexableDocument, + ): Promise; + + /** + * Any asynchronous teardown tasks can be performed here. + */ + public abstract finalize(): Promise; + + /** + * Encapsulates simple transform stream logic. + * @internal + */ + async _transform( + document: IndexableDocument, + _: any, + done: (error?: Error | null) => void, + ) { + // Wait for init before proceeding. Throw error if initialization failed. + const maybeError = await this.initialized; + if (maybeError) { + done(maybeError); + return; + } + + try { + const decorated = await this.decorate(document); + + // If undefined was returned, omit the record and move on. + if (decorated === undefined) { + done(); + return; + } + + // If an array of documents was given, push them all. + if (Array.isArray(decorated)) { + decorated.forEach(doc => { + this.push(doc); + }); + done(); + return; + } + + // Otherwise, just push the decorated document. + this.push(decorated); + done(); + } catch (e) { + assertError(e); + done(e); + } + } + + /** + * Encapsulates finalization and final error handling logic. + * @internal + */ + async _final(done: (error?: Error | null) => void) { + try { + await this.finalize(); + done(); + } catch (e) { + assertError(e); + done(e); + } + } +} diff --git a/plugins/search-backend-node/src/indexing/index.ts b/plugins/search-backend-node/src/indexing/index.ts new file mode 100644 index 0000000000..e86235df16 --- /dev/null +++ b/plugins/search-backend-node/src/indexing/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { BatchSearchEngineIndexer } from './BatchSearchEngineIndexer'; +export { DecoratorBase } from './DecoratorBase'; +export type { BatchSearchEngineOptions } from './BatchSearchEngineIndexer'; From c87be57d3a6d515b87009a642438bb1eeb0e65cf Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 18:25:05 +0100 Subject: [PATCH 107/150] Update LunrSearchEngine to be stream-based Signed-off-by: Eric Peterson --- .../src/engines/LunrSearchEngine.test.ts | 221 +++++++++++++++--- .../src/engines/LunrSearchEngine.ts | 30 +-- .../engines/LunrSearchEngineIndexer.test.ts | 109 +++++++++ .../src/engines/LunrSearchEngineIndexer.ts | 68 ++++++ .../search-backend-node/src/engines/index.ts | 1 + 5 files changed, 370 insertions(+), 59 deletions(-) create mode 100644 plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts create mode 100644 plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts index cbe48ef5a7..88a3ba9a24 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts @@ -16,28 +16,61 @@ import { getVoidLogger } from '@backstage/backend-common'; import lunr from 'lunr'; -import { SearchEngine } from '@backstage/search-common'; +import { IndexableDocument, SearchEngine } from '@backstage/search-common'; import { ConcreteLunrQuery, LunrSearchEngine, decodePageCursor, encodePageCursor, } from './LunrSearchEngine'; +import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; +import { TestPipeline } from '../test-utils'; /** * Just used to test the default translator shipped with LunrSearchEngine. */ -class LunrSearchEngineForTranslatorTests extends LunrSearchEngine { +class LunrSearchEngineForTests extends LunrSearchEngine { + getDocStore() { + return this.docStore; + } + setDocStore(docStore: Record) { + this.docStore = docStore; + } + getLunrIndices() { + return this.lunrIndices; + } getTranslator() { return this.translator; } } +const indexerMock = { + on: jest.fn(), + buildIndex: jest.fn(), + getDocumentStore: jest.fn(), +}; +jest.mock('./LunrSearchEngineIndexer', () => ({ + LunrSearchEngineIndexer: jest.fn().mockImplementation(() => indexerMock), +})); + +const getActualIndexer = (engine: SearchEngine, index: string) => { + (LunrSearchEngineIndexer as unknown as jest.Mock).mockImplementationOnce( + () => { + const ActualIndexer = jest.requireActual( + './LunrSearchEngineIndexer', + ).LunrSearchEngineIndexer; + return new ActualIndexer(); + }, + ); + return engine.getIndexer(index); +}; + describe('LunrSearchEngine', () => { let testLunrSearchEngine: SearchEngine; beforeEach(() => { testLunrSearchEngine = new LunrSearchEngine({ logger: getVoidLogger() }); + jest.clearAllMocks(); }); describe('translator', () => { @@ -65,7 +98,7 @@ describe('LunrSearchEngine', () => { }); it('should return translated query', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -107,7 +140,7 @@ describe('LunrSearchEngine', () => { }); it('should have default offset and limit', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -148,7 +181,7 @@ describe('LunrSearchEngine', () => { }); it('should return translated query with 1 filter', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -193,7 +226,7 @@ describe('LunrSearchEngine', () => { }); it('should handle single-item array filter as scalar value', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -224,7 +257,7 @@ describe('LunrSearchEngine', () => { }); it('should return translated query with multiple filters', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -273,7 +306,7 @@ describe('LunrSearchEngine', () => { }); it('should throw if translated query references missing field', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -334,7 +367,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -359,7 +398,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -392,7 +437,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -424,7 +475,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -456,7 +513,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -489,7 +552,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -522,7 +591,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -560,7 +635,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 2 documents - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -604,8 +685,21 @@ describe('LunrSearchEngine', () => { ]; // Mock 2 indices with 1 document each - await testLunrSearchEngine.index('test-index', mockDocuments); - await testLunrSearchEngine.index('test-index-2', mockDocuments2); + const indexer1 = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + const indexer2 = await getActualIndexer( + testLunrSearchEngine, + 'test-index-2', + ); + await TestPipeline.withSubject(indexer1) + .withDocuments(mockDocuments) + .execute(); + await TestPipeline.withSubject(indexer2) + .withDocuments(mockDocuments2) + .execute(); + // Perform search query scoped to "test-index-2" with a filter on the field "extraField" const mockedSearchResult = await testLunrSearchEngine.query({ term: 'testTitle', @@ -642,7 +736,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 2 documents - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -695,8 +795,20 @@ describe('LunrSearchEngine', () => { ]; // Mock 2 indices with 2 documents each - await testLunrSearchEngine.index('test-index', mockDocuments); - await testLunrSearchEngine.index('test-index-2', mockDocuments2); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); + const indexer2 = await getActualIndexer( + testLunrSearchEngine, + 'test-index-2', + ); + await TestPipeline.withSubject(indexer2) + .withDocuments(mockDocuments2) + .execute(); // Perform search query scoped to "test-index-2" const mockedSearchResult = await testLunrSearchEngine.query({ @@ -734,7 +846,13 @@ describe('LunrSearchEngine', () => { location: `test/location/${i}`, })); - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); const mockedSearchResult = await testLunrSearchEngine.query({ term: 'testTitle', @@ -767,7 +885,10 @@ describe('LunrSearchEngine', () => { location: `test/location/${i}`, })); - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer(testLunrSearchEngine, 'test-index'); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); const mockedSearchResult = await testLunrSearchEngine.query({ term: 'testTitle', @@ -793,22 +914,46 @@ describe('LunrSearchEngine', () => { }); describe('index', () => { - it('should index document', async () => { - const indexSpy = jest.spyOn(testLunrSearchEngine, 'index'); - const mockDocuments = [ - { - title: 'testTerm', - text: 'testText', - location: 'test/location', - }, - ]; + it('should get indexer', async () => { + const indexer = await testLunrSearchEngine.getIndexer('test-index'); + expect(LunrSearchEngineIndexer).toHaveBeenCalled(); + expect(indexer.on).toHaveBeenCalledWith('close', expect.any(Function)); + }); - // call index func and ensure the index func was invoked. - await testLunrSearchEngine.index('test-index', mockDocuments); - expect(indexSpy).toHaveBeenCalled(); - expect(indexSpy).toHaveBeenCalledWith('test-index', [ - { title: 'testTerm', text: 'testText', location: 'test/location' }, - ]); + it('should manage indices and docs on close', async () => { + const doc = { title: 'A doc', text: 'test', location: 'some-location' }; + + // Set up an inspectable search engine to pre-set some data. + const inspectableSearchEngine = new LunrSearchEngineForTests({ + logger: getVoidLogger(), + }); + inspectableSearchEngine.setDocStore({ 'existing-location': doc }); + + // Mock methds called by close handler. + indexerMock.buildIndex.mockReturnValueOnce('expected-index'); + indexerMock.getDocumentStore.mockReturnValueOnce({ + 'new-location': doc, + }); + + // Get the indexer and invoke its close handler. + await inspectableSearchEngine.getIndexer('test-index'); + const onClose = indexerMock.on.mock.calls[0][1] as Function; + onClose(); + + // Ensure mocked methods were called. + expect(indexerMock.buildIndex).toHaveBeenCalled(); + expect(indexerMock.getDocumentStore).toHaveBeenCalled(); + + // Ensure the lunr index was written to the search engine. + expect(inspectableSearchEngine.getLunrIndices()).toStrictEqual({ + 'test-index': 'expected-index', + }); + + // Ensure documents are merged into the existing store. + expect(inspectableSearchEngine.getDocStore()).toStrictEqual({ + 'existing-location': doc, + 'new-location': doc, + }); }); }); }); diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts index ea51ffe8e1..b2e131e56f 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts @@ -23,6 +23,7 @@ import { } from '@backstage/search-common'; import lunr from 'lunr'; import { Logger } from 'winston'; +import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; export type ConcreteLunrQuery = { lunrQueryBuilder: lunr.Index.QueryBuilder; @@ -124,30 +125,17 @@ export class LunrSearchEngine implements SearchEngine { this.translator = translator; } - async index(type: string, documents: IndexableDocument[]): Promise { - const lunrBuilder = new lunr.Builder(); + async getIndexer(type: string) { + const indexer = new LunrSearchEngineIndexer(); - lunrBuilder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer); - lunrBuilder.searchPipeline.add(lunr.stemmer); - - // Make this lunr index aware of all relevant fields. - Object.keys(documents[0]).forEach(field => { - lunrBuilder.field(field); + indexer.on('close', () => { + // Once the stream is closed, build the index and store the documents in + // memory for later retrieval. + this.lunrIndices[type] = indexer.buildIndex(); + this.docStore = { ...this.docStore, ...indexer.getDocumentStore() }; }); - // Set "location" field as reference field - lunrBuilder.ref('location'); - - documents.forEach((document: IndexableDocument) => { - // Add document to Lunar index - lunrBuilder.add(document); - // Store documents in memory to be able to look up document using the ref during query time - // This is not how you should implement your SearchEngine implementation! Do not copy! - this.docStore[document.location] = document; - }); - - // "Rotate" the index by simply overwriting any existing index of the same name. - this.lunrIndices[type] = lunrBuilder.build(); + return indexer; } async query(query: SearchQuery): Promise { diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..fbb6b153d0 --- /dev/null +++ b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import lunr from 'lunr'; +import { range } from 'lodash'; +import { TestPipeline } from '../test-utils'; +import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; + +const lunrBuilderAddSpy = jest.fn(); +const lunrBuilderRefSpy = jest.fn(); +const lunrBuilderFieldSpy = jest.fn(); +const lunrBuilderPipelineAddSpy = jest.fn(); +const lunrBuilderSearchPipelineAddSpy = jest.fn(); + +jest.mock('lunr', () => { + const actualLunr = jest.requireActual('lunr'); + return { + ...actualLunr, + Builder: jest.fn().mockImplementation(() => { + const actualBuilder = new actualLunr.Builder(); + actualBuilder.add = lunrBuilderAddSpy; + actualBuilder.ref = lunrBuilderRefSpy; + actualBuilder.field = lunrBuilderFieldSpy; + actualBuilder.pipeline.add = lunrBuilderPipelineAddSpy; + actualBuilder.searchPipeline.add = lunrBuilderSearchPipelineAddSpy; + return actualBuilder; + }), + }; +}); + +describe('LunrSearchEngineIndexer', () => { + let indexer: LunrSearchEngineIndexer; + + beforeEach(() => { + jest.clearAllMocks(); + indexer = new LunrSearchEngineIndexer(); + }); + + it('should index documents', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + expect(lunrBuilderAddSpy).toHaveBeenCalledWith(documents[0]); + }); + + it('should index documents in bulk', async () => { + const documents = range(350).map(i => ({ + title: `Hello World ${i}`, + text: 'Lorem Ipsum', + location: `location-${i}`, + })); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + expect(lunrBuilderAddSpy).toHaveBeenCalledTimes(350); + }); + + it('should initialize schema', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + extra: 'field', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Builder ref should be set to location (and only once). + expect(lunrBuilderRefSpy).toHaveBeenCalledTimes(1); + expect(lunrBuilderRefSpy).toHaveBeenLastCalledWith('location'); + + // Builder fields should be based on document fields. + expect(lunrBuilderFieldSpy).toHaveBeenCalledTimes(4); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('title'); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('text'); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('location'); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('extra'); + }); + + it('should configure lunr pipeline', async () => { + expect(lunrBuilderSearchPipelineAddSpy).toHaveBeenLastCalledWith( + lunr.stemmer, + ); + expect(lunrBuilderPipelineAddSpy).toHaveBeenCalledWith( + ...[lunr.trimmer, lunr.stopWordFilter, lunr.stemmer], + ); + }); +}); diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts new file mode 100644 index 0000000000..2454889745 --- /dev/null +++ b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts @@ -0,0 +1,68 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import lunr from 'lunr'; +import { BatchSearchEngineIndexer } from '../indexing'; + +export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { + private schemaInitialized = false; + private builder: lunr.Builder; + private docStore: Record = {}; + + constructor() { + super({ batchSize: 100 }); + + this.builder = new lunr.Builder(); + this.builder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer); + this.builder.searchPipeline.add(lunr.stemmer); + } + + // No async initialization required. + async initialize(): Promise {} + async finalize(): Promise {} + + async index(documents: IndexableDocument[]): Promise { + if (!this.schemaInitialized) { + // Make this lunr index aware of all relevant fields. + Object.keys(documents[0]).forEach(field => { + this.builder.field(field); + }); + + // Set "location" field as reference field + this.builder.ref('location'); + + this.schemaInitialized = true; + } + + documents.forEach(document => { + // Add document to Lunar index + this.builder.add(document); + + // Store documents in memory to be able to look up document using the ref during query time + // This is not how you should implement your SearchEngine implementation! Do not copy! + this.docStore[document.location] = document; + }); + } + + buildIndex() { + return this.builder.build(); + } + + getDocumentStore() { + return this.docStore; + } +} diff --git a/plugins/search-backend-node/src/engines/index.ts b/plugins/search-backend-node/src/engines/index.ts index 7e6fb86bd4..7b71873c64 100644 --- a/plugins/search-backend-node/src/engines/index.ts +++ b/plugins/search-backend-node/src/engines/index.ts @@ -16,3 +16,4 @@ export { LunrSearchEngine } from './LunrSearchEngine'; export type { ConcreteLunrQuery } from './LunrSearchEngine'; +export type { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; From a151cf2a886a9b2e72eca2b719c3bb66827799a9 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 18:25:34 +0100 Subject: [PATCH 108/150] Finalize exports for search-backend-node module Signed-off-by: Eric Peterson --- plugins/search-backend-node/api-report.md | 78 +++++++++++++++++++++-- plugins/search-backend-node/package.json | 6 +- plugins/search-backend-node/src/index.ts | 3 + 3 files changed, 80 insertions(+), 7 deletions(-) diff --git a/plugins/search-backend-node/api-report.md b/plugins/search-backend-node/api-report.md index 1929009760..d4b94a932e 100644 --- a/plugins/search-backend-node/api-report.md +++ b/plugins/search-backend-node/api-report.md @@ -3,16 +3,50 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { DocumentCollator } from '@backstage/search-common'; -import { DocumentDecorator } from '@backstage/search-common'; +/// + +import { DocumentCollatorFactory } from '@backstage/search-common'; +import { DocumentDecoratorFactory } from '@backstage/search-common'; import { DocumentTypeInfo } from '@backstage/search-common'; import { IndexableDocument } from '@backstage/search-common'; import { Logger as Logger_2 } from 'winston'; import { default as lunr_2 } from 'lunr'; import { QueryTranslator } from '@backstage/search-common'; +import { Readable } from 'stream'; import { SearchEngine } from '@backstage/search-common'; import { SearchQuery } from '@backstage/search-common'; import { SearchResultSet } from '@backstage/search-common'; +import { Transform } from 'stream'; +import { Writable } from 'stream'; + +// Warning: (ae-missing-release-tag) "BatchSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export abstract class BatchSearchEngineIndexer extends Writable { + constructor(options: BatchSearchEngineOptions); + abstract finalize(): Promise; + abstract index(documents: IndexableDocument[]): Promise; + abstract initialize(): Promise; +} + +// Warning: (ae-missing-release-tag) "BatchSearchEngineOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type BatchSearchEngineOptions = { + batchSize: number; +}; + +// Warning: (ae-missing-release-tag) "DecoratorBase" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export abstract class DecoratorBase extends Transform { + constructor(); + abstract decorate( + document: IndexableDocument, + ): Promise; + abstract finalize(): Promise; + abstract initialize(): Promise; +} // Warning: (ae-missing-release-tag) "IndexBuilder" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -22,11 +56,11 @@ export class IndexBuilder { constructor({ logger, searchEngine }: IndexBuilderOptions); // Warning: (ae-forgotten-export) The symbol "RegisterCollatorParameters" needs to be exported by the entry point index.d.ts addCollator({ - collator, + factory, defaultRefreshIntervalSeconds, }: RegisterCollatorParameters): void; // Warning: (ae-forgotten-export) The symbol "RegisterDecoratorParameters" needs to be exported by the entry point index.d.ts - addDecorator({ decorator }: RegisterDecoratorParameters): void; + addDecorator({ factory }: RegisterDecoratorParameters): void; build(): Promise<{ scheduler: Scheduler; }>; @@ -44,7 +78,7 @@ export class LunrSearchEngine implements SearchEngine { // (undocumented) protected docStore: Record; // (undocumented) - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; // (undocumented) protected logger: Logger_2; // (undocumented) @@ -59,6 +93,23 @@ export class LunrSearchEngine implements SearchEngine { protected translator: QueryTranslator; } +// Warning: (ae-missing-release-tag) "LunrSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor(); + // (undocumented) + buildIndex(): lunr_2.Index; + // (undocumented) + finalize(): Promise; + // (undocumented) + getDocumentStore(): Record; + // (undocumented) + index(documents: IndexableDocument[]): Promise; + // (undocumented) + initialize(): Promise; +} + // Warning: (ae-missing-release-tag) "Scheduler" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -70,4 +121,21 @@ export class Scheduler { } export { SearchEngine }; + +// Warning: (ae-missing-release-tag) "TestPipeline" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export class TestPipeline { + execute(): Promise; + withDocuments(documents: IndexableDocument[]): TestPipeline; + static withSubject(subject: Readable | Transform | Writable): TestPipeline; +} + +// Warning: (ae-missing-release-tag) "TestPipelineResult" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type TestPipelineResult = { + error: unknown; + documents: IndexableDocument[]; +}; ``` diff --git a/plugins/search-backend-node/package.json b/plugins/search-backend-node/package.json index dd8ef5599a..2c644e357d 100644 --- a/plugins/search-backend-node/package.json +++ b/plugins/search-backend-node/package.json @@ -23,10 +23,12 @@ "clean": "backstage-cli package clean" }, "dependencies": { + "@backstage/errors": "^0.2.2", "@backstage/search-common": "^0.2.4", - "winston": "^3.2.1", + "@types/lunr": "^2.3.3", + "lodash": "^4.17.21", "lunr": "^2.3.9", - "@types/lunr": "^2.3.3" + "winston": "^3.2.1" }, "devDependencies": { "@backstage/backend-common": "^0.11.0", diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index d49ef195f2..6ae716553c 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -23,6 +23,9 @@ export { IndexBuilder } from './IndexBuilder'; export { Scheduler } from './Scheduler'; export { LunrSearchEngine } from './engines'; +export type { LunrSearchEngineIndexer } from './engines'; +export * from './indexing'; +export * from './test-utils'; /** * @deprecated Import from @backstage/search-common instead From 0d995e0ddd165f99a29649b98fe86019da597850 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 18:50:50 +0100 Subject: [PATCH 109/150] Update PGSearchEngine to be stream-based Signed-off-by: Eric Peterson --- .../search-backend-module-pg/api-report.md | 29 +++- .../src/PgSearchEngine/PgSearchEngine.test.ts | 58 +++---- .../src/PgSearchEngine/PgSearchEngine.ts | 23 +-- .../PgSearchEngineIndexer.test.ts | 150 ++++++++++++++++++ .../PgSearchEngine/PgSearchEngineIndexer.ts | 74 +++++++++ .../src/PgSearchEngine/index.ts | 4 + .../src/database/DatabaseDocumentStore.ts | 4 + .../src/database/types.ts | 1 + 8 files changed, 288 insertions(+), 55 deletions(-) create mode 100644 plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts create mode 100644 plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts diff --git a/plugins/search-backend-module-pg/api-report.md b/plugins/search-backend-module-pg/api-report.md index e2edfc291a..ba8fde5f17 100644 --- a/plugins/search-backend-module-pg/api-report.md +++ b/plugins/search-backend-module-pg/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; import { IndexableDocument } from '@backstage/search-common'; import { Knex } from 'knex'; import { PluginDatabaseManager } from '@backstage/backend-common'; @@ -28,6 +29,8 @@ export class DatabaseDocumentStore implements DatabaseStore { // (undocumented) static create(knex: Knex): Promise; // (undocumented) + getTransaction(): Promise; + // (undocumented) insertDocuments( tx: Knex.Transaction, type: string, @@ -55,6 +58,8 @@ export interface DatabaseStore { // (undocumented) completeInsert(tx: Knex.Transaction, type: string): Promise; // (undocumented) + getTransaction(): Promise; + // (undocumented) insertDocuments( tx: Knex.Transaction, type: string, @@ -81,7 +86,7 @@ export class PgSearchEngine implements SearchEngine { database: PluginDatabaseManager; }): Promise; // (undocumented) - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; // (undocumented) query(query: SearchQuery): Promise; // (undocumented) @@ -94,6 +99,28 @@ export class PgSearchEngine implements SearchEngine { translator(query: SearchQuery): ConcretePgSearchQuery; } +// Warning: (ae-missing-release-tag) "PgSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor(options: PgSearchEngineIndexerOptions); + // (undocumented) + finalize(): Promise; + // (undocumented) + index(documents: IndexableDocument[]): Promise; + // (undocumented) + initialize(): Promise; +} + +// Warning: (ae-missing-release-tag) "PgSearchEngineIndexerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type PgSearchEngineIndexerOptions = { + batchSize: number; + type: string; + databaseStore: DatabaseStore; +}; + // Warning: (ae-missing-release-tag) "PgSearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 56a3a6e643..8618bed4c8 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { range } from 'lodash'; import { DatabaseStore } from '../database'; import { ConcretePgSearchQuery, @@ -21,6 +20,13 @@ import { encodePageCursor, PgSearchEngine, } from './PgSearchEngine'; +import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; + +jest.mock('./PgSearchEngineIndexer', () => ({ + PgSearchEngineIndexer: jest + .fn() + .mockImplementation(async () => 'the-expected-indexer'), +})); describe('PgSearchEngine', () => { const tx: any = {} as any; @@ -30,6 +36,7 @@ describe('PgSearchEngine', () => { beforeEach(() => { database = { transaction: jest.fn(), + getTransaction: jest.fn(), insertDocuments: jest.fn(), query: jest.fn(), completeInsert: jest.fn(), @@ -122,46 +129,21 @@ describe('PgSearchEngine', () => { }); }); - describe('insert', () => { - it('should insert documents', async () => { - database.transaction.mockImplementation(fn => fn(tx)); + describe('index', () => { + it('should instantiate indexer', async () => { + const indexer = await searchEngine.getIndexer('my-type'); - const documents = [ - { title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' }, - { - location: 'location-2', - text: 'Hello World', - title: 'Dolor sit amet', - }, - ]; - - await searchEngine.index('my-type', documents); - - expect(database.transaction).toHaveBeenCalledTimes(1); - expect(database.prepareInsert).toHaveBeenCalledTimes(1); - expect(database.insertDocuments).toHaveBeenCalledWith( - tx, - 'my-type', - documents, + // Indexer instantiated with expected args. + expect(PgSearchEngineIndexer).toHaveBeenCalledWith( + expect.objectContaining({ + batchSize: 100, + type: 'my-type', + databaseStore: database, + }), ); - expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); - }); - it('should batch insert documents', async () => { - database.transaction.mockImplementation(fn => fn(tx)); - - const documents = range(350).map(i => ({ - title: `Hello World ${i}`, - text: 'Lorem Ipsum', - location: `location-${i}`, - })); - - await searchEngine.index('my-type', documents); - - expect(database.transaction).toHaveBeenCalledTimes(1); - expect(database.prepareInsert).toHaveBeenCalledTimes(1); - expect(database.insertDocuments).toBeCalledTimes(4); - expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + // Indexer is as expected. + expect(indexer).toBe('the-expected-indexer'); }); }); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index 6fd6c571a8..ac42007401 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -15,12 +15,8 @@ */ import { PluginDatabaseManager } from '@backstage/backend-common'; import { SearchEngine } from '@backstage/plugin-search-backend-node'; -import { - IndexableDocument, - SearchQuery, - SearchResultSet, -} from '@backstage/search-common'; -import { chunk } from 'lodash'; +import { SearchQuery, SearchResultSet } from '@backstage/search-common'; +import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; import { DatabaseDocumentStore, DatabaseStore, @@ -77,16 +73,11 @@ export class PgSearchEngine implements SearchEngine { this.translator = translator; } - async index(type: string, documents: IndexableDocument[]): Promise { - await this.databaseStore.transaction(async tx => { - await this.databaseStore.prepareInsert(tx); - - const batchSize = 100; - for (const documentBatch of chunk(documents, batchSize)) { - await this.databaseStore.insertDocuments(tx, type, documentBatch); - } - - await this.databaseStore.completeInsert(tx, type); + async getIndexer(type: string) { + return new PgSearchEngineIndexer({ + batchSize: 100, + type, + databaseStore: this.databaseStore, }); } diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..1fc7e74fc4 --- /dev/null +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { range } from 'lodash'; +import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; +import { DatabaseStore } from '../database'; + +describe('PgSearchEngineIndexer', () => { + const tx = { + rollback: jest.fn(), + commit: jest.fn(), + } as any; + let database: jest.Mocked; + let indexer: PgSearchEngineIndexer; + + beforeEach(() => { + jest.clearAllMocks(); + database = { + transaction: jest.fn().mockImplementation(fn => fn(tx)), + getTransaction: jest.fn().mockReturnValue(tx), + insertDocuments: jest.fn(), + query: jest.fn(), + completeInsert: jest.fn(), + prepareInsert: jest.fn(), + }; + indexer = new PgSearchEngineIndexer({ + batchSize: 100, + type: 'my-type', + databaseStore: database, + }); + }); + + it('should insert documents', async () => { + const documents = [ + { title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' }, + { + location: 'location-2', + text: 'Hello World', + title: 'Dolor sit amet', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toHaveBeenCalledWith( + tx, + 'my-type', + documents, + ); + expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + expect(tx.commit).toHaveBeenCalled(); + }); + + it('should batch insert documents', async () => { + const documents = range(350).map(i => ({ + title: `Hello World ${i}`, + text: 'Lorem Ipsum', + location: `location-${i}`, + })); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toBeCalledTimes(4); + expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + }); + + it('should close out stream and bubble up error on prepare', async () => { + const expectedError = new Error('Prepare error'); + const documents = [ + { + title: `Hello World`, + text: 'Lorem Ipsum', + location: `location`, + }, + ]; + + database.prepareInsert.mockRejectedValueOnce(expectedError); + const result = await TestPipeline.withSubject(indexer) + .withDocuments(documents) + .execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).not.toHaveBeenCalled(); + expect(database.completeInsert).not.toHaveBeenCalled(); + expect(result.error).toBe(expectedError); + expect(tx.rollback).toHaveBeenCalledWith(expectedError); + }); + + it('should close tx and bubble up error on insert', async () => { + const expectedError = new Error('Index error'); + const documents = [ + { + title: `Hello World`, + text: 'Lorem Ipsum', + location: `location`, + }, + ]; + + database.insertDocuments.mockRejectedValueOnce(expectedError); + const result = await TestPipeline.withSubject(indexer) + .withDocuments(documents) + .execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.completeInsert).not.toHaveBeenCalled(); + expect(result.error).toBe(expectedError); + expect(tx.rollback).toHaveBeenCalledWith(expectedError); + }); + + it('should close tx and bubble up error on completion', async () => { + const expectedError = new Error('Completion error'); + const documents = [ + { + title: `Hello World`, + text: 'Lorem Ipsum', + location: `location`, + }, + ]; + + database.completeInsert.mockRejectedValueOnce(expectedError); + const result = await TestPipeline.withSubject(indexer) + .withDocuments(documents) + .execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toHaveBeenCalledTimes(1); + expect(database.completeInsert).toHaveBeenCalledTimes(1); + expect(result.error).toBe(expectedError); + expect(tx.rollback).toHaveBeenCalledWith(expectedError); + }); +}); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts new file mode 100644 index 0000000000..53d040cb87 --- /dev/null +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { IndexableDocument } from '@backstage/search-common'; +import { Knex } from 'knex'; +import { DatabaseStore } from '../database'; + +export type PgSearchEngineIndexerOptions = { + batchSize: number; + type: string; + databaseStore: DatabaseStore; +}; + +export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { + private store: DatabaseStore; + private type: string; + private tx: Knex.Transaction | undefined; + + constructor(options: PgSearchEngineIndexerOptions) { + super({ batchSize: options.batchSize }); + this.store = options.databaseStore; + this.type = options.type; + } + + async initialize(): Promise { + this.tx = await this.store.getTransaction(); + try { + await this.store.prepareInsert(this.tx); + } catch (e) { + // In case of error, rollback the transaction and re-throw the error so + // that the stream can be closed and destroyed properly. + this.tx.rollback(e); + throw e; + } + } + + async index(documents: IndexableDocument[]): Promise { + try { + await this.store.insertDocuments(this.tx!, this.type, documents); + } catch (e) { + // In case of error, rollback the transaction and re-throw the error so + // that the stream can be closed and destroyed properly. + this.tx!.rollback(e); + throw e; + } + } + + async finalize(): Promise { + // Attempt to complete and commit the transaction. + try { + await this.store.completeInsert(this.tx!, this.type); + this.tx!.commit(); + } catch (e) { + // Otherwise, rollback the transaction and re-throw the error so that the + // stream can be closed and destroyed properly. + this.tx!.rollback!(e); + throw e; + } + } +} diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts index 7994998baf..7f8e297648 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts @@ -15,3 +15,7 @@ */ export { PgSearchEngine } from './PgSearchEngine'; export type { ConcretePgSearchQuery } from './PgSearchEngine'; +export type { + PgSearchEngineIndexer, + PgSearchEngineIndexerOptions, +} from './PgSearchEngineIndexer'; diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index 8c180ea019..0d3ee63ca4 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -71,6 +71,10 @@ export class DatabaseDocumentStore implements DatabaseStore { return await this.db.transaction(fn); } + async getTransaction(): Promise { + return this.db.transaction(); + } + async prepareInsert(tx: Knex.Transaction): Promise { // We create a temporary table to collect the hashes of the documents that // we expect to be in the documents table at the end. The table is deleted diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts index 0c0596160e..0a0dc35682 100644 --- a/plugins/search-backend-module-pg/src/database/types.ts +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -26,6 +26,7 @@ export interface PgSearchQuery { export interface DatabaseStore { transaction(fn: (tx: Knex.Transaction) => Promise): Promise; + getTransaction(): Promise; prepareInsert(tx: Knex.Transaction): Promise; insertDocuments( tx: Knex.Transaction, From 2c171166a07663f9572b7e3914d2dc4a121df10e Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 19:03:35 +0100 Subject: [PATCH 110/150] Update ElasticSearchSearchEngine to be stream-based Signed-off-by: Eric Peterson --- .../api-report.md | 31 ++- .../package.json | 1 + .../engines/ElasticSearchSearchEngine.test.ts | 91 ++++++-- .../src/engines/ElasticSearchSearchEngine.ts | 94 +++----- .../ElasticSearchSearchEngineIndexer.test.ts | 211 ++++++++++++++++++ .../ElasticSearchSearchEngineIndexer.ts | 176 +++++++++++++++ .../src/engines/index.ts | 4 + .../src/index.ts | 6 +- 8 files changed, 527 insertions(+), 87 deletions(-) create mode 100644 plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts create mode 100644 plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index 6dc0256586..d4edcc6b25 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -5,6 +5,8 @@ ```ts /// +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { Client } from '@elastic/elasticsearch'; import { Config } from '@backstage/config'; import type { ConnectionOptions } from 'tls'; import { IndexableDocument } from '@backstage/search-common'; @@ -114,7 +116,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { indexPrefix, }: ElasticSearchOptions): Promise; // (undocumented) - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; newClient(create: (options: ElasticSearchClientOptions) => T): T; // (undocumented) query(query: SearchQuery): Promise; @@ -127,4 +129,31 @@ export class ElasticSearchSearchEngine implements SearchEngine { // (undocumented) protected translator(query: SearchQuery): ConcreteElasticSearchQuery; } + +// Warning: (ae-missing-release-tag) "ElasticSearchSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor(options: ElasticSearchSearchEngineIndexerOptions); + // (undocumented) + finalize(): Promise; + // (undocumented) + index(documents: IndexableDocument[]): Promise; + // (undocumented) + readonly indexName: string; + // (undocumented) + initialize(): Promise; +} + +// Warning: (ae-missing-release-tag) "ElasticSearchSearchEngineIndexerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type ElasticSearchSearchEngineIndexerOptions = { + type: string; + indexPrefix: string; + indexSeparator: string; + alias: string; + logger: Logger_2; + elasticSearchClient: Client; +}; ``` diff --git a/plugins/search-backend-module-elasticsearch/package.json b/plugins/search-backend-module-elasticsearch/package.json index 2a2dbd6f85..0a279d6b4c 100644 --- a/plugins/search-backend-module-elasticsearch/package.json +++ b/plugins/search-backend-module-elasticsearch/package.json @@ -25,6 +25,7 @@ "dependencies": { "@backstage/config": "^0.1.15", "@backstage/search-common": "^0.2.4", + "@backstage/plugin-search-backend-node": "^0.4.7", "@elastic/elasticsearch": "7.13.0", "@acuris/aws-es-connection": "^2.2.0", "aws-sdk": "^2.948.0", diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts index 1566a35b57..467e1b4e77 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -15,7 +15,8 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { Client } from '@elastic/elasticsearch'; +import { ConfigReader } from '@backstage/config'; +import { Client, errors } from '@elastic/elasticsearch'; import Mock from '@elastic/elasticsearch-mock'; import { ConcreteElasticSearchQuery, @@ -23,7 +24,7 @@ import { ElasticSearchSearchEngine, encodePageCursor, } from './ElasticSearchSearchEngine'; -import { ConfigReader } from '@backstage/config'; +import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; class ElasticSearchSearchEngineForTranslatorTests extends ElasticSearchSearchEngine { getTranslator() { @@ -37,6 +38,16 @@ const options = { Connection: mock.getConnection(), }; +const indexerMock = { + on: jest.fn(), + indexName: 'expected-index-name', +}; +jest.mock('./ElasticSearchSearchEngineIndexer', () => ({ + ElasticSearchSearchEngineIndexer: jest + .fn() + .mockImplementation(() => indexerMock), +})); + describe('ElasticSearchSearchEngine', () => { let testSearchEngine: ElasticSearchSearchEngine; let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests; @@ -542,23 +553,67 @@ describe('ElasticSearchSearchEngine', () => { }); }); - describe('index', () => { - it('should index document', async () => { - const indexSpy = jest.spyOn(testSearchEngine, 'index'); - const mockDocuments = [ - { - title: 'testTerm', - text: 'testText', - location: 'test/location', - }, - ]; + describe('indexer', () => { + it('should get indexer', async () => { + const indexer = await testSearchEngine.getIndexer('test-index'); - // call index func and ensure the index func was invoked. - await testSearchEngine.index('test-index', mockDocuments); - expect(indexSpy).toHaveBeenCalled(); - expect(indexSpy).toHaveBeenCalledWith('test-index', [ - { title: 'testTerm', text: 'testText', location: 'test/location' }, - ]); + expect(indexer).toStrictEqual(indexerMock); + expect(ElasticSearchSearchEngineIndexer).toHaveBeenCalledWith( + expect.objectContaining({ + alias: 'test-index__search', + type: 'test-index', + indexPrefix: '', + indexSeparator: '-index__', + elasticSearchClient: client, + }), + ); + expect(indexerMock.on).toHaveBeenCalledWith( + 'error', + expect.any(Function), + ); + }); + + describe('onError', () => { + let errorHandler: Function; + const error = new Error('some error'); + + beforeEach(async () => { + mock.clearAll(); + await testSearchEngine.getIndexer('test-index'); + errorHandler = indexerMock.on.mock.calls[0][1]; + }); + + it('should check for and delete expected index', async () => { + const existsSpy = jest.fn().mockReturnValue('truthy value'); + const deleteSpy = jest.fn().mockReturnValue({}); + mock.add({ method: 'HEAD', path: '/expected-index-name' }, existsSpy); + mock.add({ method: 'DELETE', path: '/expected-index-name' }, deleteSpy); + + await errorHandler(error); + + // Check and delete HTTP requests were made. + expect(existsSpy).toHaveBeenCalled(); + expect(deleteSpy).toHaveBeenCalled(); + }); + + it('should not delete index if none exists', async () => { + // Exists call returns 404 on no index. + const existsSpy = jest.fn().mockReturnValue( + new errors.ResponseError({ + statusCode: 404, + body: { status: 404 }, + } as unknown as any), + ); + const deleteSpy = jest.fn().mockReturnValue({}); + mock.add({ method: 'HEAD', path: '/expected-index-name' }, existsSpy); + mock.add({ method: 'DELETE', path: '/expected-index-name' }, deleteSpy); + + await errorHandler(error); + + // Check request was made, but no delete request was made. + expect(existsSpy).toHaveBeenCalled(); + expect(deleteSpy).not.toHaveBeenCalled(); + }); }); }); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 48caf0d4f1..c16567fee0 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -29,8 +29,8 @@ import { Client } from '@elastic/elasticsearch'; import esb from 'elastic-builder'; import { isEmpty, isNaN as nan, isNumber } from 'lodash'; import { Logger } from 'winston'; - import type { ElasticSearchClientOptions } from './ElasticSearchClientOptions'; +import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; export type { ElasticSearchClientOptions }; @@ -58,12 +58,6 @@ type ElasticSearchResult = { _source: IndexableDocument; }; -function duration(startTimestamp: [number, number]): string { - const delta = process.hrtime(startTimestamp); - const seconds = delta[0] + delta[1] / 1e9; - return `${seconds.toFixed(1)}s`; -} - function isBlank(str: string) { return (isEmpty(str) && !isNumber(str)) || nan(str); } @@ -165,67 +159,37 @@ export class ElasticSearchSearchEngine implements SearchEngine { this.translator = translator; } - async index(type: string, documents: IndexableDocument[]): Promise { - this.logger.info( - `Started indexing ${documents.length} documents for index ${type}`, - ); - const startTimestamp = process.hrtime(); + async getIndexer(type: string) { const alias = this.constructSearchAlias(type); - const index = this.constructIndexName(type, `${Date.now()}`); - try { - const aliases = await this.elasticSearchClient.cat.aliases({ - format: 'json', - name: alias, - }); - const removableIndices = aliases.body.map( - (r: Record) => r.index, - ); + const indexer = new ElasticSearchSearchEngineIndexer({ + type, + indexPrefix: this.indexPrefix, + indexSeparator: this.indexSeparator, + alias, + elasticSearchClient: this.elasticSearchClient, + logger: this.logger, + }); - await this.elasticSearchClient.indices.create({ - index, - }); - const result = await this.elasticSearchClient.helpers.bulk({ - datasource: documents, - onDocument() { - return { - index: { _index: index }, - }; - }, - refreshOnCompletion: index, - }); - - this.logger.info( - `Indexing completed for index ${type} in ${duration(startTimestamp)}`, - result, - ); - await this.elasticSearchClient.indices.updateAliases({ - body: { - actions: [ - { remove: { index: this.constructIndexName(type, '*'), alias } }, - { add: { index, alias } }, - ], - }, - }); - - this.logger.info('Removing stale search indices', removableIndices); - if (removableIndices.length) { - await this.elasticSearchClient.indices.delete({ - index: removableIndices, - }); - } - } catch (e) { + // Attempt cleanup upon failure. + indexer.on('error', async e => { this.logger.error(`Failed to index documents for type ${type}`, e); - const response = await this.elasticSearchClient.indices.exists({ - index, - }); - const indexCreated = response.body; - if (indexCreated) { - this.logger.info(`Removing created index ${index}`); - await this.elasticSearchClient.indices.delete({ - index, + try { + const response = await this.elasticSearchClient.indices.exists({ + index: indexer.indexName, }); + const indexCreated = response.body; + if (indexCreated) { + this.logger.info(`Removing created index ${indexer.indexName}`); + await this.elasticSearchClient.indices.delete({ + index: indexer.indexName, + }); + } + } catch (error) { + this.logger.error(`Unable to clean up elastic index: ${error}`); } - } + }); + + return indexer; } async query(query: SearchQuery): Promise { @@ -268,10 +232,6 @@ export class ElasticSearchSearchEngine implements SearchEngine { private readonly indexSeparator = '-index__'; - private constructIndexName(type: string, postFix: string) { - return `${this.indexPrefix}${type}${this.indexSeparator}${postFix}`; - } - private getTypeFromIndex(index: string) { return index .substring(this.indexPrefix.length) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..0867887d9a --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -0,0 +1,211 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { Client } from '@elastic/elasticsearch'; +import Mock from '@elastic/elasticsearch-mock'; +import { range } from 'lodash'; +import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; + +const mock = new Mock(); +const client = new Client({ + node: 'http://localhost:9200', + Connection: mock.getConnection(), +}); + +describe('ElasticSearchSearchEngineIndexer', () => { + let indexer: ElasticSearchSearchEngineIndexer; + let bulkSpy: jest.Mock; + let catSpy: jest.Mock; + let createSpy: jest.Mock; + let aliasesSpy: jest.Mock; + let deleteSpy: jest.Mock; + + beforeEach(() => { + // Instantiate the indexer to be tested. + indexer = new ElasticSearchSearchEngineIndexer({ + type: 'some-type', + indexPrefix: '', + indexSeparator: '-index__', + alias: 'some-type-index__search', + logger: getVoidLogger(), + elasticSearchClient: client, + }); + + // Set up all requisite Elastic mocks. + mock.clearAll(); + bulkSpy = jest.fn().mockReturnValue({ took: 9, errors: false, items: [] }); + mock.add( + { + method: 'POST', + path: '/_bulk', + }, + bulkSpy, + ); + mock.add( + { + method: 'GET', + path: '/:index/_refresh', + }, + jest.fn().mockReturnValue({}), + ); + + catSpy = jest.fn().mockReturnValue([ + { + alias: 'some-type-index__search', + index: 'some-type-index__123tobedeleted', + filter: '-', + 'routing.index': '-', + 'routing.search': '-', + is_write_index: '-', + }, + ]); + mock.add( + { + method: 'GET', + path: '/_cat/aliases/some-type-index__search', + }, + catSpy, + ); + + createSpy = jest.fn().mockReturnValue({ + acknowledged: true, + shards_acknowledged: true, + index: 'single_index', + }); + mock.add( + { + method: 'PUT', + path: '/:index', + }, + createSpy, + ); + + aliasesSpy = jest.fn().mockReturnValue({}); + mock.add( + { + method: 'POST', + path: '*', + }, + aliasesSpy, + ); + + deleteSpy = jest.fn().mockReturnValue({}); + mock.add( + { + method: 'DELETE', + path: '/some-type-index__123tobedeleted', + }, + deleteSpy, + ); + }); + + it('indexes documents', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + { + title: 'Another test', + text: 'Some more text', + location: 'test/location/2', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Older indices should have been queried for. + expect(catSpy).toHaveBeenCalled(); + + // A new index should have been created. + const createdIndex = createSpy.mock.calls[0][0].path.slice(1); + expect(createdIndex).toContain('some-type-index__'); + + // Bulk helper should have been called with documents. + const bulkBody = bulkSpy.mock.calls[0][0].body; + expect(bulkBody[0]).toStrictEqual({ index: { _index: createdIndex } }); + expect(bulkBody[1]).toStrictEqual(documents[0]); + expect(bulkBody[2]).toStrictEqual({ index: { _index: createdIndex } }); + expect(bulkBody[3]).toStrictEqual(documents[1]); + + // Alias should have been rotated. + expect(aliasesSpy).toHaveBeenCalled(); + const aliasActions = aliasesSpy.mock.calls[0][0].body.actions; + expect(aliasActions[0]).toStrictEqual({ + remove: { index: 'some-type-index__*', alias: 'some-type-index__search' }, + }); + expect(aliasActions[1]).toStrictEqual({ + add: { index: createdIndex, alias: 'some-type-index__search' }, + }); + + // Old index should be cleaned up. + expect(deleteSpy).toHaveBeenCalled(); + }); + + it('handles bulk and batching during indexing', async () => { + const documents = range(550).map(i => ({ + title: `Hello World ${i}`, + location: `location-${i}`, + // Generate large document sizes to trigger ES bulk flushing. + text: range(2000).join(', '), + })); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Ensure multiple bulk requests were made. + expect(bulkSpy).toHaveBeenCalledTimes(2); + + // Ensure the first and last documents were included in the payloads. + const docLocations: string[] = [ + ...bulkSpy.mock.calls[0][0].body.map((l: any) => l.location), + ...bulkSpy.mock.calls[1][0].body.map((l: any) => l.location), + ]; + expect(docLocations).toContain('location-0'); + expect(docLocations).toContain('location-549'); + }); + + it('ignores cleanup when no existing indices exist', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + ]; + + // Update initial alias cat to return nothing. + catSpy = jest.fn().mockReturnValue([]); + mock.clear({ + method: 'GET', + path: '/_cat/aliases/some-type-index__search', + }); + mock.add( + { + method: 'GET', + path: '/_cat/aliases/some-type-index__search', + }, + catSpy, + ); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Final deletion shouldn't be called. + expect(deleteSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts new file mode 100644 index 0000000000..2e4996cd2f --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -0,0 +1,176 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { IndexableDocument } from '@backstage/search-common'; +import { Client } from '@elastic/elasticsearch'; +import { Readable } from 'stream'; +import { Logger } from 'winston'; + +export type ElasticSearchSearchEngineIndexerOptions = { + type: string; + indexPrefix: string; + indexSeparator: string; + alias: string; + logger: Logger; + elasticSearchClient: Client; +}; + +function duration(startTimestamp: [number, number]): string { + const delta = process.hrtime(startTimestamp); + const seconds = delta[0] + delta[1] / 1e9; + return `${seconds.toFixed(1)}s`; +} + +export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { + private received: number = 0; + private processed: number = 0; + private removableIndices: string[] = []; + + private readonly startTimestamp: [number, number]; + private readonly type: string; + public readonly indexName: string; + private readonly indexPrefix: string; + private readonly indexSeparator: string; + private readonly alias: string; + private readonly logger: Logger; + private readonly sourceStream: Readable; + private readonly elasticSearchClient: Client; + private bulkResult: Promise; + + constructor(options: ElasticSearchSearchEngineIndexerOptions) { + super({ batchSize: 100 }); + this.logger = options.logger; + this.startTimestamp = process.hrtime(); + this.type = options.type; + this.indexPrefix = options.indexPrefix; + this.indexSeparator = options.indexSeparator; + this.indexName = this.constructIndexName(`${Date.now()}`); + this.alias = options.alias; + this.elasticSearchClient = options.elasticSearchClient; + + // The ES client bulk helper supports stream-based indexing, but we have to + // supply the stream directly to it at instantiation-time. We can't supply + // this class itself, so instead, we create this inline stream instead. + this.sourceStream = new Readable({ objectMode: true }); + this.sourceStream._read = () => {}; + + // eslint-disable-next-line consistent-this + const that = this; + + // Keep a reference to the ES Bulk helper so that we can know when all + // documents have been successfully written to ES. + this.bulkResult = this.elasticSearchClient.helpers.bulk({ + datasource: this.sourceStream, + onDocument() { + that.processed++; + return { + index: { _index: that.indexName }, + }; + }, + refreshOnCompletion: that.indexName, + }); + } + + async initialize(): Promise { + this.logger.info(`Started indexing documents for index ${this.type}`); + + const aliases = await this.elasticSearchClient.cat.aliases({ + format: 'json', + name: this.alias, + }); + + this.removableIndices = aliases.body.map( + (r: Record) => r.index, + ); + + await this.elasticSearchClient.indices.create({ + index: this.indexName, + }); + } + + async index(documents: IndexableDocument[]): Promise { + await this.isReady(); + documents.forEach(document => { + this.received++; + this.sourceStream.push(document); + }); + } + + async finalize(): Promise { + // Wait for all documents to be processed. + await this.isReady(); + + // Close off the underlying stream connected to ES, indicating that no more + // documents will be written. + this.sourceStream.push(null); + + // Wait for the bulk helper to finish processing. + const result = await this.bulkResult; + + // Rotate aliases upon completion. Allow errors to bubble up so that we can + // clean up the create index. + this.logger.info( + `Indexing completed for index ${this.type} in ${duration( + this.startTimestamp, + )}`, + result, + ); + await this.elasticSearchClient.indices.updateAliases({ + body: { + actions: [ + { + remove: { index: this.constructIndexName('*'), alias: this.alias }, + }, + { add: { index: this.indexName, alias: this.alias } }, + ], + }, + }); + + // If any indices are removable, remove them. Do not bubble up this error, + // as doing so would delete the now aliased index. Log instead. + if (this.removableIndices.length) { + this.logger.info('Removing stale search indices', this.removableIndices); + try { + await this.elasticSearchClient.indices.delete({ + index: this.removableIndices, + }); + } catch (e) { + this.logger.warn(`Failed to remove stale search indices: ${e}`); + } + } + } + + /** + * Ensures that the number of documents sent over the wire to ES matches the + * number of documents this stream has received so far. This helps manage + * backpressure in other parts of the indexing pipeline. + */ + private isReady(): Promise { + return new Promise(resolve => { + const interval = setInterval(() => { + if (this.received === this.processed) { + clearInterval(interval); + resolve(); + } + }, 50); + }); + } + + private constructIndexName(postFix: string) { + return `${this.indexPrefix}${this.type}${this.indexSeparator}${postFix}`; + } +} diff --git a/plugins/search-backend-module-elasticsearch/src/engines/index.ts b/plugins/search-backend-module-elasticsearch/src/engines/index.ts index d5eee37803..19c24b37ff 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/index.ts @@ -19,3 +19,7 @@ export type { ConcreteElasticSearchQuery, ElasticSearchClientOptions, } from './ElasticSearchSearchEngine'; +export type { + ElasticSearchSearchEngineIndexer, + ElasticSearchSearchEngineIndexerOptions, +} from './ElasticSearchSearchEngineIndexer'; diff --git a/plugins/search-backend-module-elasticsearch/src/index.ts b/plugins/search-backend-module-elasticsearch/src/index.ts index 8cf96de858..223141be6b 100644 --- a/plugins/search-backend-module-elasticsearch/src/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/index.ts @@ -21,4 +21,8 @@ */ export { ElasticSearchSearchEngine } from './engines'; -export type { ElasticSearchClientOptions } from './engines'; +export type { + ElasticSearchClientOptions, + ElasticSearchSearchEngineIndexer, + ElasticSearchSearchEngineIndexerOptions, +} from './engines'; From 45f68efd405d42f6d3dcd02bf634ef99b5acb3b2 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 19:35:34 +0100 Subject: [PATCH 111/150] Update Catalog Collator to be stream-based Signed-off-by: Eric Peterson --- plugins/catalog-backend/api-report.md | 32 ++- plugins/catalog-backend/package.json | 1 + .../src/search/DefaultCatalogCollator.ts | 19 +- .../DefaultCatalogCollatorFactory.test.ts | 213 ++++++++++++++++++ .../search/DefaultCatalogCollatorFactory.ts | 173 ++++++++++++++ plugins/catalog-backend/src/search/index.ts | 8 +- 6 files changed, 430 insertions(+), 16 deletions(-) create mode 100644 plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts create mode 100644 plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 6f0147a1f4..d5541678e6 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -10,7 +10,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { ConditionalPolicyDecision } from '@backstage/plugin-permission-node'; import { Conditions } from '@backstage/plugin-permission-node'; import { Config } from '@backstage/config'; -import { DocumentCollator } from '@backstage/search-common'; +import { DocumentCollatorFactory } from '@backstage/search-common'; import { Entity } from '@backstage/catalog-model'; import { EntityName } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; @@ -30,6 +30,7 @@ import { PermissionCriteria } from '@backstage/plugin-permission-common'; import { PermissionRule } from '@backstage/plugin-permission-node'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Readable } from 'stream'; import { Router } from 'express'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { TokenManager } from '@backstage/backend-common'; @@ -415,8 +416,8 @@ export function createRandomRefreshInterval(options: { // @public export function createRouter(options: RouterOptions): Promise; -// @public (undocumented) -export class DefaultCatalogCollator implements DocumentCollator { +// @public @deprecated (undocumented) +export class DefaultCatalogCollator { constructor(options: { discovery: PluginEndpointDiscovery; tokenManager: TokenManager; @@ -456,6 +457,31 @@ export class DefaultCatalogCollator implements DocumentCollator { readonly visibilityPermission: Permission; } +// @public (undocumented) +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + static fromConfig( + _config: Config, + options: DefaultCatalogCollatorFactoryOptions, + ): DefaultCatalogCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; + // (undocumented) + readonly visibilityPermission: Permission; +} + +// @public (undocumented) +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; +}; + // @public (undocumented) export class DefaultCatalogProcessingOrchestrator implements CatalogProcessingOrchestrator diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index be0772e199..f7d946c531 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -73,6 +73,7 @@ "@backstage/backend-test-utils": "^0.1.19", "@backstage/cli": "^0.14.1", "@backstage/plugin-permission-common": "^0.5.1", + "@backstage/plugin-search-backend-node": "0.4.7", "@backstage/test-utils": "^0.2.6", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts index 96704a6c1a..c59318d0af 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts @@ -23,7 +23,6 @@ import { stringifyEntityRef, UserEntity, } from '@backstage/catalog-model'; -import { IndexableDocument, DocumentCollator } from '@backstage/search-common'; import { Config } from '@backstage/config'; import { CatalogApi, @@ -31,18 +30,14 @@ import { GetEntitiesRequest, } from '@backstage/catalog-client'; import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; +import { CatalogEntityDocument } from './DefaultCatalogCollatorFactory'; -/** @public */ -export interface CatalogEntityDocument extends IndexableDocument { - componentType: string; - namespace: string; - kind: string; - lifecycle: string; - owner: string; -} - -/** @public */ -export class DefaultCatalogCollator implements DocumentCollator { +/** + * @public + * @deprecated Upgrade to a more recent `@backstage/search-backend-node` and + * use `DefaultCatalogCollatorFactory` instead. + */ +export class DefaultCatalogCollator { protected discovery: PluginEndpointDiscovery; protected locationTemplate: string; protected filter?: GetEntitiesRequest['filter']; diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts new file mode 100644 index 0000000000..ccb61b8680 --- /dev/null +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts @@ -0,0 +1,213 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { Readable } from 'stream'; +import { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; + +const server = setupServer(); + +const expectedEntities: Entity[] = [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'test-entity', + description: 'The expected description', + }, + spec: { + type: 'some-type', + lifecycle: 'experimental', + owner: 'someone', + }, + }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + title: 'Test Entity', + name: 'test-entity-2', + description: 'The expected description 2', + }, + spec: { + type: 'some-type', + lifecycle: 'experimental', + owner: 'someone', + }, + }, +]; + +describe('DefaultCatalogCollatorFactory', () => { + const config = new ConfigReader({}); + const mockDiscoveryApi: jest.Mocked = { + getBaseUrl: jest.fn().mockResolvedValue('http://localhost:7007'), + getExternalBaseUrl: jest.fn(), + }; + const mockTokenManager: jest.Mocked = { + getToken: jest.fn().mockResolvedValue({ token: '' }), + authenticate: jest.fn(), + }; + const options = { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + }; + + beforeAll(() => { + server.listen(); + }); + + beforeEach(() => { + server.use( + rest.get('http://localhost:7007/entities', (req, res, ctx) => { + if (req.url.searchParams.has('filter')) { + const filter = req.url.searchParams.get('filter'); + if (filter === 'kind=Foo,kind=Bar') { + // When filtering on the 'Foo,Bar' kinds we simply return no items, to simulate a filter + return res(ctx.json([])); + } + throw new Error('Unexpected filter parameter'); + } + + // Imitate offset/limit pagination. + const offset = parseInt(req.url.searchParams.get('offset') || '0', 10); + const limit = parseInt(req.url.searchParams.get('limit') || '500', 10); + return res(ctx.json(expectedEntities.slice(offset, limit + offset))); + }), + ); + }); + + afterAll(() => { + server.close(); + }); + + afterEach(() => server.resetHandlers()); + + it('has expected type', () => { + const factory = DefaultCatalogCollatorFactory.fromConfig(config, options); + expect(factory.type).toBe('software-catalog'); + }); + + describe('getCollator', () => { + let factory: DefaultCatalogCollatorFactory; + let collator: Readable; + + beforeEach(async () => { + factory = DefaultCatalogCollatorFactory.fromConfig(config, options); + collator = await factory.getCollator(); + }); + + it('returns a readable stream', async () => { + expect(collator).toBeInstanceOf(Readable); + }); + + it('fetches from the configured catalog service', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog'); + expect(documents).toHaveLength(expectedEntities.length); + }); + + it('maps a returned entity to an expected CatalogEntityDocument', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + expect(documents[0]).toMatchObject({ + title: expectedEntities[0].metadata.name, + location: '/catalog/default/component/test-entity', + text: expectedEntities[0].metadata.description, + namespace: 'default', + componentType: expectedEntities[0]!.spec!.type, + lifecycle: expectedEntities[0]!.spec!.lifecycle, + owner: expectedEntities[0]!.spec!.owner, + authorization: { + resourceRef: 'component:default/test-entity', + }, + }); + expect(documents[1]).toMatchObject({ + title: expectedEntities[1].metadata.title, + location: '/catalog/default/component/test-entity-2', + text: expectedEntities[1].metadata.description, + namespace: 'default', + componentType: expectedEntities[1]!.spec!.type, + lifecycle: expectedEntities[1]!.spec!.lifecycle, + owner: expectedEntities[1]!.spec!.owner, + authorization: { + resourceRef: 'component:default/test-entity-2', + }, + }); + }); + + it('maps a returned entity with a custom locationTemplate', async () => { + // Provide an alternate location template. + factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + locationTemplate: '/software/:name', + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + expect(documents[0]).toMatchObject({ + location: '/software/test-entity', + }); + }); + + it('allows filtering of the retrieved catalog entities', async () => { + // Provide a custom filter. + factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + filter: { + kind: ['Foo', 'Bar'], + }, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + // The simulated 'Foo,Bar' filter should return in an empty list + expect(documents).toHaveLength(0); + }); + + it('paginates through catalog entities using batchSize', async () => { + factory = DefaultCatalogCollatorFactory.fromConfig(config, { + ...options, + batchSize: 1, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + expect(documents).toHaveLength(expectedEntities.length); + expect(documents[0].location).toBe( + '/catalog/default/component/test-entity', + ); + expect(documents[1].location).toBe( + '/catalog/default/component/test-entity-2', + ); + }); + }); +}); diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts new file mode 100644 index 0000000000..99377b2ca0 --- /dev/null +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -0,0 +1,173 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { + CatalogApi, + CatalogClient, + GetEntitiesRequest, +} from '@backstage/catalog-client'; +import { + Entity, + stringifyEntityRef, + UserEntity, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + DocumentCollatorFactory, + IndexableDocument, +} from '@backstage/search-common'; +import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; +import { Readable } from 'stream'; + +/** @public */ +export interface CatalogEntityDocument extends IndexableDocument { + componentType: string; + namespace: string; + kind: string; + lifecycle: string; + owner: string; +} + +/** @public */ +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; +}; + +/** @public */ +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'software-catalog'; + public readonly visibilityPermission = catalogEntityReadPermission; + + private locationTemplate: string; + private filter?: GetEntitiesRequest['filter']; + private batchSize: number; + private readonly catalogClient: CatalogApi; + private tokenManager: TokenManager; + + static fromConfig( + _config: Config, + options: DefaultCatalogCollatorFactoryOptions, + ) { + return new DefaultCatalogCollatorFactory(options); + } + + private constructor(options: DefaultCatalogCollatorFactoryOptions) { + const { + batchSize, + discovery, + locationTemplate, + filter, + catalogClient, + tokenManager, + } = options; + + this.locationTemplate = + locationTemplate || '/catalog/:namespace/:kind/:name'; + this.filter = filter; + this.batchSize = batchSize || 500; + this.catalogClient = + catalogClient || new CatalogClient({ discoveryApi: discovery }); + this.tokenManager = tokenManager; + } + + async getCollator(): Promise { + return Readable.from(this.execute()); + } + + private applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + return formatted.toLowerCase(); + } + + private isUserEntity(entity: Entity): entity is UserEntity { + return entity.kind.toLocaleUpperCase('en-US') === 'USER'; + } + + private getDocumentText(entity: Entity): string { + let documentText = entity.metadata.description || ''; + if (this.isUserEntity(entity)) { + if (entity.spec?.profile?.displayName && documentText) { + // combine displayName and description + const displayName = entity.spec?.profile?.displayName; + documentText = displayName.concat(' : ', documentText); + } else { + documentText = entity.spec?.profile?.displayName || documentText; + } + } + return documentText; + } + + private async *execute(): AsyncGenerator { + const { token } = await this.tokenManager.getToken(); + let entitiesRetrieved = 0; + let moreEntitiesToGet = true; + + // Offset/limit pagination is used on the Catalog Client in order to + // limit (and allow some control over) memory used by the search backend + // at index-time. + while (moreEntitiesToGet) { + const entities = ( + await this.catalogClient.getEntities( + { + filter: this.filter, + limit: this.batchSize, + offset: entitiesRetrieved, + }, + { token }, + ) + ).items; + + // Control looping through entity batches. + moreEntitiesToGet = entities.length === this.batchSize; + entitiesRetrieved += entities.length; + + for (const entity of entities) { + yield { + title: entity.metadata.title ?? entity.metadata.name, + location: this.applyArgsToFormat(this.locationTemplate, { + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + name: entity.metadata.name, + }), + text: this.getDocumentText(entity), + componentType: entity.spec?.type?.toString() || 'other', + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: (entity.spec?.owner as string) || '', + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + }; + } + } + } +} diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index 5641670333..93ff0b8b32 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -14,5 +14,11 @@ * limitations under the License. */ +export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; +export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; +export type { CatalogEntityDocument } from './DefaultCatalogCollatorFactory'; + +/** + * todo(backstage/techdocs-core): stop exporting this in a future release. + */ export { DefaultCatalogCollator } from './DefaultCatalogCollator'; -export type { CatalogEntityDocument } from './DefaultCatalogCollator'; From 5185e27bdf3f37c535faca052838864af9b92e16 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 2 Mar 2022 17:23:42 +0100 Subject: [PATCH 112/150] chore: fix api-report Signed-off-by: blam --- .changeset/large-dancers-learn.md | 2 +- plugins/catalog-react/api-report.md | 17 ++++++++++------- .../src/components/EntityRefLink/humanize.ts | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.changeset/large-dancers-learn.md b/.changeset/large-dancers-learn.md index 43f9d27b11..9f25990924 100644 --- a/.changeset/large-dancers-learn.md +++ b/.changeset/large-dancers-learn.md @@ -2,4 +2,4 @@ '@backstage/plugin-scaffolder': minor --- -- **BREAKING**: Removed the `FavouriteEntity` export in favor of the `FavoriteEntity` from `@backstage/plguin-catalog-react`. Please migrate any usages to that component instead if you are creating your own `TemplateCard` page. +- **BREAKING**: Removed the `FavouriteTemplate` export in favor of the `FavoriteEntity` from `@backstage/plugin-catalog-react`. Please migrate any usages to that component instead if you are creating your own `TemplateCard` page. diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 78ad5d4b97..394a45f278 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -413,13 +413,8 @@ export const favoriteEntityTooltip: ( isStarred: boolean, ) => 'Remove from favorites' | 'Add to favorites'; -// @public (undocumented) -export function formatEntityRefTitle( - entityRef: Entity | EntityName, - opts?: { - defaultKind?: string; - }, -): string; +// @public @deprecated (undocumented) +export const formatEntityRefTitle: typeof humanizeEntityRef; // @public @deprecated (undocumented) export function getEntityMetadataEditUrl(entity: Entity): string | undefined; @@ -442,6 +437,14 @@ export function getEntitySourceLocation( scmIntegrationsApi: ScmIntegrationRegistry, ): EntitySourceLocation | undefined; +// @public (undocumented) +export function humanizeEntityRef( + entityRef: Entity | EntityName, + opts?: { + defaultKind?: string; + }, +): string; + // @public export function InspectEntityDialog(props: { open: boolean; diff --git a/plugins/catalog-react/src/components/EntityRefLink/humanize.ts b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts index af70ae71f4..e2984c34ab 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/humanize.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts @@ -20,7 +20,7 @@ import { DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; -/** @deprecated please use {@link humanizeEntityRef} instead */ +/** @public @deprecated please use {@link humanizeEntityRef} instead */ export const formatEntityRefTitle = humanizeEntityRef; /** @public */ From 0087554f5ca9f95de4d1338f4d1099b5077d09e6 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 20:01:30 +0100 Subject: [PATCH 113/150] Update TechDocs Collator to be stream-based Signed-off-by: Eric Peterson --- plugins/techdocs-backend/api-report.md | 35 ++- plugins/techdocs-backend/package.json | 1 + plugins/techdocs-backend/src/index.ts | 10 +- .../search/DefaultTechDocsCollator.test.ts | 12 - .../src/search/DefaultTechDocsCollator.ts | 5 +- .../DefaultTechDocsCollatorFactory.test.ts | 245 +++++++++++++++++ .../search/DefaultTechDocsCollatorFactory.ts | 254 ++++++++++++++++++ plugins/techdocs-backend/src/search/index.ts | 8 +- 8 files changed, 550 insertions(+), 20 deletions(-) create mode 100644 plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts create mode 100644 plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index 2bf4664f77..155f096614 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -3,9 +3,11 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { CatalogApi } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; -import { DocumentCollator } from '@backstage/search-common'; +import { DocumentCollatorFactory } from '@backstage/search-common'; import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { GeneratorBuilder } from '@backstage/techdocs-common'; @@ -16,14 +18,15 @@ import { PluginCacheManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PreparerBuilder } from '@backstage/techdocs-common'; import { PublisherBase } from '@backstage/techdocs-common'; +import { Readable } from 'stream'; import { TechDocsDocument } from '@backstage/techdocs-common'; import { TokenManager } from '@backstage/backend-common'; // @public export function createRouter(options: RouterOptions): Promise; -// @public -export class DefaultTechDocsCollator implements DocumentCollator { +// @public @deprecated +export class DefaultTechDocsCollator { // (undocumented) protected applyArgsToFormat( format: string, @@ -42,6 +45,21 @@ export class DefaultTechDocsCollator implements DocumentCollator { readonly visibilityPermission: Permission; } +// @public +export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + static fromConfig( + config: Config, + options: TechDocsCollatorFactoryOptions, + ): DefaultTechDocsCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; + // (undocumented) + readonly visibilityPermission: Permission; +} + // @public export interface DocsBuildStrategy { // (undocumented) @@ -81,6 +99,17 @@ export type ShouldBuildParameters = { entity: Entity; }; +// @public +export type TechDocsCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger_2; + tokenManager: TokenManager; + locationTemplate?: string; + catalogClient?: CatalogApi; + parallelismLimit?: number; + legacyPathCasing?: boolean; +}; + // @public export type TechDocsCollatorOptions = { discovery: PluginEndpointDiscovery; diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 7e8241bfea..20100f4848 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -56,6 +56,7 @@ }, "devDependencies": { "@backstage/cli": "^0.14.1", + "@backstage/plugin-search-backend-node": "0.4.7", "@backstage/test-utils": "^0.2.6", "@types/dockerode": "^3.3.0", "msw": "^0.35.0", diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts index 01acbea2cc..570da95092 100644 --- a/plugins/techdocs-backend/src/index.ts +++ b/plugins/techdocs-backend/src/index.ts @@ -29,8 +29,14 @@ export type { ShouldBuildParameters, } from './service'; -export { DefaultTechDocsCollator } from './search'; -export type { TechDocsCollatorOptions } from './search'; +export { + DefaultTechDocsCollator, + DefaultTechDocsCollatorFactory, +} from './search'; +export type { + TechDocsCollatorFactoryOptions, + TechDocsCollatorOptions, +} from './search'; /** * @deprecated Use directly from @backstage/techdocs-common diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts index d9f75767a7..887dc60b1a 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts @@ -72,18 +72,6 @@ const expectedEntities: Entity[] = [ owner: 'someone', }, }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'test-entity', - description: 'The expected description', - }, - spec: { - type: 'some-type', - lifecycle: 'experimental', - }, - }, ]; describe('DefaultTechDocsCollator with legacyPathCasing configuration', () => { diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts index 73c030cc51..d36e1c0ae5 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts @@ -24,7 +24,6 @@ import { RELATION_OWNED_BY, stringifyEntityRef, } from '@backstage/catalog-model'; -import { DocumentCollator } from '@backstage/search-common'; import fetch from 'node-fetch'; import unescape from 'lodash/unescape'; import { Logger } from 'winston'; @@ -69,8 +68,10 @@ type EntityInfo = { * A search collator responsible for gathering and transforming TechDocs documents. * * @public + * @deprecated Upgrade to a more recent `@backstage/search-backend-node` and + * use `DefaultTechDocsCollatorFactory` instead. */ -export class DefaultTechDocsCollator implements DocumentCollator { +export class DefaultTechDocsCollator { public readonly type: string = 'techdocs'; public readonly visibilityPermission = catalogEntityReadPermission; diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts new file mode 100644 index 0000000000..fafaae71b9 --- /dev/null +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts @@ -0,0 +1,245 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + getVoidLogger, + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { setupRequestMockHandlers } from '@backstage/test-utils'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { Readable } from 'stream'; +import { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory'; + +const logger = getVoidLogger(); + +const mockSearchDocIndex = { + config: { + lang: ['en'], + min_search_length: 3, + prebuild_index: false, + separator: '[\\s\\-]+', + }, + docs: [ + { + location: '', + text: 'docs docs docs', + title: 'Home', + }, + { + location: 'local-development/', + text: 'Docs for first subtitle', + title: 'Local development', + }, + { + location: 'local-development/#development', + text: 'Docs for sub-subtitle', + title: 'Development', + }, + ], +}; + +const expectedEntities: Entity[] = [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + title: 'Test Entity with Docs!', + name: 'test-entity-with-docs', + description: 'Documented description', + annotations: { + 'backstage.io/techdocs-ref': './', + }, + }, + spec: { + type: 'dog', + lifecycle: 'experimental', + owner: 'someone', + }, + }, +]; + +describe('DefaultTechDocsCollatorFactory', () => { + const config = new ConfigReader({}); + const mockDiscoveryApi: jest.Mocked = { + getBaseUrl: jest.fn().mockResolvedValue('http://test-backend'), + getExternalBaseUrl: jest.fn(), + }; + const mockTokenManager: jest.Mocked = { + getToken: jest.fn().mockResolvedValue({ token: '' }), + authenticate: jest.fn(), + }; + const options = { + discovery: mockDiscoveryApi, + logger: getVoidLogger(), + tokenManager: mockTokenManager, + }; + + it('has expected type', () => { + const factory = DefaultTechDocsCollatorFactory.fromConfig(config, options); + expect(factory.type).toBe('techdocs'); + }); + + describe('getCollator', () => { + let factory: DefaultTechDocsCollatorFactory; + let collator: Readable; + + const worker = setupServer(); + setupRequestMockHandlers(worker); + + beforeEach(async () => { + factory = DefaultTechDocsCollatorFactory.fromConfig(config, options); + collator = await factory.getCollator(); + + worker.use( + rest.get( + 'http://test-backend/static/docs/default/Component/test-entity-with-docs/search/search_index.json', + (_, res, ctx) => res(ctx.status(200), ctx.json(mockSearchDocIndex)), + ), + rest.get('http://test-backend/entities', (req, res, ctx) => { + // Imitate offset/limit pagination. + const offset = parseInt( + req.url.searchParams.get('offset') || '0', + 10, + ); + const limit = parseInt( + req.url.searchParams.get('limit') || '500', + 10, + ); + + // Limit 50 corresponds to a case testing pagination. + if (limit === 50) { + // Return 50 copies of invalid entities on the first request. + if (offset === 0) { + return res(ctx.status(200), ctx.json(Array(50).fill({}))); + } + // Then just the regular 2 on the second. + return res(ctx.status(200), ctx.json(expectedEntities)); + } + return res( + ctx.status(200), + ctx.json(expectedEntities.slice(offset, limit + offset)), + ); + }), + ); + }); + + it('returns a readable stream', async () => { + expect(collator).toBeInstanceOf(Readable); + }); + + it('fetches from the configured catalog and tech docs services', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog'); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('techdocs'); + expect(documents).toHaveLength(mockSearchDocIndex.docs.length); + }); + + it('should create documents for each tech docs search index', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + const entity = expectedEntities[0]; + documents.forEach((document, idx) => { + expect(document).toMatchObject({ + title: mockSearchDocIndex.docs[idx].title, + location: `/docs/default/component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`, + text: mockSearchDocIndex.docs[idx].text, + namespace: 'default', + entityTitle: entity!.metadata.title, + componentType: entity!.spec!.type, + lifecycle: entity!.spec!.lifecycle, + owner: '', + kind: entity.kind.toLocaleLowerCase('en-US'), + name: entity.metadata.name, + }); + }); + }); + + it('maps a returned entity with a custom locationTemplate', async () => { + // Provide an alternate location template. + factory = DefaultTechDocsCollatorFactory.fromConfig(config, { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + locationTemplate: '/software/:name', + logger, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + expect(documents[0]).toMatchObject({ + location: '/software/test-entity-with-docs', + }); + }); + + it('paginates through catalog entities using batchSize', async () => { + // A parallelismLimit of 1 is a catalog limit of 50 per request. Code + // above in the /entities handler ensures valid entities are only + // returned on the second page. + factory = DefaultTechDocsCollatorFactory.fromConfig(config, { + ...options, + parallelismLimit: 1, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + // Only 1 entity with TechDocs configured multipled by 3 pages. + expect(documents).toHaveLength(3); + }); + + describe('with legacyPathCasing configuration', () => { + beforeEach(async () => { + const legacyConfig = new ConfigReader({ + techdocs: { + legacyUseCaseSensitiveTripletPaths: true, + }, + }); + factory = DefaultTechDocsCollatorFactory.fromConfig( + legacyConfig, + options, + ); + collator = await factory.getCollator(); + }); + + it('should create documents for each tech docs search index', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + const entity = expectedEntities[0]; + documents.forEach((document, idx) => { + expect(document).toMatchObject({ + title: mockSearchDocIndex.docs[idx].title, + location: `/docs/default/Component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`, + text: mockSearchDocIndex.docs[idx].text, + namespace: 'default', + entityTitle: entity!.metadata.title, + componentType: entity!.spec!.type, + lifecycle: entity!.spec!.lifecycle, + owner: '', + kind: entity.kind, + name: entity.metadata.name, + }); + }); + }); + }); + }); +}); diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts new file mode 100644 index 0000000000..181f032b54 --- /dev/null +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts @@ -0,0 +1,254 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { + CatalogApi, + CatalogClient, + CATALOG_FILTER_EXISTS, +} from '@backstage/catalog-client'; +import { + Entity, + parseEntityRef, + RELATION_OWNED_BY, + stringifyEntityRef, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; +import { DocumentCollatorFactory } from '@backstage/search-common'; +import { TechDocsDocument } from '@backstage/techdocs-common'; +import unescape from 'lodash/unescape'; +import fetch from 'node-fetch'; +import pLimit from 'p-limit'; +import { Readable } from 'stream'; +import { Logger } from 'winston'; + +interface MkSearchIndexDoc { + title: string; + text: string; + location: string; +} + +/** + * Options to configure the TechDocs collator factory + * + * @public + */ +export type TechDocsCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger; + tokenManager: TokenManager; + locationTemplate?: string; + catalogClient?: CatalogApi; + parallelismLimit?: number; + legacyPathCasing?: boolean; +}; + +type EntityInfo = { + name: string; + namespace: string; + kind: string; +}; + +/** + * A search collator factory responsible for gathering and transforming + * TechDocs documents. + * + * @public + */ +export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'techdocs'; + public readonly visibilityPermission = catalogEntityReadPermission; + + private discovery: PluginEndpointDiscovery; + private locationTemplate: string; + private readonly logger: Logger; + private readonly catalogClient: CatalogApi; + private readonly tokenManager: TokenManager; + private readonly parallelismLimit: number; + private readonly legacyPathCasing: boolean; + + private constructor(options: TechDocsCollatorFactoryOptions) { + this.discovery = options.discovery; + this.locationTemplate = + options.locationTemplate || '/docs/:namespace/:kind/:name/:path'; + this.logger = options.logger; + this.catalogClient = + options.catalogClient || + new CatalogClient({ discoveryApi: options.discovery }); + this.parallelismLimit = options.parallelismLimit ?? 10; + this.legacyPathCasing = options.legacyPathCasing ?? false; + this.tokenManager = options.tokenManager; + } + + static fromConfig(config: Config, options: TechDocsCollatorFactoryOptions) { + const legacyPathCasing = + config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + return new DefaultTechDocsCollatorFactory({ ...options, legacyPathCasing }); + } + + async getCollator(): Promise { + return Readable.from(this.execute()); + } + + private async *execute(): AsyncGenerator { + const limit = pLimit(this.parallelismLimit); + const techDocsBaseUrl = await this.discovery.getBaseUrl('techdocs'); + const { token } = await this.tokenManager.getToken(); + let entitiesRetrieved = 0; + let moreEntitiesToGet = true; + + // Offset/limit pagination is used on the Catalog Client in order to + // limit (and allow some control over) memory used by the search backend + // at index-time. The batchSize is calculated as a factor of the given + // parallelism limit to simplify configuration. + const batchSize = this.parallelismLimit * 50; + while (moreEntitiesToGet) { + const entities = ( + await this.catalogClient.getEntities( + { + filter: { + 'metadata.annotations.backstage.io/techdocs-ref': + CATALOG_FILTER_EXISTS, + }, + fields: [ + 'kind', + 'namespace', + 'metadata.annotations', + 'metadata.name', + 'metadata.title', + 'metadata.namespace', + 'spec.type', + 'spec.lifecycle', + 'relations', + ], + limit: batchSize, + offset: entitiesRetrieved, + }, + { token }, + ) + ).items; + + // Control looping through entity batches. + moreEntitiesToGet = entities.length === batchSize; + entitiesRetrieved += entities.length; + + const docPromises = entities + .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref']) + .map((entity: Entity) => + limit(async (): Promise => { + const entityInfo = + DefaultTechDocsCollatorFactory.handleEntityInfoCasing( + this.legacyPathCasing, + { + kind: entity.kind, + namespace: entity.metadata.namespace || 'default', + name: entity.metadata.name, + }, + ); + + try { + const searchIndexResponse = await fetch( + DefaultTechDocsCollatorFactory.constructDocsIndexUrl( + techDocsBaseUrl, + entityInfo, + ), + { + headers: { + Authorization: `Bearer ${token}`, + }, + }, + ); + const searchIndex = await searchIndexResponse.json(); + + return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({ + title: unescape(doc.title), + text: unescape(doc.text || ''), + location: this.applyArgsToFormat( + this.locationTemplate || '/docs/:namespace/:kind/:name/:path', + { + ...entityInfo, + path: doc.location, + }, + ), + path: doc.location, + ...entityInfo, + entityTitle: entity.metadata.title, + componentType: entity.spec?.type?.toString() || 'other', + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: getSimpleEntityOwnerString(entity), + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + })); + } catch (e) { + this.logger.debug( + `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`, + e, + ); + return []; + } + }), + ); + yield* (await Promise.all(docPromises)).flat(); + } + } + + private applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + return formatted; + } + + private static constructDocsIndexUrl( + techDocsBaseUrl: string, + entityInfo: { kind: string; namespace: string; name: string }, + ) { + return `${techDocsBaseUrl}/static/docs/${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}/search/search_index.json`; + } + + private static handleEntityInfoCasing( + legacyPaths: boolean, + entityInfo: EntityInfo, + ): EntityInfo { + return legacyPaths + ? entityInfo + : Object.entries(entityInfo).reduce((acc, [key, value]) => { + return { ...acc, [key]: value.toLocaleLowerCase('en-US') }; + }, {} as EntityInfo); + } +} + +function getSimpleEntityOwnerString(entity: Entity): string { + if (entity.relations) { + const owner = entity.relations.find(r => r.type === RELATION_OWNED_BY); + if (owner) { + const { name } = parseEntityRef(owner.targetRef); + return name; + } + } + return ''; +} diff --git a/plugins/techdocs-backend/src/search/index.ts b/plugins/techdocs-backend/src/search/index.ts index fbbd23b964..68e3b4edc7 100644 --- a/plugins/techdocs-backend/src/search/index.ts +++ b/plugins/techdocs-backend/src/search/index.ts @@ -13,6 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { DefaultTechDocsCollator } from './DefaultTechDocsCollator'; +export { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory'; +export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFactory'; + +/** + * todo(backstage/techdocs-core): stop exporting these in a future release. + */ +export { DefaultTechDocsCollator } from './DefaultTechDocsCollator'; export type { TechDocsCollatorOptions } from './DefaultTechDocsCollator'; From d0993bc6f1956b285ebeac910afa5c315809228a Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 20:48:20 +0100 Subject: [PATCH 114/150] Wire up app/create-app to use stream-based search implementations Signed-off-by: Eric Peterson --- packages/backend/src/plugins/search.ts | 8 ++++---- .../default-app/packages/backend/src/plugins/search.ts | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 4b5363e73d..a90b47cd81 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -18,7 +18,7 @@ import { useHotCleanup, } from '@backstage/backend-common'; import { Config } from '@backstage/config'; -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; import { createRouter } from '@backstage/plugin-search-backend'; import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch'; import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg'; @@ -27,7 +27,7 @@ import { LunrSearchEngine, SearchEngine, } from '@backstage/plugin-search-backend-node'; -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; import { Logger } from 'winston'; import { PluginEnvironment } from '../types'; @@ -70,7 +70,7 @@ export default async function createPlugin({ // particular collator gathers entities from the software catalog. indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultCatalogCollator.fromConfig(config, { + factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery, tokenManager, }), @@ -78,7 +78,7 @@ export default async function createPlugin({ indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultTechDocsCollator.fromConfig(config, { + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { discovery, logger, tokenManager, diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts index a0a1cc3701..c359cb4986 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts @@ -5,8 +5,8 @@ import { LunrSearchEngine, } from '@backstage/plugin-search-backend-node'; import { PluginEnvironment } from '../types'; -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; export default async function createPlugin({ logger, @@ -23,7 +23,7 @@ export default async function createPlugin({ // collator gathers entities from the software catalog. indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultCatalogCollator.fromConfig(config, { + factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery, tokenManager, }), @@ -32,7 +32,7 @@ export default async function createPlugin({ // collator gathers entities from techdocs. indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultTechDocsCollator.fromConfig(config, { + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { discovery, logger, tokenManager, From 022507c8603a7e8b3901d9a77ed18d63251e7767 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 21:05:24 +0100 Subject: [PATCH 115/150] Document stream-based search Signed-off-by: Eric Peterson --- .changeset/search-blankly-have-a-nice-life.md | 14 + .changeset/search-done-me-no-favor.md | 6 + .changeset/search-just-smile-politely.md | 13 + .changeset/search-like-a-bank-teller.md | 43 +++ .changeset/search-selfless-cold-composed.md | 6 + .changeset/search-throw-me-a-right.md | 13 + .github/styles/vocab.txt | 1 + docs/features/search/concepts.md | 25 +- docs/features/search/how-to-guides.md | 274 +++++++++++++++++- 9 files changed, 380 insertions(+), 15 deletions(-) create mode 100644 .changeset/search-blankly-have-a-nice-life.md create mode 100644 .changeset/search-done-me-no-favor.md create mode 100644 .changeset/search-just-smile-politely.md create mode 100644 .changeset/search-like-a-bank-teller.md create mode 100644 .changeset/search-selfless-cold-composed.md create mode 100644 .changeset/search-throw-me-a-right.md diff --git a/.changeset/search-blankly-have-a-nice-life.md b/.changeset/search-blankly-have-a-nice-life.md new file mode 100644 index 0000000000..eb6d02110c --- /dev/null +++ b/.changeset/search-blankly-have-a-nice-life.md @@ -0,0 +1,14 @@ +--- +'@backstage/plugin-search-backend-node': minor +'@backstage/search-common': minor +--- + +The Backstage Search Platform's indexing process has been rewritten as a stream +pipeline in order to improve efficiency and performance on large document sets. + +The concepts of `Collator` and `Decorator` have been replaced with readable and +transform object streams (respectively), as well as factory classes to +instantiate them. + +Accordingly, the `SearchEngine.index()` method has also been replaced with a +`getIndexer()` factory method that resolves to a writable object stream. diff --git a/.changeset/search-done-me-no-favor.md b/.changeset/search-done-me-no-favor.md new file mode 100644 index 0000000000..620c9dd923 --- /dev/null +++ b/.changeset/search-done-me-no-favor.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-search-backend-module-pg': minor +--- + +The `PgSearchEngine` implements the new stream-based indexing process expected +by the latest `@backstage/search-backend-node`. diff --git a/.changeset/search-just-smile-politely.md b/.changeset/search-just-smile-politely.md new file mode 100644 index 0000000000..e1cb57a17e --- /dev/null +++ b/.changeset/search-just-smile-politely.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-techdocs-backend': patch +--- + +A `DefaultTechDocsCollatorFactory`, which works with the new stream-based +search indexing subsystem, is now available. The `DefaultTechDocsCollator` will +continue to be available for those unable to upgrade to the stream-based +`@backstage/search-backend-node` (and related packages), however it is now +marked as deprecated and will be removed in a future version. + +To upgrade this plugin and the search indexing subsystem in one go, check +[this changelog](https://github.com/backstage/backstage/blob/master/packages/create-app/CHANGELOG.md) +for necessary changes to your search backend plugin configuration. diff --git a/.changeset/search-like-a-bank-teller.md b/.changeset/search-like-a-bank-teller.md new file mode 100644 index 0000000000..a9b0330eb2 --- /dev/null +++ b/.changeset/search-like-a-bank-teller.md @@ -0,0 +1,43 @@ +--- +'@backstage/create-app': patch +--- + +The Backstage Search Platform's indexing process has been rewritten as a stream +pipeline in order to improve efficiency and performance on large document sets. + +To take advantage of this, upgrade to the latest version of +`@backstage/plugin-search-backend-node`, as well as any backend plugins whose +collators you are using. Then, make the following changes to your +`/packages/backend/src/plugins/search.ts` file: + +```diff +-import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; +-import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; ++import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; ++import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; + +// ... + + const indexBuilder = new IndexBuilder({ logger, searchEngine }); + + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, +- collator: DefaultCatalogCollator.fromConfig(config, { discovery }), ++ factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery }), + }); + + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, +- collator: DefaultTechDocsCollator.fromConfig(config, { ++ factory: DefaultTechDocsCollatorFactory.fromConfig(config, { + discovery, + logger, + }), + }); +``` + +If you've written custom collators, decorators, or search engines in your +Backstage backend instance, you will need to re-implement them as readable, +transform, and writable streams respectively (including factory classes for +instantiating them). [A how-to guide for refactoring](https://backstage.io/docs/features/search/how-to-guides#rewriting-alpha-style-collators-for-beta) +existing implementations is available. diff --git a/.changeset/search-selfless-cold-composed.md b/.changeset/search-selfless-cold-composed.md new file mode 100644 index 0000000000..31c0e86baf --- /dev/null +++ b/.changeset/search-selfless-cold-composed.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': minor +--- + +The `ElasticSearchSearchEngine` implements the new stream-based indexing +process expected by the latest `@backstage/search-backend-node`. diff --git a/.changeset/search-throw-me-a-right.md b/.changeset/search-throw-me-a-right.md new file mode 100644 index 0000000000..e504888cfb --- /dev/null +++ b/.changeset/search-throw-me-a-right.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +A `DefaultCatalogCollatorFactory`, which works with the new stream-based +search indexing subsystem, is now available. The `DefaultCatalogCollator` will +continue to be available for those unable to upgrade to the stream-based +`@backstage/search-backend-node` (and related packages), however it is now +marked as deprecated and will be removed in a future version. + +To upgrade this plugin and the search indexing subsystem in one go, check +[this changelog](https://github.com/backstage/backstage/blob/master/packages/create-app/CHANGELOG.md) +for necessary changes to your search backend plugin configuration. diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 6359493d4b..e4b457dc1d 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -213,6 +213,7 @@ parallelization Patrik Peloton performant +Performant plantuml Platformize Podman diff --git a/docs/features/search/concepts.md b/docs/features/search/concepts.md index 8cb17b12e6..052f71376f 100644 --- a/docs/features/search/concepts.md +++ b/docs/features/search/concepts.md @@ -54,13 +54,14 @@ An index is a collection of such documents of a given type. ### Collators You need to be able to search something! Collators are the way to define what -can be searched. Specifically, they're classes which return documents conforming -to a minimum set of fields (including a document title, location, and text), but -which can contain any other fields as defined by the collator itself. One -collator is responsible for defining and collecting documents of a type. +can be searched. Specifically, they're readable object streams of documents that +conform to a minimum set of fields (including a document title, location, and +text), but which can contain any other fields as defined by the collator itself. +One collator is responsible for defining and collecting documents of a type. -Some plugins, like the Catalog Backend, provide so-called "default" collators -which you can use out-of-the-box to start searching across Backstage quickly. +Some plugins, like the Catalog Backend, provide so-called "default" collator +factories which you can use out-of-the-box to start searching across Backstage +quickly. ### Decorators @@ -68,9 +69,15 @@ Sometimes you want to add extra information to a set of documents in your search index that the collator may not be aware of. For example, the Software Catalog knows about software entities, but it may not know about their usage or quality. -Decorators are classes which can add extra fields to pre-collated documents. -This extra metadata could then be used to bias search results or otherwise -improve the search experience in your Backstage instance. +Decorators are transform streams which sit between a collator (read stream) and +an indexer (write stream) during the indexing process. It can be used to add +extra fields to documents as they are being collated and indexed. This extra +metadata could then be used to bias search results or otherwise improve the +search experience in your Backstage instance. + +In addition to adding extra metadata, decorators (like any transform stream) can +also be used to remove metadata, filter out, or even add extra documents at +index-time. ### The Scheduler diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index a3c42c682c..ac36d64673 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -48,10 +48,10 @@ const app = createApp({ ## How to index TechDocs documents The TechDocs plugin has supported integrations to Search, meaning that it -provides a default collator ready to be used. +provides a default collator factory ready to be used. The purpose of this guide is to walk you through how to register the -[DefaultTechDocsCollator](https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts) +[DefaultTechDocsCollatorFactory](https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts) in your App, so that you can get TechDocs documents indexed. If you have been through the @@ -60,18 +60,19 @@ you should have the `packages/backend/src/plugins/search.ts` file available. If so, you can go ahead and follow this guide - if not, start by going through the getting started guide. -1. Import the DefaultTechDocsCollator from `@backstage/plugin-techdocs-backend`. +1. Import the `DefaultTechDocsCollatorFactory` from + `@backstage/plugin-techdocs-backend`. ```typescript -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; ``` -2. Register the DefaultTechDocsCollator with the IndexBuilder. +2. Register the `DefaultTechDocsCollatorFactory` with the IndexBuilder. ```typescript indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultTechDocsCollator.fromConfig(config, { + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { discovery, logger, tokenManager, @@ -131,3 +132,264 @@ indexBuilder.addCollator({ As shown above, you can add a catalog entity filter to narrow down what catalog entities are indexed by the search engine. + +## How to migrate from Search Alpha to Beta + +For the purposes of this guide, Search Beta version is defined as: + +- **Search Plugin**: At least `v0.x.y` +- **Search Backend Plugin**: At least `v0.x.y` +- **Search Backend Node**: At least `v0.x.y` + +In the Beta version, the Search Platform's indexing process has been rewritten +as a stream pipeline in order to improve efficiency and performance on large +sets of documents. + +If you've not yet extended the Search Platform with custom code, and have +instead taken advantage of default collators, decorators, and search engines +provided by existing plugins, the migration process is fairly straightforward: + +1. Upgrade to at least version `0.x.y` of + `@backstage/plugin-search-backend-node`, as well as any backend plugins whose + collators you are using (e.g. at least version `0.x.y` of + `@backstage/plugin-catalog-backend` and/or version `0.x.y` of + `@backstage/plugin-techdocs-backend`). +2. Then, make the following changes to your + `/packages/backend/src/plugins/search.ts` file: + + ```diff + -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; + -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; + +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; + +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; + // ... + const indexBuilder = new IndexBuilder({ logger, searchEngine }); + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, + - collator: DefaultCatalogCollator.fromConfig(config, { discovery }), + + factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery }), + }); + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, + - collator: DefaultTechDocsCollator.fromConfig(config, { + + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { + discovery, + logger, + }), + }); + ``` + +Any custom collators, decorators, or search engine implementations will require +minor refactoring. Continue on for details. + +### Rewriting alpha-style collators for beta + +In alpha versions of the Backstage Search Platform, collators were classes that +implemented an `execute` method which resolved an `IndexableDocument` array. + +In beta versions, the logic encapsulated by the aforementioned `execute` method +is contained within an [object-mode][obj-mode] `Readable` stream where each +object pushed onto the stream is of type `IndexableDocument`. Instances of this +stream are instantiated by a factory class conforming to the +`DocumentCollatorFactory` interface. + +The optimal conversion strategy will vary depending on the collator's logic, but +the simplest conversion can follow a process like this: + +1. Rename your collator class to something like `YourCollatorFactory` and update + it to implement `DocumentCollatorFactory` instead of `DocumentCollator`. +2. Update its `execute` method so that it resolves + `AsyncGenerator` instead of `YourIndexableDocument[]`. +3. Implement `DocumentCollatorFactory`'s `getCollator` method which resolves to + `Readable.from(this.execute())` (which is a utility for creating [readable + streams][read-stream] from [async generators][async-gen]). + +```ts +import { DocumentCollatorFactory } from '@backstage/plugin-search-backend-node'; +import { Readable } from 'stream'; +export class YourCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'your-type'; + async *execute(): AsyncGenerator { + const widgets = await this.client.getWidgets(); + for (const widget of widgets) { + yield { + title: widget.name, + location: widget.url, + text: widget.description, + }; + } + } + getCollator() { + return Readable.from(this.execute()); + } +} +``` + +Note: it may be possible to simplify your collator dramatically! If your custom +collator was previously using streams under the hood (for example, by reading +newline delimited JSON from a local or remote file), you could just expose the +stream directly via a simple factory class: + +```ts +import { DocumentCollatorFactory } from '@backstage/plugin-search-backend-node'; +import { createReadStream } from 'fs'; +import { parse } from '@jsonlines/core'; +export class YourCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'your-type'; + async getCollator() { + const parseStream = parse(); + return createReadStream('./documents.ndjson').pipe(parseStream); + } +} +``` + +### Rewriting alpha-style decorators for beta + +In alpha versions of the Backstage Search Platform, decorators were classes that +implemented an `execute` method which took an `IndexableDocument` array as an +argument, and resolved a modified array of the same type. + +In beta versions, the logic encapsulated by the aforementioned `execute` method +is contained within an object-mode `Transform` stream which reads objects of +type `IndexableDocument`, and writes objects of a conforming type. Similar to +collators, instances of this stream are instantiated by a factory class +conforming to the `DocumentDecoratorFactory` interface. + +Although you can choose to implement a `Transform` stream from scratch, the +`@backstage/plugin-search-backend-node` package provides a `DecoratorBase` class +in order to simplify the developer experience. With this base class, all that's +needed is to transfer your old decorator class logic into the base class' three +methods (`initialize`, `decorate`, and `finalize`), and implement the factory +class that instantiates the stream: + +```ts +import { DecoratorBase } from '@backstage/plugin-search-backend-node'; +export class YourDecorator extends DecoratorBase { + async initialize() { + // Setup logic. Performed once before any documents are consumed. + } + async decorate( + document: YourIndexableDocument, + ): Promise { + // Perform transformation logic here. + return document; + } + async finalize() { + // Teardown logic. Performed once after all documents have been consumed. + } +} +export class YourDecoratorFactory implements DocumentDecoratorFactory { + async getDecorator() { + return new YourDecorator(); + } +} +``` + +Note the return type of the `decorate` method and how each can be used to +different effect. + +- By resolving a single `YourIndexableDocument` object, your decorator can be + used to make simple transformations: + + ```ts + class BooleanWidgetCoolnessDecorator extends DecoratorBase { + async decorator(widget) { + // Perform a simple, 1:1 transformation. + widget.isCool = widget.isCool === 'true' ? true : false; + return widget; + } + } + ``` + +- By resolving `undefined`, your decorator can filter out documents which + shouldn't be in the index: + + ```ts + class OnlyCoolWidgetsDecorator extends DecoratorBase { + async decorator(widget) { + // Perform a simple filter operation. + return widget.isCool ? widget : undefined; + } + } + ``` + +- By resolving an array of `YourIndexableDocument` objects, you can generate + multiple documents based on the content of one: + + ```ts + class WidgetByVariantDecorator extends DecoratorBase { + async decorator(widget) { + // Generate one widget doc per widget variant. + return widget.variants.map(variant => { + // Each widget doc is the given widget plus a "variant" property + // pulled from a widget.variants string array. + return { + ...widget, + variant, + }; + }); + } + } + ``` + +In alpha versions, a decorator had access to every `IndexableDocument` +simultaneously. This is no longer possible in beta versions (precisely to make +the indexing process more efficient and performant). You will need to modify +your decorator's logic so that it does not need access to every document at +once. + +### Rewriting alpha-style search engines for beta + +Search Engines are responsible for both querying and indexing documents to an +underlying search engine technology. While the search engine query interface +didn't change between alpha and beta versions, the indexing half of the +interface _did_ change. + +In alpha versions of the Backstage Search Platform, a search engine implemented +an `index` method which took a `type` and an `IndexableDocument` array and was +responsible for writing these documents to the underlying search engine. + +In beta versions, the logic encapsulated by the aforementioned `index` method is +contained within an object-mode `Writable` stream which expects objects of type +`IndexableDocument`. On the search engine class itself, the `index` method is +replaced with a `getIndexer` factory method which still takes the `type`, but +resolves an instance of the aforementioned `Writable` stream. + +Although you can choose to implement a `Writable` stream from scratch, the +`@backstage/plugin-search-backend-node` package provides a +`BatchSearchEngineIndexer` class in order to simplify the developer experience. +With this base class, which collects documents in batches of a configurable size +on your behalf, all that's needed is to transfer your old `index` method logic +into the base class' three methods (`initialize`, `index`, and `finalize`), and +implement the factory method that instantiates the stream: + +```ts +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { SearchEngine } from '@backstage/search-common'; +export class YourSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor({ type }: { type: string }) { + // Customize the number of documents passed to the index method per batch. + super({ batchSize: 500 }); + // An imaginary search engine indexing client. + this.index = new SomeSearchEngineIndex({ indexName: type }); + } + async initialize() { + // Setup logic. Performed once before any documents are consumed. + } + async index(documents: IndexableDocument[]) { + await this.index.batchOf(documents); + } + async finalize() { + // Teardown logic. Performed once after all documents have been consumed. + } +} +export class YourSearchEngine implements SearchEngine { + async getIndexer(type: string) { + return new YourSearchEngineIndexer({ type }); + } +} +``` + +[obj-mode]: https://nodejs.org/docs/latest-v14.x/api/stream.html#stream_object_mode +[read-stream]: https://nodejs.org/docs/latest-v14.x/api/stream.html#stream_readable_streams +[async-gen]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of#iterating_over_async_generators From 37b9ff3b1946d309f92d4a241ee25dfb6c73b992 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 21:28:11 +0100 Subject: [PATCH 116/150] Update AuthorizedSearchEngine for stream-based indexing. Signed-off-by: Eric Peterson --- .../src/service/AuthorizedSearchEngine.test.ts | 2 +- .../search-backend/src/service/AuthorizedSearchEngine.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts b/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts index 0a7b45e164..137ffc24f1 100644 --- a/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts +++ b/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts @@ -62,7 +62,7 @@ describe('AuthorizedSearchEngine', () => { setTranslator: () => { throw new Error('Function not implemented. 1'); }, - index: () => { + getIndexer: () => { throw new Error('Function not implemented.2'); }, query: mockedQuery, diff --git a/plugins/search-backend/src/service/AuthorizedSearchEngine.ts b/plugins/search-backend/src/service/AuthorizedSearchEngine.ts index d0488af00d..82ed061196 100644 --- a/plugins/search-backend/src/service/AuthorizedSearchEngine.ts +++ b/plugins/search-backend/src/service/AuthorizedSearchEngine.ts @@ -25,7 +25,6 @@ import { } from '@backstage/plugin-permission-common'; import { DocumentTypeInfo, - IndexableDocument, QueryRequestOptions, QueryTranslator, SearchEngine, @@ -35,6 +34,7 @@ import { } from '@backstage/search-common'; import { Config } from '@backstage/config'; import { InputError } from '@backstage/errors'; +import { Writable } from 'stream'; export function decodePageCursor(pageCursor?: string): { page: number } { if (!pageCursor) { @@ -78,8 +78,8 @@ export class AuthorizedSearchEngine implements SearchEngine { this.searchEngine.setTranslator(translator); } - async index(type: string, documents: IndexableDocument[]): Promise { - this.searchEngine.index(type, documents); + async getIndexer(type: string): Promise { + return this.searchEngine.getIndexer(type); } async query( From 0547246b84489dbe10a67f88ebf9b79342132919 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 1 Mar 2022 14:21:23 +0100 Subject: [PATCH 117/150] Clean up search-common/search-backend-node APIs, indicating beta. Signed-off-by: Eric Peterson --- packages/search-common/api-report.md | 40 +++-------- packages/search-common/src/types.ts | 19 ++++++ plugins/search-backend-node/api-report.md | 68 ++++++++++--------- .../search-backend-node/src/IndexBuilder.ts | 9 ++- plugins/search-backend-node/src/Scheduler.ts | 3 + .../src/engines/LunrSearchEngine.ts | 11 ++- .../src/engines/LunrSearchEngineIndexer.ts | 3 + .../search-backend-node/src/engines/index.ts | 5 +- plugins/search-backend-node/src/index.ts | 11 ++- .../src/indexing/BatchSearchEngineIndexer.ts | 4 ++ .../src/indexing/DecoratorBase.ts | 1 + .../src/test-utils/TestPipeline.ts | 2 + plugins/search-backend-node/src/types.ts | 12 ++++ scripts/api-extractor.ts | 2 + 14 files changed, 120 insertions(+), 70 deletions(-) diff --git a/packages/search-common/api-report.md b/packages/search-common/api-report.md index e47140d274..475d85e11c 100644 --- a/packages/search-common/api-report.md +++ b/packages/search-common/api-report.md @@ -11,33 +11,25 @@ import { Readable } from 'stream'; import { Transform } from 'stream'; import { Writable } from 'stream'; -// Warning: (ae-missing-release-tag) "DocumentCollatorFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export interface DocumentCollatorFactory { getCollator(): Promise; readonly type: string; readonly visibilityPermission?: Permission; } -// Warning: (ae-missing-release-tag) "DocumentDecoratorFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export interface DocumentDecoratorFactory { getDecorator(): Promise; readonly types?: string[]; } -// Warning: (ae-missing-release-tag) "DocumentTypeInfo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export type DocumentTypeInfo = { visibilityPermission?: Permission; }; -// Warning: (ae-missing-release-tag) "IndexableDocument" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export interface IndexableDocument { authorization?: { resourceRef: string; @@ -47,21 +39,15 @@ export interface IndexableDocument { title: string; } -// Warning: (ae-missing-release-tag) "QueryRequestOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta export type QueryRequestOptions = { token?: string; }; -// Warning: (ae-missing-release-tag) "QueryTranslator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export type QueryTranslator = (query: SearchQuery) => unknown; -// Warning: (ae-missing-release-tag) "SearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export interface SearchEngine { getIndexer(type: string): Promise; query( @@ -71,9 +57,7 @@ export interface SearchEngine { setTranslator(translator: QueryTranslator): void; } -// Warning: (ae-missing-release-tag) "SearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export interface SearchQuery { // (undocumented) filters?: JsonObject; @@ -85,9 +69,7 @@ export interface SearchQuery { types?: string[]; } -// Warning: (ae-missing-release-tag) "SearchResult" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export interface SearchResult { // (undocumented) document: IndexableDocument; @@ -95,9 +77,7 @@ export interface SearchResult { type: string; } -// Warning: (ae-missing-release-tag) "SearchResultSet" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export interface SearchResultSet { // (undocumented) nextPageCursor?: string; diff --git a/packages/search-common/src/types.ts b/packages/search-common/src/types.ts index 6e106d3899..51ce45617b 100644 --- a/packages/search-common/src/types.ts +++ b/packages/search-common/src/types.ts @@ -18,6 +18,9 @@ import { Permission } from '@backstage/plugin-permission-common'; import { JsonObject } from '@backstage/types'; import { Readable, Transform, Writable } from 'stream'; +/** + * @beta + */ export interface SearchQuery { term: string; filters?: JsonObject; @@ -25,11 +28,17 @@ export interface SearchQuery { pageCursor?: string; } +/** + * @beta + */ export interface SearchResult { type: string; document: IndexableDocument; } +/** + * @beta + */ export interface SearchResultSet { results: SearchResult[]; nextPageCursor?: string; @@ -39,6 +48,7 @@ export interface SearchResultSet { /** * Base properties that all indexed documents must include, as well as some * common properties that documents are encouraged to use where appropriate. + * @beta */ export interface IndexableDocument { /** @@ -73,6 +83,7 @@ export interface IndexableDocument { * Information about a specific document type. Intended to be used in the * {@link @backstage/search-backend-node#IndexBuilder} to collect information * about the types stored in the index. + * @beta */ export type DocumentTypeInfo = { /** @@ -84,6 +95,7 @@ export type DocumentTypeInfo = { /** * Factory class for instantiating collators. + * @beta */ export interface DocumentCollatorFactory { /** @@ -106,6 +118,7 @@ export interface DocumentCollatorFactory { /** * Factory class for instantiating decorators. + * @beta */ export interface DocumentDecoratorFactory { /** @@ -124,9 +137,14 @@ export interface DocumentDecoratorFactory { /** * A type of function responsible for translating an abstract search query into * a concrete query relevant to a particular search engine. + * @beta */ export type QueryTranslator = (query: SearchQuery) => unknown; +/** + * Options when querying a search engine. + * @beta + */ export type QueryRequestOptions = { token?: string; }; @@ -135,6 +153,7 @@ export type QueryRequestOptions = { * Interface that must be implemented by specific search engines, responsible * for performing indexing and querying and translating abstract queries into * concrete, search engine-specific queries. + * @beta */ export interface SearchEngine { /** diff --git a/plugins/search-backend-node/api-report.md b/plugins/search-backend-node/api-report.md index d4b94a932e..37945c8902 100644 --- a/plugins/search-backend-node/api-report.md +++ b/plugins/search-backend-node/api-report.md @@ -19,9 +19,7 @@ import { SearchResultSet } from '@backstage/search-common'; import { Transform } from 'stream'; import { Writable } from 'stream'; -// Warning: (ae-missing-release-tag) "BatchSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export abstract class BatchSearchEngineIndexer extends Writable { constructor(options: BatchSearchEngineOptions); abstract finalize(): Promise; @@ -29,16 +27,19 @@ export abstract class BatchSearchEngineIndexer extends Writable { abstract initialize(): Promise; } -// Warning: (ae-missing-release-tag) "BatchSearchEngineOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export type BatchSearchEngineOptions = { batchSize: number; }; -// Warning: (ae-missing-release-tag) "DecoratorBase" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta (undocumented) +export type ConcreteLunrQuery = { + lunrQueryBuilder: lunr_2.Index.QueryBuilder; + documentTypes?: string[]; + pageSize: number; +}; + +// @beta export abstract class DecoratorBase extends Transform { constructor(); abstract decorate( @@ -48,18 +49,13 @@ export abstract class DecoratorBase extends Transform { abstract initialize(): Promise; } -// Warning: (ae-missing-release-tag) "IndexBuilder" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export class IndexBuilder { - // Warning: (ae-forgotten-export) The symbol "IndexBuilderOptions" needs to be exported by the entry point index.d.ts constructor({ logger, searchEngine }: IndexBuilderOptions); - // Warning: (ae-forgotten-export) The symbol "RegisterCollatorParameters" needs to be exported by the entry point index.d.ts addCollator({ factory, defaultRefreshIntervalSeconds, }: RegisterCollatorParameters): void; - // Warning: (ae-forgotten-export) The symbol "RegisterDecoratorParameters" needs to be exported by the entry point index.d.ts addDecorator({ factory }: RegisterDecoratorParameters): void; build(): Promise<{ scheduler: Scheduler; @@ -70,9 +66,16 @@ export class IndexBuilder { getSearchEngine(): SearchEngine; } -// Warning: (ae-missing-release-tag) "LunrSearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) +export type IndexBuilderOptions = { + searchEngine: SearchEngine; + logger: Logger_2; +}; + +// @beta (undocumented) +export type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery; + +// @beta (undocumented) export class LunrSearchEngine implements SearchEngine { constructor({ logger }: { logger: Logger_2 }); // (undocumented) @@ -85,17 +88,13 @@ export class LunrSearchEngine implements SearchEngine { protected lunrIndices: Record; // (undocumented) query(query: SearchQuery): Promise; - // Warning: (ae-forgotten-export) The symbol "LunrQueryTranslator" needs to be exported by the entry point index.d.ts - // // (undocumented) setTranslator(translator: LunrQueryTranslator): void; // (undocumented) protected translator: QueryTranslator; } -// Warning: (ae-missing-release-tag) "LunrSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { constructor(); // (undocumented) @@ -110,9 +109,18 @@ export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { initialize(): Promise; } -// Warning: (ae-missing-release-tag) "Scheduler" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta +export interface RegisterCollatorParameters { + defaultRefreshIntervalSeconds: number; + factory: DocumentCollatorFactory; +} + +// @beta +export interface RegisterDecoratorParameters { + factory: DocumentDecoratorFactory; +} + +// @beta (undocumented) export class Scheduler { constructor({ logger }: { logger: Logger_2 }); addToSchedule(task: Function, interval: number): void; @@ -122,18 +130,14 @@ export class Scheduler { export { SearchEngine }; -// Warning: (ae-missing-release-tag) "TestPipeline" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export class TestPipeline { execute(): Promise; withDocuments(documents: IndexableDocument[]): TestPipeline; static withSubject(subject: Readable | Transform | Writable): TestPipeline; } -// Warning: (ae-missing-release-tag) "TestPipelineResult" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export type TestPipelineResult = { error: unknown; documents: IndexableDocument[]; diff --git a/plugins/search-backend-node/src/IndexBuilder.ts b/plugins/search-backend-node/src/IndexBuilder.ts index 5f39c2fa33..e0a059e303 100644 --- a/plugins/search-backend-node/src/IndexBuilder.ts +++ b/plugins/search-backend-node/src/IndexBuilder.ts @@ -24,6 +24,7 @@ import { Transform, pipeline } from 'stream'; import { Logger } from 'winston'; import { Scheduler } from './index'; import { + IndexBuilderOptions, RegisterCollatorParameters, RegisterDecoratorParameters, } from './types'; @@ -33,11 +34,9 @@ interface CollatorEnvelope { refreshInterval: number; } -type IndexBuilderOptions = { - searchEngine: SearchEngine; - logger: Logger; -}; - +/** + * @beta + */ export class IndexBuilder { private collators: Record; private decorators: Record; diff --git a/plugins/search-backend-node/src/Scheduler.ts b/plugins/search-backend-node/src/Scheduler.ts index 3e356aa6aa..6debaa3dcd 100644 --- a/plugins/search-backend-node/src/Scheduler.ts +++ b/plugins/search-backend-node/src/Scheduler.ts @@ -26,6 +26,9 @@ type TaskEnvelope = { * TODO: coordination, error handling */ +/** + * @beta + */ export class Scheduler { private logger: Logger; private schedule: TaskEnvelope[]; diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts index b2e131e56f..394cfffac8 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts @@ -25,6 +25,9 @@ import lunr from 'lunr'; import { Logger } from 'winston'; import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; +/** + * @beta + */ export type ConcreteLunrQuery = { lunrQueryBuilder: lunr.Index.QueryBuilder; documentTypes?: string[]; @@ -36,8 +39,14 @@ type LunrResultEnvelope = { type: string; }; -type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery; +/** + * @beta + */ +export type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery; +/** + * @beta + */ export class LunrSearchEngine implements SearchEngine { protected lunrIndices: Record = {}; protected docStore: Record; diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts index 2454889745..01c95f07d4 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts @@ -18,6 +18,9 @@ import { IndexableDocument } from '@backstage/search-common'; import lunr from 'lunr'; import { BatchSearchEngineIndexer } from '../indexing'; +/** + * @beta + */ export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { private schemaInitialized = false; private builder: lunr.Builder; diff --git a/plugins/search-backend-node/src/engines/index.ts b/plugins/search-backend-node/src/engines/index.ts index 7b71873c64..0d710eadc2 100644 --- a/plugins/search-backend-node/src/engines/index.ts +++ b/plugins/search-backend-node/src/engines/index.ts @@ -15,5 +15,8 @@ */ export { LunrSearchEngine } from './LunrSearchEngine'; -export type { ConcreteLunrQuery } from './LunrSearchEngine'; +export type { + ConcreteLunrQuery, + LunrQueryTranslator, +} from './LunrSearchEngine'; export type { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index 6ae716553c..2edab1b01d 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -23,7 +23,16 @@ export { IndexBuilder } from './IndexBuilder'; export { Scheduler } from './Scheduler'; export { LunrSearchEngine } from './engines'; -export type { LunrSearchEngineIndexer } from './engines'; +export type { + ConcreteLunrQuery, + LunrQueryTranslator, + LunrSearchEngineIndexer, +} from './engines'; +export type { + IndexBuilderOptions, + RegisterCollatorParameters, + RegisterDecoratorParameters, +} from './types'; export * from './indexing'; export * from './test-utils'; diff --git a/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts index 4c29d0b573..b368dd8184 100644 --- a/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts +++ b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts @@ -18,6 +18,9 @@ import { assertError } from '@backstage/errors'; import { IndexableDocument } from '@backstage/search-common'; import { Writable } from 'stream'; +/** + * @beta + */ export type BatchSearchEngineOptions = { batchSize: number; }; @@ -25,6 +28,7 @@ export type BatchSearchEngineOptions = { /** * Base class encapsulating batch-based stream processing. Useful as a base * class for search engine indexers. + * @beta */ export abstract class BatchSearchEngineIndexer extends Writable { private batchSize: number; diff --git a/plugins/search-backend-node/src/indexing/DecoratorBase.ts b/plugins/search-backend-node/src/indexing/DecoratorBase.ts index a28d652fd7..382ce443e3 100644 --- a/plugins/search-backend-node/src/indexing/DecoratorBase.ts +++ b/plugins/search-backend-node/src/indexing/DecoratorBase.ts @@ -21,6 +21,7 @@ import { Transform } from 'stream'; /** * Base class encapsulating simple async transformations. Useful as a base * class for Backstage search decorators. + * @beta */ export abstract class DecoratorBase extends Transform { private initialized: Promise; diff --git a/plugins/search-backend-node/src/test-utils/TestPipeline.ts b/plugins/search-backend-node/src/test-utils/TestPipeline.ts index 2dbdeb85ad..c0c2bf5f89 100644 --- a/plugins/search-backend-node/src/test-utils/TestPipeline.ts +++ b/plugins/search-backend-node/src/test-utils/TestPipeline.ts @@ -19,6 +19,7 @@ import { pipeline, Readable, Transform, Writable } from 'stream'; /** * Object resolved after a test pipeline is executed. + * @beta */ export type TestPipelineResult = { /** @@ -36,6 +37,7 @@ export type TestPipelineResult = { /** * Test utility for Backstage Search collators, decorators, and indexers. + * @beta */ export class TestPipeline { private collator?: Readable; diff --git a/plugins/search-backend-node/src/types.ts b/plugins/search-backend-node/src/types.ts index 4bcc8ec114..68ccfe4c7e 100644 --- a/plugins/search-backend-node/src/types.ts +++ b/plugins/search-backend-node/src/types.ts @@ -17,10 +17,21 @@ import { DocumentCollatorFactory, DocumentDecoratorFactory, + SearchEngine, } from '@backstage/search-common'; +import { Logger } from 'winston'; + +/** + * @beta + */ +export type IndexBuilderOptions = { + searchEngine: SearchEngine; + logger: Logger; +}; /** * Parameters required to register a collator. + * @beta */ export interface RegisterCollatorParameters { /** @@ -36,6 +47,7 @@ export interface RegisterCollatorParameters { /** * Parameters required to register a decorator + * @beta */ export interface RegisterDecoratorParameters { /** diff --git a/scripts/api-extractor.ts b/scripts/api-extractor.ts index 6635dd8b22..afe2c1a1bb 100644 --- a/scripts/api-extractor.ts +++ b/scripts/api-extractor.ts @@ -213,6 +213,7 @@ const NO_WARNING_PACKAGES = [ 'packages/errors', 'packages/integration', 'packages/integration-react', + 'packages/search-common', 'packages/techdocs-common', 'packages/test-utils', 'packages/theme', @@ -235,6 +236,7 @@ const NO_WARNING_PACKAGES = [ 'plugins/scaffolder-backend-module-rails', 'plugins/scaffolder-backend-module-yeoman', 'plugins/scaffolder-common', + 'plugins/search-backend-node', 'plugins/techdocs-backend', 'plugins/tech-insights', 'plugins/tech-insights-backend', From 81f27861b2ddc013405291587bc2f6b528e2ec3c Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Wed, 2 Mar 2022 17:22:43 +0100 Subject: [PATCH 118/150] More clearly mark breaking changes as such. Signed-off-by: Eric Peterson --- .changeset/search-blankly-have-a-nice-life.md | 10 +++++++--- .changeset/search-done-me-no-favor.md | 6 ++++++ .changeset/search-just-smile-politely.md | 2 +- .changeset/search-selfless-cold-composed.md | 6 ++++++ .changeset/search-throw-me-a-right.md | 2 +- 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.changeset/search-blankly-have-a-nice-life.md b/.changeset/search-blankly-have-a-nice-life.md index eb6d02110c..63dc6051bd 100644 --- a/.changeset/search-blankly-have-a-nice-life.md +++ b/.changeset/search-blankly-have-a-nice-life.md @@ -3,12 +3,16 @@ '@backstage/search-common': minor --- +**BREAKING** + The Backstage Search Platform's indexing process has been rewritten as a stream pipeline in order to improve efficiency and performance on large document sets. The concepts of `Collator` and `Decorator` have been replaced with readable and transform object streams (respectively), as well as factory classes to -instantiate them. +instantiate them. Accordingly, the `SearchEngine.index()` method has also been +replaced with a `getIndexer()` factory method that resolves to a writable +object stream. -Accordingly, the `SearchEngine.index()` method has also been replaced with a -`getIndexer()` factory method that resolves to a writable object stream. +Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for further details. diff --git a/.changeset/search-done-me-no-favor.md b/.changeset/search-done-me-no-favor.md index 620c9dd923..0b4e24c4a8 100644 --- a/.changeset/search-done-me-no-favor.md +++ b/.changeset/search-done-me-no-favor.md @@ -2,5 +2,11 @@ '@backstage/plugin-search-backend-module-pg': minor --- +**BREAKING** + The `PgSearchEngine` implements the new stream-based indexing process expected by the latest `@backstage/search-backend-node`. + +When updating to this version, you must also update to the latest version of +`@backstage/search-backend-node`. Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for further details. diff --git a/.changeset/search-just-smile-politely.md b/.changeset/search-just-smile-politely.md index e1cb57a17e..a7831155da 100644 --- a/.changeset/search-just-smile-politely.md +++ b/.changeset/search-just-smile-politely.md @@ -9,5 +9,5 @@ continue to be available for those unable to upgrade to the stream-based marked as deprecated and will be removed in a future version. To upgrade this plugin and the search indexing subsystem in one go, check -[this changelog](https://github.com/backstage/backstage/blob/master/packages/create-app/CHANGELOG.md) +[this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) for necessary changes to your search backend plugin configuration. diff --git a/.changeset/search-selfless-cold-composed.md b/.changeset/search-selfless-cold-composed.md index 31c0e86baf..7e7dafd421 100644 --- a/.changeset/search-selfless-cold-composed.md +++ b/.changeset/search-selfless-cold-composed.md @@ -2,5 +2,11 @@ '@backstage/plugin-search-backend-module-elasticsearch': minor --- +**BREAKING** + The `ElasticSearchSearchEngine` implements the new stream-based indexing process expected by the latest `@backstage/search-backend-node`. + +When updating to this version, you must also update to the latest version of +`@backstage/search-backend-node`. Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for further details. diff --git a/.changeset/search-throw-me-a-right.md b/.changeset/search-throw-me-a-right.md index e504888cfb..32f4e82fe4 100644 --- a/.changeset/search-throw-me-a-right.md +++ b/.changeset/search-throw-me-a-right.md @@ -9,5 +9,5 @@ continue to be available for those unable to upgrade to the stream-based marked as deprecated and will be removed in a future version. To upgrade this plugin and the search indexing subsystem in one go, check -[this changelog](https://github.com/backstage/backstage/blob/master/packages/create-app/CHANGELOG.md) +[this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) for necessary changes to your search backend plugin configuration. From fc8b84aa2c50a96e1644cf043562cea295777dd2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 2 Mar 2022 18:22:34 +0100 Subject: [PATCH 119/150] vocab: removed old code of conduct words Signed-off-by: Patrik Oldsberg --- .github/styles/vocab.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 6359493d4b..c3097ad3f0 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -20,7 +20,6 @@ Autoscaling autoselect Avro aws -backrub backported backporting Bigtable @@ -39,8 +38,6 @@ Changesets chanwit Chanwit ci -cisphobia -cissexist classname cli cloudbuild @@ -68,7 +65,6 @@ css Datadog dataflow dayjs -deadnaming debounce Debounce declaratively @@ -171,7 +167,6 @@ Minikube Minio misconfiguration misconfigured -misgendering mkdocs Mkdocs monorepo From 8dcf469c14aae508485bcc31ad413f13a619aca3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 2 Mar 2022 19:15:02 +0100 Subject: [PATCH 120/150] home: fixed starred entities initialization in stories Signed-off-by: Patrik Oldsberg --- .../StarredEntities.stories.tsx | 24 ++++++------------- .../src/templates/DefaultTemplate.stories.tsx | 18 ++++++-------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx b/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx index 315c0d0d16..1b01820482 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx @@ -15,11 +15,7 @@ */ import { HomePageStarredEntities } from '../../plugin'; -import { - wrapInTestApp, - TestApiProvider, - MockStorageApi, -} from '@backstage/test-utils'; +import { wrapInTestApp, TestApiProvider } from '@backstage/test-utils'; import { starredEntitiesApiRef, MockStarredEntitiesApi, @@ -28,24 +24,18 @@ import { import { Grid } from '@material-ui/core'; import React, { ComponentType } from 'react'; -const mockStorageApi = MockStorageApi.create(); -mockStorageApi - .forBucket('starredEntities') - .set('entityRefs', [ - 'component:default/example-starred-entity', - 'component:default/example-starred-entity-2', - 'component:default/example-starred-entity-3', - 'component:default/example-starred-entity-4', - ]); +const starredEntitiesApi = new MockStarredEntitiesApi(); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-2'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-3'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-4'); export default { title: 'Plugins/Home/Components/StarredEntities', decorators: [ (Story: ComponentType<{}>) => wrapInTestApp( - + , { diff --git a/plugins/home/src/templates/DefaultTemplate.stories.tsx b/plugins/home/src/templates/DefaultTemplate.stories.tsx index 5dc4ff61a0..c726294ba6 100644 --- a/plugins/home/src/templates/DefaultTemplate.stories.tsx +++ b/plugins/home/src/templates/DefaultTemplate.stories.tsx @@ -21,7 +21,7 @@ import { HomePageCompanyLogo, HomePageStarredEntities, } from '../plugin'; -import { wrapInTestApp, TestApiProvider, MockStorageApi} from '@backstage/test-utils'; +import { wrapInTestApp, TestApiProvider} from '@backstage/test-utils'; import { Content, Page, InfoCard } from '@backstage/core-components'; import { starredEntitiesApiRef, @@ -37,15 +37,11 @@ import { import { Grid, makeStyles } from '@material-ui/core'; import React, { ComponentType } from 'react'; -const mockStorageApi = MockStorageApi.create(); -mockStorageApi - .forBucket('starredEntities') - .set('entityRefs', [ - 'component:default/example-starred-entity', - 'component:default/example-starred-entity-2', - 'component:default/example-starred-entity-3', - 'component:default/example-starred-entity-4' - ]); +const starredEntitiesApi = new MockStarredEntitiesApi(); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-2'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-3'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-4'); export default { title: 'Plugins/Home/Templates', @@ -57,7 +53,7 @@ export default { apis={[ [ starredEntitiesApiRef, - new MockStarredEntitiesApi(), + starredEntitiesApi, ], [searchApiRef, { query: () => Promise.resolve({ results: [] }) }], ]} From 6c7a8796601cc467dff1791f90b215b1a7f57b3d Mon Sep 17 00:00:00 2001 From: slougheed Date: Wed, 2 Mar 2022 14:15:10 -0500 Subject: [PATCH 121/150] Cleanup based on code review and added changeset Signed-off-by: slougheed --- .changeset/tidy-hairs-sip.md | 5 +++ .../src/actions/fetch/cookiecutter.ts | 39 +++++++------------ 2 files changed, 20 insertions(+), 24 deletions(-) create mode 100644 .changeset/tidy-hairs-sip.md diff --git a/.changeset/tidy-hairs-sip.md b/.changeset/tidy-hairs-sip.md new file mode 100644 index 0000000000..512447236d --- /dev/null +++ b/.changeset/tidy-hairs-sip.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend-module-cookiecutter': patch +--- + +Fixed bug where existing cookiecutter.json file is not used. diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts index efc8b6215e..01174e0f7a 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts @@ -57,39 +57,36 @@ export class CookiecutterRunner { workspacePath, values, logStream, + imageName, + templateDir, + templateContentsDir, }: { workspacePath: string; values: JsonObject; logStream: Writable; + imageName?: string; + templateDir: string; + templateContentsDir: string; }): Promise { const intermediateDir = path.join(workspacePath, 'intermediate'); await fs.ensureDir(intermediateDir); const resultDir = path.join(workspacePath, 'result'); - const { - templateContentsDir, - templateDir, - imageName, - ...valuesForCookieCutterJson - } = values; // First lets grab the default cookiecutter.json file const cookieCutterJson = await this.fetchTemplateCookieCutter( - templateContentsDir as string, + templateContentsDir, ); const cookieInfo = { ...cookieCutterJson, - ...valuesForCookieCutterJson, + ...values, }; - await fs.writeJSON( - path.join(templateDir as string, 'cookiecutter.json'), - cookieInfo, - ); + await fs.writeJSON(path.join(templateDir, 'cookiecutter.json'), cookieInfo); // Directories to bind on container const mountDirs = { - [templateDir as string]: '/input', + [templateDir]: '/input', [intermediateDir]: '/output', }; @@ -100,13 +97,7 @@ export class CookiecutterRunner { if (cookieCutterInstalled) { await runCommand({ command: 'cookiecutter', - args: [ - '--no-input', - '-o', - intermediateDir, - templateDir as string, - '--verbose', - ], + args: ['--no-input', '-o', intermediateDir, templateDir, '--verbose'], logStream, }); } else { @@ -247,16 +238,16 @@ export function createFetchCookiecutterAction(options: { ...ctx.input.values, _copy_without_render: ctx.input.copyWithoutRender, _extensions: ctx.input.extensions, - imageName: ctx.input.imageName, - templateDir: templateDir, - templateContentsDir: templateContentsDir, }; // Will execute the template in ./template and put the result in ./result await cookiecutter.run({ workspacePath: workDir, logStream: ctx.logStream, - values, + values: values, + imageName: ctx.input.imageName, + templateDir: templateDir, + templateContentsDir: templateContentsDir, }); // Finally move the template result into the task workspace From 7d533a7b5b7db11ba8ca1358a08c660cb1f86aa7 Mon Sep 17 00:00:00 2001 From: slougheed Date: Wed, 2 Mar 2022 14:25:20 -0500 Subject: [PATCH 122/150] Removed string casting of imageName Signed-off-by: slougheed --- .../src/actions/fetch/cookiecutter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts index 4e09c81311..7575627506 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts @@ -102,7 +102,7 @@ export class CookiecutterRunner { }); } else { await this.containerRunner.runContainer({ - imageName: (imageName as string) ?? 'spotify/backstage-cookiecutter', + imageName: imageName ?? 'spotify/backstage-cookiecutter', command: 'cookiecutter', args: ['--no-input', '-o', '/output', '/input', '--verbose'], mountDirs, From 36aa63022baa3671d6ed22398b04ec9b30227475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 2 Mar 2022 21:26:22 +0100 Subject: [PATCH 123/150] deprecate EntityName, introduce CompoundEntityRef deprecate getEntityName, introduce getCompoundEntityRef MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/nasty-pets-join.md | 9 +++++ .changeset/tricky-students-promise.md | 22 ++++++++++++ packages/catalog-client/api-report.md | 6 ++-- packages/catalog-client/src/CatalogClient.ts | 4 +-- packages/catalog-client/src/types/api.ts | 4 +-- packages/catalog-model/api-report.md | 26 ++++++++------ packages/catalog-model/src/entity/Entity.ts | 4 +-- packages/catalog-model/src/entity/index.ts | 1 + packages/catalog-model/src/entity/ref.ts | 25 +++++++++---- packages/catalog-model/src/index.ts | 2 +- packages/catalog-model/src/types.ts | 13 +++++-- .../techdocs-cli-embedded-app/src/apis.ts | 10 +++--- .../components/TechDocsPage/TechDocsPage.tsx | 4 +-- packages/techdocs-common/api-report.md | 6 ++-- .../src/stages/publish/awsS3.ts | 4 +-- .../src/stages/publish/azureBlobStorage.ts | 4 +-- .../src/stages/publish/googleStorage.ts | 6 ++-- .../src/stages/publish/local.ts | 4 +-- .../src/stages/publish/openStackSwift.test.ts | 4 +-- .../src/stages/publish/openStackSwift.ts | 4 +-- .../src/stages/publish/types.ts | 6 ++-- .../src/lib/catalog/CatalogIdentityClient.ts | 4 +-- plugins/catalog-backend/api-report.md | 6 ++-- plugins/catalog-backend/src/api/common.ts | 6 ++-- .../core/BuiltinKindsEntityProcessor.ts | 4 +-- plugins/catalog-graph/api-report.md | 4 +-- plugins/catalog-graph/dev/index.tsx | 6 ++-- .../CatalogGraphCard/CatalogGraphCard.tsx | 4 +-- .../CatalogGraphPage/useCatalogGraphPage.ts | 17 ++++----- .../EntityRelationsGraph.tsx | 7 ++-- plugins/catalog-import/api-report.md | 10 +++--- plugins/catalog-import/dev/index.tsx | 4 +-- .../src/api/CatalogImportApi.ts | 4 +-- .../src/api/CatalogImportClient.ts | 4 +-- .../EntityListComponent.tsx | 9 +++-- .../src/components/useImportState.test.tsx | 4 +-- .../src/components/useImportState.ts | 6 ++-- plugins/catalog-react/api-report.md | 22 +++++++----- .../EntityRefLink/EntityRefLink.tsx | 4 +-- .../EntityRefLink/EntityRefLinks.tsx | 4 +-- .../src/components/EntityRefLink/humanize.ts | 4 +-- .../src/components/EntityTable/columns.tsx | 4 +-- .../useUnregisterEntityDialogState.ts | 8 ++--- .../src/hooks/useStarredEntities.ts | 18 ++++++---- .../src/hooks/useStarredEntity.test.tsx | 4 +-- .../src/hooks/useStarredEntity.ts | 10 ++++-- .../src/utils/getEntityRelations.ts | 8 +++-- plugins/catalog-react/src/utils/isOwnerOf.ts | 4 +-- plugins/catalog/api-report.md | 6 ++-- .../src/components/CatalogTable/types.ts | 6 ++-- .../src/service/types.ts | 6 ++-- plugins/code-coverage/src/api.ts | 19 ++++++---- plugins/code-coverage/src/types.ts | 7 ++-- .../src/components/FossaPage/FossaPage.tsx | 4 +-- plugins/jenkins-backend/api-report.md | 6 ++-- .../src/service/jenkinsInfoProvider.test.ts | 4 +-- .../src/service/jenkinsInfoProvider.ts | 6 ++-- .../src/service/standaloneServer.ts | 6 ++-- plugins/jenkins/api-report.md | 14 ++++---- plugins/jenkins/src/api/JenkinsApi.ts | 14 ++++---- .../src/components/useBuildWithSteps.ts | 4 +-- plugins/jenkins/src/components/useBuilds.ts | 6 ++-- .../processor/ScaffolderEntitiesProcessor.ts | 4 +-- .../scaffolder-backend/src/service/helpers.ts | 4 +-- .../src/service/router.ts | 8 +++-- plugins/tech-insights/api-report.md | 9 +++-- .../tech-insights/src/api/TechInsightsApi.ts | 9 +++-- .../src/api/TechInsightsClient.ts | 6 ++-- .../src/service/CachedEntityLoader.test.ts | 4 +-- .../src/service/CachedEntityLoader.ts | 6 ++-- plugins/techdocs/api-report.md | 36 ++++++++++--------- plugins/techdocs/dev/index.tsx | 7 ++-- plugins/techdocs/src/api.ts | 14 ++++---- plugins/techdocs/src/client.ts | 17 +++++---- .../src/home/components/Tables/types.ts | 4 +-- .../techdocs/src/reader/components/Reader.tsx | 34 +++++++++--------- .../reader/components/TechDocsReaderPage.tsx | 6 ++-- .../components/TechDocsReaderPageHeader.tsx | 4 +-- .../src/reader/components/useRawPage.ts | 4 +-- .../src/reader/transformers/addBaseUrl.ts | 4 +-- .../src/search/components/TechDocsSearch.tsx | 4 +-- plugins/todo-backend/api-report.md | 4 +-- plugins/todo-backend/src/service/router.ts | 4 +-- plugins/todo-backend/src/service/types.ts | 4 +-- 84 files changed, 393 insertions(+), 268 deletions(-) create mode 100644 .changeset/nasty-pets-join.md create mode 100644 .changeset/tricky-students-promise.md diff --git a/.changeset/nasty-pets-join.md b/.changeset/nasty-pets-join.md new file mode 100644 index 0000000000..1ff07274cf --- /dev/null +++ b/.changeset/nasty-pets-join.md @@ -0,0 +1,9 @@ +--- +'@backstage/catalog-model': patch +--- + +**DEPRECATION**: Deprecated the `EntityName` type, and added the better-named `CompoundEntityRef` to replace it. + +**DEPRECATION**: Deprecated the `getEntityName` function, and added the better-named `getCompoundEntityRef` to replace it. + +Please switch over to using the new symbols, as the old ones may be removed in a future release. diff --git a/.changeset/tricky-students-promise.md b/.changeset/tricky-students-promise.md new file mode 100644 index 0000000000..b9c6150a4a --- /dev/null +++ b/.changeset/tricky-students-promise.md @@ -0,0 +1,22 @@ +--- +'@backstage/catalog-client': patch +'@backstage/plugin-auth-backend': patch +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-catalog-graph': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-code-coverage': patch +'@backstage/plugin-code-coverage-backend': patch +'@backstage/plugin-fossa': patch +'@backstage/plugin-jenkins': patch +'@backstage/plugin-jenkins-backend': patch +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-tech-insights': patch +'@backstage/plugin-tech-insights-backend': patch +'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs-backend': patch +'@backstage/plugin-todo-backend': patch +--- + +Use `CompoundEntityRef` instead of `EntityName`, and `getCompoundEntityRef` instead of `getEntityName`, from `@backstage/catalog-model`. diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index c957859fec..ce6397f51e 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -3,8 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; // @public export type AddLocationRequest = { @@ -39,7 +39,7 @@ export interface CatalogApi { options?: CatalogRequestOptions, ): Promise; getEntityByName( - name: EntityName, + name: CompoundEntityRef, options?: CatalogRequestOptions, ): Promise; getEntityFacets( @@ -91,7 +91,7 @@ export class CatalogClient implements CatalogApi { options?: CatalogRequestOptions, ): Promise; getEntityByName( - compoundName: EntityName, + compoundName: CompoundEntityRef, options?: CatalogRequestOptions, ): Promise; getEntityFacets( diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index bb3c9604b3..da580ae09c 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -18,7 +18,7 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, Entity, - EntityName, + CompoundEntityRef, parseEntityRef, stringifyEntityRef, stringifyLocationRef, @@ -174,7 +174,7 @@ export class CatalogClient implements CatalogApi { * {@inheritdoc CatalogApi.getEntityByName} */ async getEntityByName( - compoundName: EntityName, + compoundName: CompoundEntityRef, options?: CatalogRequestOptions, ): Promise { const { kind, namespace = 'default', name } = compoundName; diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 4a86ca41f3..0ee2b57ffb 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef, Entity } from '@backstage/catalog-model'; /** * This symbol can be used in place of a value when passed to filters in e.g. @@ -310,7 +310,7 @@ export interface CatalogApi { * @param options - Additional options */ getEntityByName( - name: EntityName, + name: CompoundEntityRef, options?: CatalogRequestOptions, ): Promise; diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 4574129d6f..99b4b39038 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -88,6 +88,13 @@ export { ComponentEntityV1alpha1 }; // @public export const componentEntityV1alpha1Validator: KindValidator; +// @public +export type CompoundEntityRef = { + kind: string; + namespace: string; + name: string; +}; + // @public export const DEFAULT_NAMESPACE = 'default'; @@ -166,12 +173,8 @@ export type EntityMeta = JsonObject & { links?: EntityLink[]; }; -// @public -export type EntityName = { - kind: string; - namespace: string; - name: string; -}; +// @public @deprecated +export type EntityName = CompoundEntityRef; // @public export const EntityPolicies: { @@ -196,7 +199,7 @@ export type EntityRef = // @public export type EntityRelation = { type: string; - target: EntityName; + target: CompoundEntityRef; targetRef: string; }; @@ -229,7 +232,10 @@ export class FieldFormatEntityPolicy implements EntityPolicy { } // @public -export function getEntityName(entity: Entity): EntityName; +export function getCompoundEntityRef(entity: Entity): CompoundEntityRef; + +// @public @deprecated +export const getEntityName: typeof getCompoundEntityRef; // @public export function getEntitySourceLocation(entity: Entity): { @@ -337,7 +343,7 @@ export function parseEntityName( defaultKind?: string; defaultNamespace?: string; }, -): EntityName; +): CompoundEntityRef; // @public export function parseEntityRef( @@ -352,7 +358,7 @@ export function parseEntityRef( defaultKind?: string; defaultNamespace?: string; }, -): EntityName; +): CompoundEntityRef; // @public export function parseLocationRef(ref: string): { diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts index ef9b4352da..029d9e095b 100644 --- a/packages/catalog-model/src/entity/Entity.ts +++ b/packages/catalog-model/src/entity/Entity.ts @@ -15,7 +15,7 @@ */ import { JsonObject } from '@backstage/types'; -import { EntityName } from '../types'; +import { CompoundEntityRef } from '../types'; import { EntityStatus } from './EntityStatus'; /** @@ -201,7 +201,7 @@ export type EntityRelation = { * * @deprecated use targetRef instead */ - target: EntityName; + target: CompoundEntityRef; /** * The entity ref of the target of this relation. diff --git a/packages/catalog-model/src/entity/index.ts b/packages/catalog-model/src/entity/index.ts index 3a346b217c..fb9eee0353 100644 --- a/packages/catalog-model/src/entity/index.ts +++ b/packages/catalog-model/src/entity/index.ts @@ -34,6 +34,7 @@ export type { } from './EntityStatus'; export * from './policies'; export { + getCompoundEntityRef, getEntityName, parseEntityName, parseEntityRef, diff --git a/packages/catalog-model/src/entity/ref.ts b/packages/catalog-model/src/entity/ref.ts index f9bfcf38cd..4529b1174c 100644 --- a/packages/catalog-model/src/entity/ref.ts +++ b/packages/catalog-model/src/entity/ref.ts @@ -15,7 +15,7 @@ */ import { DEFAULT_NAMESPACE } from './constants'; -import { EntityName } from '../types'; +import { CompoundEntityRef } from '../types'; import { Entity } from './Entity'; function parseRefString(ref: string): { @@ -38,14 +38,25 @@ function parseRefString(ref: string): { } /** - * Extracts the kind, namespace and name that form the name triplet of the - * given entity. + * Extracts the kind, namespace and name that form the compound entity ref + * triplet of the given entity. + * + * @public + * @deprecated Use getCompoundEntityRef instead + * @param entity - An entity + * @returns The compound entity ref + */ +export const getEntityName = getCompoundEntityRef; + +/** + * Extracts the kind, namespace and name that form the compound entity ref + * triplet of the given entity. * * @public * @param entity - An entity - * @returns The complete entity name + * @returns The compound entity ref */ -export function getEntityName(entity: Entity): EntityName { +export function getCompoundEntityRef(entity: Entity): CompoundEntityRef { return { kind: entity.kind, namespace: entity.metadata.namespace || DEFAULT_NAMESPACE, @@ -77,7 +88,7 @@ export function parseEntityName( /** The default namespace, if none is given in the reference */ defaultNamespace?: string; } = {}, -): EntityName { +): CompoundEntityRef { const { kind, namespace, name } = parseEntityRef(ref, { defaultNamespace: DEFAULT_NAMESPACE, ...context, @@ -114,7 +125,7 @@ export function parseEntityRef( /** The default namespace, if none is given in the reference */ defaultNamespace?: string; }, -): EntityName { +): CompoundEntityRef { if (!ref) { throw new Error(`Entity reference must not be empty`); } diff --git a/packages/catalog-model/src/index.ts b/packages/catalog-model/src/index.ts index 986341bd33..19eacd4925 100644 --- a/packages/catalog-model/src/index.ts +++ b/packages/catalog-model/src/index.ts @@ -24,5 +24,5 @@ export * from './entity'; export { EntityPolicies } from './EntityPolicies'; export * from './kinds'; export * from './location'; -export type { EntityName, EntityRef } from './types'; +export type { EntityName, EntityRef, CompoundEntityRef } from './types'; export * from './validation'; diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts index 5b5f063ca3..e7d819ace9 100644 --- a/packages/catalog-model/src/types.ts +++ b/packages/catalog-model/src/types.ts @@ -15,16 +15,25 @@ */ /** - * A complete entity name, with the full kind-namespace-name triplet. + * All parts of a complete entity ref, forming a full kind-namespace-name + * triplet. * * @public */ -export type EntityName = { +export type CompoundEntityRef = { kind: string; namespace: string; name: string; }; +/** + * A complete entity name, with the full kind-namespace-name triplet. + * + * @deprecated Use CompoundEntityRef instead + * @public + */ +export type EntityName = CompoundEntityRef; + /** * A reference by name to an entity, either as a compact string representation, * or as a compound reference structure. diff --git a/packages/techdocs-cli-embedded-app/src/apis.ts b/packages/techdocs-cli-embedded-app/src/apis.ts index 6234e3b719..1fe6ba8086 100644 --- a/packages/techdocs-cli-embedded-app/src/apis.ts +++ b/packages/techdocs-cli-embedded-app/src/apis.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { scmIntegrationsApiRef, @@ -81,7 +81,7 @@ class TechDocsDevStorageApi implements TechDocsStorageApi { return this.configApi.getString('techdocs.builder'); } - async getEntityDocs(_entityId: EntityName, path: string) { + async getEntityDocs(_entityId: CompoundEntityRef, path: string) { const apiOrigin = await this.getApiOrigin(); // Irrespective of the entity, use mkdocs server to find the file for the path. const url = `${apiOrigin}/${path}`; @@ -97,7 +97,7 @@ class TechDocsDevStorageApi implements TechDocsStorageApi { return request.text(); } - async syncEntityDocs(_: EntityName): Promise { + async syncEntityDocs(_: CompoundEntityRef): Promise { // this is just stub of this function as we don't need to check if docs are up to date, // we always want to retrigger a new build return 'cached'; @@ -106,7 +106,7 @@ class TechDocsDevStorageApi implements TechDocsStorageApi { // Used by transformer to modify the request to assets (CSS, Image) from inside the HTML. async getBaseUrl( oldBaseUrl: string, - _entityId: EntityName, + _entityId: CompoundEntityRef, path: string, ): Promise { const apiOrigin = await this.getApiOrigin(); @@ -154,7 +154,7 @@ class TechDocsDevApi implements TechDocsApi { }; } - async getTechDocsMetadata(_entityId: EntityName) { + async getTechDocsMetadata(_entityId: CompoundEntityRef) { return { site_name: 'Live preview environment', site_description: '', diff --git a/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx b/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx index d9a518c165..f2dcefc00e 100644 --- a/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx +++ b/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx @@ -29,7 +29,7 @@ import LightIcon from '@material-ui/icons/Brightness7'; import DarkIcon from '@material-ui/icons/Brightness4'; import { lightTheme, darkTheme } from '@backstage/theme'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Content } from '@backstage/core-components'; @@ -127,7 +127,7 @@ const TechDocsPageContent = ({ onReady, entityRef, }: { - entityRef: EntityName; + entityRef: CompoundEntityRef; onReady: () => void; }) => { const classes = useStyles(); diff --git a/packages/techdocs-common/api-report.md b/packages/techdocs-common/api-report.md index 7c4fc95abb..13d610f66b 100644 --- a/packages/techdocs-common/api-report.md +++ b/packages/techdocs-common/api-report.md @@ -5,10 +5,10 @@ ```ts /// +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { ContainerRunner } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import express from 'express'; import { IndexableDocument } from '@backstage/search-common'; import { Logger as Logger_2 } from 'winston'; @@ -157,7 +157,9 @@ export class Publisher { // @public export interface PublisherBase { docsRouter(): express.Handler; - fetchTechDocsMetadata(entityName: EntityName): Promise; + fetchTechDocsMetadata( + entityName: CompoundEntityRef, + ): Promise; getReadiness(): Promise; hasDocsBeenGenerated(entityName: Entity): Promise; migrateDocsCase?(migrateRequest: MigrateRequest): Promise; diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index ed76edc0cb..e5914d2fdf 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, ForwardedError } from '@backstage/errors'; import aws, { Credentials } from 'aws-sdk'; @@ -321,7 +321,7 @@ export class AwsS3Publish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { try { return await new Promise(async (resolve, reject) => { diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index b082079be2..bcbc10a94d 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -19,7 +19,7 @@ import { ContainerClient, StorageSharedKeyCredential, } from '@azure/storage-blob'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, ForwardedError } from '@backstage/errors'; import express from 'express'; @@ -300,7 +300,7 @@ export class AzureBlobStoragePublish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; const entityRootDir = this.legacyPathCasing diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index c3d9d3330b..987dfc6f09 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError } from '@backstage/errors'; import { File, FileExistsResponse, Storage } from '@google-cloud/storage'; @@ -238,7 +238,9 @@ export class GoogleGCSPublish implements PublisherBase { return { objects }; } - fetchTechDocsMetadata(entityName: EntityName): Promise { + fetchTechDocsMetadata( + entityName: CompoundEntityRef, + ): Promise { return new Promise((resolve, reject) => { const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; const entityDir = this.legacyPathCasing diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 9c146d2935..af9bb30d46 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -17,7 +17,7 @@ import { PluginEndpointDiscovery, resolvePackagePath, } from '@backstage/backend-common'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import express from 'express'; import fs from 'fs-extra'; @@ -142,7 +142,7 @@ export class LocalPublish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { const metadataPath = this.staticEntityPathJoin( entityName.namespace, diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts index b240ce73da..da0a60343f 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; @@ -45,7 +45,7 @@ const createMockEntity = (annotations = {}): Entity => { }; }; -const createMockEntityName = (): EntityName => ({ +const createMockEntityName = (): CompoundEntityRef => ({ kind: 'TestKind', name: 'test-component-name', namespace: 'test-namespace', diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.ts index 62b40f9a76..734a7e4451 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import express from 'express'; import fs from 'fs-extra'; @@ -194,7 +194,7 @@ export class OpenStackSwiftPublish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { return await new Promise(async (resolve, reject) => { const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index c6ad032484..86772c25f4 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { Logger } from 'winston'; import express from 'express'; @@ -133,7 +133,9 @@ export interface PublisherBase { * Retrieve TechDocs Metadata about a site e.g. name, contributors, last updated, etc. * This API uses the techdocs_metadata.json file that co-exists along with the generated docs. */ - fetchTechDocsMetadata(entityName: EntityName): Promise; + fetchTechDocsMetadata( + entityName: CompoundEntityRef, + ): Promise; /** * Route middleware to serve static documentation files for an entity. diff --git a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts index d948af2e05..502da913af 100644 --- a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts +++ b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts @@ -18,7 +18,7 @@ import { Logger } from 'winston'; import { ConflictError, NotFoundError } from '@backstage/errors'; import { CatalogApi } from '@backstage/catalog-client'; import { - EntityName, + CompoundEntityRef, parseEntityRef, RELATION_MEMBER_OF, stringifyEntityRef, @@ -96,7 +96,7 @@ export class CatalogIdentityClient { return null; } }) - .filter((ref): ref is EntityName => ref !== null); + .filter((ref): ref is CompoundEntityRef => ref !== null); const filter = resolvedEntityRefs.map(ref => ({ kind: ref.kind, diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index d5541678e6..ee7b419a71 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -7,12 +7,12 @@ import { BitbucketIntegration } from '@backstage/integration'; import { CatalogApi } from '@backstage/catalog-client'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConditionalPolicyDecision } from '@backstage/plugin-permission-node'; import { Conditions } from '@backstage/plugin-permission-node'; import { Config } from '@backstage/config'; import { DocumentCollatorFactory } from '@backstage/search-common'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; import express from 'express'; import { GetEntitiesRequest } from '@backstage/catalog-client'; @@ -649,9 +649,9 @@ export type EntityProviderMutation = // @public export type EntityRelationSpec = { - source: EntityName; + source: CompoundEntityRef; type: string; - target: EntityName; + target: CompoundEntityRef; }; // @public (undocumented) diff --git a/plugins/catalog-backend/src/api/common.ts b/plugins/catalog-backend/src/api/common.ts index 1fd09aec32..f3b4a387ba 100644 --- a/plugins/catalog-backend/src/api/common.ts +++ b/plugins/catalog-backend/src/api/common.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; /** * Holds the entity location information. @@ -42,7 +42,7 @@ export type EntityRelationSpec = { /** * The source entity of this relation. */ - source: EntityName; + source: CompoundEntityRef; /** * The type of the relation. @@ -52,5 +52,5 @@ export type EntityRelationSpec = { /** * The target entity of this relation. */ - target: EntityName; + target: CompoundEntityRef; }; diff --git a/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts index 548cdf3115..9e420b9d65 100644 --- a/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts @@ -22,7 +22,7 @@ import { DomainEntity, domainEntityV1alpha1Validator, Entity, - getEntityName, + getCompoundEntityRef, GroupEntity, groupEntityV1alpha1Validator, locationEntityV1alpha1Validator, @@ -93,7 +93,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { _location: LocationSpec, emit: CatalogProcessorEmit, ): Promise { - const selfRef = getEntityName(entity); + const selfRef = getCompoundEntityRef(entity); /* * Utilities diff --git a/plugins/catalog-graph/api-report.md b/plugins/catalog-graph/api-report.md index 515a578d3c..82db1dac3b 100644 --- a/plugins/catalog-graph/api-report.md +++ b/plugins/catalog-graph/api-report.md @@ -6,8 +6,8 @@ /// import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { DependencyGraphTypes } from '@backstage/core-components'; -import { EntityName } from '@backstage/catalog-model'; import { ExternalRouteRef } from '@backstage/core-plugin-api'; import { InfoCardVariants } from '@backstage/core-components'; import { MouseEvent as MouseEvent_2 } from 'react'; @@ -129,7 +129,7 @@ export const EntityRelationsGraph: ({ className, zoom, }: { - rootEntityNames: EntityName | EntityName[]; + rootEntityNames: CompoundEntityRef | CompoundEntityRef[]; maxDepth?: number | undefined; unidirectional?: boolean | undefined; mergeRelations?: boolean | undefined; diff --git a/plugins/catalog-graph/dev/index.tsx b/plugins/catalog-graph/dev/index.tsx index 62d5eccef3..7413634497 100644 --- a/plugins/catalog-graph/dev/index.tsx +++ b/plugins/catalog-graph/dev/index.tsx @@ -16,7 +16,7 @@ import { GetEntitiesResponse } from '@backstage/catalog-client'; import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, RELATION_API_CONSUMED_BY, RELATION_API_PROVIDED_BY, @@ -139,7 +139,9 @@ createDevApp() deps: {}, factory() { return { - async getEntityByName(name: EntityName): Promise { + async getEntityByName( + name: CompoundEntityRef, + ): Promise { return entities[stringifyEntityRef(name)]; }, async getEntities(): Promise { diff --git a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx index e6811aad62..4a21418107 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ import { - getEntityName, + getCompoundEntityRef, parseEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; @@ -77,7 +77,7 @@ export const CatalogGraphCard = ({ zoom?: 'enabled' | 'disabled' | 'enable-on-click'; }) => { const { entity } = useEntity(); - const entityName = getEntityName(entity); + const entityName = getCompoundEntityRef(entity); const catalogEntityRoute = useRouteRef(entityRouteRef); const catalogGraphRoute = useRouteRef(catalogGraphRouteRef); const navigate = useNavigate(); diff --git a/plugins/catalog-graph/src/components/CatalogGraphPage/useCatalogGraphPage.ts b/plugins/catalog-graph/src/components/CatalogGraphPage/useCatalogGraphPage.ts index 198cbd397f..308418f612 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphPage/useCatalogGraphPage.ts +++ b/plugins/catalog-graph/src/components/CatalogGraphPage/useCatalogGraphPage.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { - EntityName, + CompoundEntityRef, parseEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; @@ -32,8 +32,8 @@ import usePrevious from 'react-use/lib/usePrevious'; import { Direction } from '../EntityRelationsGraph'; export type CatalogGraphPageValue = { - rootEntityNames: EntityName[]; - setRootEntityNames: Dispatch>; + rootEntityNames: CompoundEntityRef[]; + setRootEntityNames: Dispatch>; maxDepth: number; setMaxDepth: Dispatch>; selectedRelations: string[] | undefined; @@ -82,11 +82,12 @@ export function useCatalogGraphPage({ ); // Initial state - const [rootEntityNames, setRootEntityNames] = useState(() => - (Array.isArray(query.rootEntityRefs) - ? query.rootEntityRefs - : initialState?.rootEntityRefs ?? [] - ).map(r => parseEntityRef(r)), + const [rootEntityNames, setRootEntityNames] = useState( + () => + (Array.isArray(query.rootEntityRefs) + ? query.rootEntityRefs + : initialState?.rootEntityRefs ?? [] + ).map(r => parseEntityRef(r)), ); const [maxDepth, setMaxDepth] = useState(() => typeof query.maxDepth === 'string' diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx index 76cd44394a..c5c83a54cd 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName, stringifyEntityRef } from '@backstage/catalog-model'; +import { + CompoundEntityRef, + stringifyEntityRef, +} from '@backstage/catalog-model'; import { DependencyGraph, DependencyGraphTypes, @@ -77,7 +80,7 @@ export const EntityRelationsGraph = ({ className, zoom = 'enabled', }: { - rootEntityNames: EntityName | EntityName[]; + rootEntityNames: CompoundEntityRef | CompoundEntityRef[]; maxDepth?: number; unidirectional?: boolean; mergeRelations?: boolean; diff --git a/plugins/catalog-import/api-report.md b/plugins/catalog-import/api-report.md index 164d831c96..5a9e8f03fd 100644 --- a/plugins/catalog-import/api-report.md +++ b/plugins/catalog-import/api-report.md @@ -8,11 +8,11 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; import { CatalogApi } from '@backstage/catalog-client'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConfigApi } from '@backstage/core-plugin-api'; import { Controller } from 'react-hook-form'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { FieldErrors } from 'react-hook-form'; import { IdentityApi } from '@backstage/core-plugin-api'; import { InfoCardVariants } from '@backstage/core-components'; @@ -33,7 +33,7 @@ export type AnalyzeResult = locations: Array<{ target: string; exists?: boolean; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { @@ -165,7 +165,7 @@ export interface EntityListComponentProps { // (undocumented) locations: Array<{ target: string; - entities: (Entity | EntityName)[]; + entities: (Entity | CompoundEntityRef)[]; }>; // (undocumented) onItemClick?: (target: string) => void; @@ -246,7 +246,7 @@ export type PrepareResult = locations: Array<{ exists?: boolean; target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { @@ -258,7 +258,7 @@ export type PrepareResult = }; locations: Array<{ target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; }; diff --git a/plugins/catalog-import/dev/index.tsx b/plugins/catalog-import/dev/index.tsx index a0cf034782..e2e5e1d22c 100644 --- a/plugins/catalog-import/dev/index.tsx +++ b/plugins/catalog-import/dev/index.tsx @@ -15,7 +15,7 @@ */ import { CatalogApi } from '@backstage/catalog-client'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { createDevApp } from '@backstage/dev-utils'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { Grid, ListItem, ListItemIcon, ListItemText } from '@material-ui/core'; @@ -32,7 +32,7 @@ import { import { ImportPage } from '../src/components/ImportPage'; import { Content, Header, InfoCard, Page } from '@backstage/core-components'; -const getEntityNames = (url: string): EntityName[] => [ +const getEntityNames = (url: string): CompoundEntityRef[] => [ { kind: 'Component', namespace: url.replace(/^.*(folder-[^/]+).*|.*()$/, '$1') || 'default', diff --git a/plugins/catalog-import/src/api/CatalogImportApi.ts b/plugins/catalog-import/src/api/CatalogImportApi.ts index 0923f4ac98..3cfd53470a 100644 --- a/plugins/catalog-import/src/api/CatalogImportApi.ts +++ b/plugins/catalog-import/src/api/CatalogImportApi.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { createApiRef } from '@backstage/core-plugin-api'; import { PartialEntity } from '../types'; @@ -38,7 +38,7 @@ export type AnalyzeResult = locations: Array<{ target: string; exists?: boolean; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { diff --git a/plugins/catalog-import/src/api/CatalogImportClient.ts b/plugins/catalog-import/src/api/CatalogImportClient.ts index b856a4e359..c12e391ec9 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.ts @@ -15,7 +15,7 @@ */ import { CatalogApi } from '@backstage/catalog-client'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConfigApi, DiscoveryApi, @@ -212,7 +212,7 @@ the component will become available.\n\nFor more information, read an \ }): Promise< Array<{ target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }> > { const { url, owner, repo, githubIntegrationConfig } = options; diff --git a/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx b/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx index ceec4b97e8..47f0139e5e 100644 --- a/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx +++ b/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { useApp } from '@backstage/core-plugin-api'; import { EntityRefLink, @@ -41,7 +41,7 @@ const useStyles = makeStyles(theme => ({ }, })); -function sortEntities(entities: Array) { +function sortEntities(entities: Array) { return entities.sort((a, b) => humanizeEntityRef(a).localeCompare(humanizeEntityRef(b)), ); @@ -53,7 +53,10 @@ function sortEntities(entities: Array) { * @public */ export interface EntityListComponentProps { - locations: Array<{ target: string; entities: (Entity | EntityName)[] }>; + locations: Array<{ + target: string; + entities: (Entity | CompoundEntityRef)[]; + }>; locationListItemIcon: (target: string) => React.ReactElement; collapsed?: boolean; firstListItem?: React.ReactElement; diff --git a/plugins/catalog-import/src/components/useImportState.test.tsx b/plugins/catalog-import/src/components/useImportState.test.tsx index 1a01c61449..155adb8290 100644 --- a/plugins/catalog-import/src/components/useImportState.test.tsx +++ b/plugins/catalog-import/src/components/useImportState.test.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { cleanup } from '@testing-library/react'; import { act, renderHook } from '@testing-library/react-hooks'; import { AnalyzeResult } from '../api'; @@ -37,7 +37,7 @@ describe('useImportState', () => { locations: [ { target: 'https://0', - entities: [] as EntityName[], + entities: [] as CompoundEntityRef[], }, ], }; diff --git a/plugins/catalog-import/src/components/useImportState.ts b/plugins/catalog-import/src/components/useImportState.ts index 0cb6b59917..965f855f30 100644 --- a/plugins/catalog-import/src/components/useImportState.ts +++ b/plugins/catalog-import/src/components/useImportState.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { useReducer } from 'react'; import { AnalyzeResult } from '../api'; @@ -43,7 +43,7 @@ export type PrepareResult = locations: Array<{ exists?: boolean; target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { @@ -55,7 +55,7 @@ export type PrepareResult = }; locations: Array<{ target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; }; diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 394a45f278..75414e3d59 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -11,8 +11,8 @@ import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; import { CatalogApi } from '@backstage/catalog-client'; import { ComponentEntity } from '@backstage/catalog-model'; import { ComponentProps } from 'react'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { GetEntitiesResponse } from '@backstage/catalog-client'; import { IconButton } from '@material-ui/core'; import { LinkProps } from '@backstage/core-components'; @@ -250,7 +250,7 @@ export const EntityRefLink: (props: EntityRefLinkProps) => JSX.Element; // @public export type EntityRefLinkProps = { - entityRef: Entity | EntityName | string; + entityRef: Entity | CompoundEntityRef | string; defaultKind?: string; title?: string; children?: React_2.ReactNode; @@ -265,7 +265,7 @@ export const EntityRefLinks: ({ // @public export type EntityRefLinksProps = { - entityRefs: (Entity | EntityName)[]; + entityRefs: (Entity | CompoundEntityRef)[]; defaultKind?: string; } & Omit; @@ -429,7 +429,7 @@ export function getEntityRelations( filter?: { kind: string; }, -): EntityName[]; +): CompoundEntityRef[]; // @public (undocumented) export function getEntitySourceLocation( @@ -439,7 +439,7 @@ export function getEntitySourceLocation( // @public (undocumented) export function humanizeEntityRef( - entityRef: Entity | EntityName, + entityRef: Entity | CompoundEntityRef, opts?: { defaultKind?: string; }, @@ -627,12 +627,18 @@ export type UserListPickerProps = { // @public (undocumented) export function useStarredEntities(): { starredEntities: Set; - toggleStarredEntity: (entityOrRef: Entity | EntityName | string) => void; - isStarredEntity: (entityOrRef: Entity | EntityName | string) => boolean; + toggleStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => void; + isStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => boolean; }; // @public (undocumented) -export function useStarredEntity(entityOrRef: Entity | EntityName | string): { +export function useStarredEntity( + entityOrRef: Entity | CompoundEntityRef | string, +): { toggleStarredEntity: () => void; isStarredEntity: boolean; }; diff --git a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx index 4db48351a9..06a632232e 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx +++ b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx @@ -16,7 +16,7 @@ import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, parseEntityRef, } from '@backstage/catalog-model'; @@ -33,7 +33,7 @@ import { Tooltip } from '@material-ui/core'; * @public */ export type EntityRefLinkProps = { - entityRef: Entity | EntityName | string; + entityRef: Entity | CompoundEntityRef | string; defaultKind?: string; title?: string; children?: React.ReactNode; diff --git a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx index 9be970aa56..d92cd2ec87 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx +++ b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import React from 'react'; import { EntityRefLink } from './EntityRefLink'; import { LinkProps } from '@backstage/core-components'; @@ -25,7 +25,7 @@ import { LinkProps } from '@backstage/core-components'; * @public */ export type EntityRefLinksProps = { - entityRefs: (Entity | EntityName)[]; + entityRefs: (Entity | CompoundEntityRef)[]; defaultKind?: string; } & Omit; diff --git a/plugins/catalog-react/src/components/EntityRefLink/humanize.ts b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts index e2984c34ab..0ade1da0b9 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/humanize.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts @@ -16,7 +16,7 @@ import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; @@ -25,7 +25,7 @@ export const formatEntityRefTitle = humanizeEntityRef; /** @public */ export function humanizeEntityRef( - entityRef: Entity | EntityName, + entityRef: Entity | CompoundEntityRef, opts?: { defaultKind?: string }, ) { const defaultKind = opts?.defaultKind; diff --git a/plugins/catalog-react/src/components/EntityTable/columns.tsx b/plugins/catalog-react/src/components/EntityTable/columns.tsx index d5ef791831..08eb31bea3 100644 --- a/plugins/catalog-react/src/components/EntityTable/columns.tsx +++ b/plugins/catalog-react/src/components/EntityTable/columns.tsx @@ -16,7 +16,7 @@ import { Entity, - EntityName, + CompoundEntityRef, RELATION_OWNED_BY, RELATION_PART_OF, } from '@backstage/catalog-model'; @@ -81,7 +81,7 @@ export const columnFactories = Object.freeze({ defaultKind?: string; filter?: { kind: string }; }): TableColumn { - function getRelations(entity: T): EntityName[] { + function getRelations(entity: T): CompoundEntityRef[] { return getEntityRelations(entity, relation, entityFilter); } diff --git a/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts b/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts index 6420ebd5ae..c14b6523c0 100644 --- a/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts +++ b/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts @@ -16,8 +16,8 @@ import { Entity, - EntityName, - getEntityName, + CompoundEntityRef, + getCompoundEntityRef, ANNOTATION_ORIGIN_LOCATION, } from '@backstage/catalog-model'; import { catalogApiRef } from '../../api'; @@ -44,7 +44,7 @@ export type UseUnregisterEntityDialogState = | { type: 'unregister'; location: string; - colocatedEntities: EntityName[]; + colocatedEntities: CompoundEntityRef[]; unregisterLocation: () => Promise; deleteEntity: () => Promise; } @@ -141,7 +141,7 @@ export function useUnregisterEntityDialogState( return { type: 'unregister', location: locationRef!, - colocatedEntities: colocatedEntities.map(getEntityName), + colocatedEntities: colocatedEntities.map(getCompoundEntityRef), unregisterLocation, deleteEntity, }; diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.ts b/plugins/catalog-react/src/hooks/useStarredEntities.ts index 5ddabfdd2e..328a26e0f8 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.ts +++ b/plugins/catalog-react/src/hooks/useStarredEntities.ts @@ -16,7 +16,7 @@ import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; import { useApi } from '@backstage/core-plugin-api'; @@ -24,7 +24,9 @@ import { useCallback } from 'react'; import useObservable from 'react-use/lib/useObservable'; import { starredEntitiesApiRef } from '../apis'; -function getEntityRef(entityOrRef: Entity | EntityName | string): string { +function getEntityRef( + entityOrRef: Entity | CompoundEntityRef | string, +): string { return typeof entityOrRef === 'string' ? entityOrRef : stringifyEntityRef(entityOrRef); @@ -33,8 +35,12 @@ function getEntityRef(entityOrRef: Entity | EntityName | string): string { /** @public */ export function useStarredEntities(): { starredEntities: Set; - toggleStarredEntity: (entityOrRef: Entity | EntityName | string) => void; - isStarredEntity: (entityOrRef: Entity | EntityName | string) => boolean; + toggleStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => void; + isStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => boolean; } { const starredEntitiesApi = useApi(starredEntitiesApiRef); @@ -44,13 +50,13 @@ export function useStarredEntities(): { ); const isStarredEntity = useCallback( - (entityOrRef: Entity | EntityName | string) => + (entityOrRef: Entity | CompoundEntityRef | string) => starredEntities.has(getEntityRef(entityOrRef)), [starredEntities], ); const toggleStarredEntity = useCallback( - (entityOrRef: Entity | EntityName | string) => + (entityOrRef: Entity | CompoundEntityRef | string) => starredEntitiesApi.toggleStarred(getEntityRef(entityOrRef)).then(), [starredEntitiesApi], ); diff --git a/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx b/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx index 8ffc891092..317a0e391c 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx +++ b/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { TestApiProvider } from '@backstage/test-utils'; import { renderHook } from '@testing-library/react-hooks'; import React, { PropsWithChildren } from 'react'; @@ -44,7 +44,7 @@ describe('useStarredEntity', () => { describe.each` title | entityOrRef ${'entity reference'} | ${'component:default/mock'} - ${'entity name'} | ${{ kind: 'component', namespace: 'default', name: 'mock' } as EntityName} + ${'entity name'} | ${{ kind: 'component', namespace: 'default', name: 'mock' } as CompoundEntityRef} ${'entity'} | ${{ apiVersion: '1', kind: 'Component', metadata: { name: 'mock' } } as Entity} `('with $title', ({ entityOrRef }) => { describe('toggleStarredEntity', () => { diff --git a/plugins/catalog-react/src/hooks/useStarredEntity.ts b/plugins/catalog-react/src/hooks/useStarredEntity.ts index 2c647163aa..23d5677512 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntity.ts +++ b/plugins/catalog-react/src/hooks/useStarredEntity.ts @@ -16,21 +16,25 @@ import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; import { useApi } from '@backstage/core-plugin-api'; import { useCallback, useEffect, useState } from 'react'; import { starredEntitiesApiRef } from '../apis'; -function getEntityRef(entityOrRef: Entity | EntityName | string): string { +function getEntityRef( + entityOrRef: Entity | CompoundEntityRef | string, +): string { return typeof entityOrRef === 'string' ? entityOrRef : stringifyEntityRef(entityOrRef); } /** @public */ -export function useStarredEntity(entityOrRef: Entity | EntityName | string): { +export function useStarredEntity( + entityOrRef: Entity | CompoundEntityRef | string, +): { toggleStarredEntity: () => void; isStarredEntity: boolean; } { diff --git a/plugins/catalog-react/src/utils/getEntityRelations.ts b/plugins/catalog-react/src/utils/getEntityRelations.ts index 45cbc6967f..13086db4ea 100644 --- a/plugins/catalog-react/src/utils/getEntityRelations.ts +++ b/plugins/catalog-react/src/utils/getEntityRelations.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { Entity, EntityName, parseEntityRef } from '@backstage/catalog-model'; +import { + Entity, + CompoundEntityRef, + parseEntityRef, +} from '@backstage/catalog-model'; // TODO(freben): This should be returning entity refs instead /** @@ -26,7 +30,7 @@ export function getEntityRelations( entity: Entity | undefined, relationType: string, filter?: { kind: string }, -): EntityName[] { +): CompoundEntityRef[] { let entityNames = entity?.relations ?.filter(r => r.type === relationType) diff --git a/plugins/catalog-react/src/utils/isOwnerOf.ts b/plugins/catalog-react/src/utils/isOwnerOf.ts index 3ef6b4b2a0..f1ba0fa35f 100644 --- a/plugins/catalog-react/src/utils/isOwnerOf.ts +++ b/plugins/catalog-react/src/utils/isOwnerOf.ts @@ -16,7 +16,7 @@ import { Entity, - getEntityName, + getCompoundEntityRef, RELATION_MEMBER_OF, RELATION_OWNED_BY, stringifyEntityRef, @@ -31,7 +31,7 @@ export function isOwnerOf(owner: Entity, owned: Entity) { const possibleOwners = new Set( [ ...getEntityRelations(owner, RELATION_MEMBER_OF, { kind: 'group' }), - ...(owner ? [getEntityName(owner)] : []), + ...(owner ? [getCompoundEntityRef(owner)] : []), ].map(stringifyEntityRef), ); diff --git a/plugins/catalog/api-report.md b/plugins/catalog/api-report.md index f314932b9a..97a36ad020 100644 --- a/plugins/catalog/api-report.md +++ b/plugins/catalog/api-report.md @@ -7,8 +7,8 @@ import { ApiHolder } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { ExternalRouteRef } from '@backstage/core-plugin-api'; import { IconComponent } from '@backstage/core-plugin-api'; import { IndexableDocument } from '@backstage/search-common'; @@ -157,9 +157,9 @@ export interface CatalogTableRow { resolved: { name: string; partOfSystemRelationTitle?: string; - partOfSystemRelations: EntityName[]; + partOfSystemRelations: CompoundEntityRef[]; ownedByRelationsTitle?: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; } diff --git a/plugins/catalog/src/components/CatalogTable/types.ts b/plugins/catalog/src/components/CatalogTable/types.ts index 7eaff6faed..5bbb3af122 100644 --- a/plugins/catalog/src/components/CatalogTable/types.ts +++ b/plugins/catalog/src/components/CatalogTable/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; /** @public */ export interface CatalogTableRow { @@ -22,8 +22,8 @@ export interface CatalogTableRow { resolved: { name: string; partOfSystemRelationTitle?: string; - partOfSystemRelations: EntityName[]; + partOfSystemRelations: CompoundEntityRef[]; ownedByRelationsTitle?: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; } diff --git a/plugins/code-coverage-backend/src/service/types.ts b/plugins/code-coverage-backend/src/service/types.ts index c4c622bfaf..31edad88d1 100644 --- a/plugins/code-coverage-backend/src/service/types.ts +++ b/plugins/code-coverage-backend/src/service/types.ts @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; export type JsonCodeCoverage = { metadata: CoverageMetadata; - entity: EntityName; + entity: CompoundEntityRef; files: Array; }; export type JsonCoverageHistory = { - entity: EntityName; + entity: CompoundEntityRef; history: Array; }; diff --git a/plugins/code-coverage/src/api.ts b/plugins/code-coverage/src/api.ts index 61f920106b..081f1c7999 100644 --- a/plugins/code-coverage/src/api.ts +++ b/plugins/code-coverage/src/api.ts @@ -14,20 +14,25 @@ * limitations under the License. */ -import { EntityName, stringifyEntityRef } from '@backstage/catalog-model'; +import { + CompoundEntityRef, + stringifyEntityRef, +} from '@backstage/catalog-model'; import { ResponseError } from '@backstage/errors'; import { JsonCodeCoverage, JsonCoverageHistory } from './types'; import { createApiRef, DiscoveryApi } from '@backstage/core-plugin-api'; export type CodeCoverageApi = { discovery: DiscoveryApi; - getCoverageForEntity: (entity: EntityName) => Promise; + getCoverageForEntity: ( + entity: CompoundEntityRef, + ) => Promise; getFileContentFromEntity: ( - entity: EntityName, + entity: CompoundEntityRef, filePath: string, ) => Promise; getCoverageHistoryForEntity: ( - entity: EntityName, + entity: CompoundEntityRef, limit?: number, ) => Promise; }; @@ -59,7 +64,7 @@ export class CodeCoverageRestApi implements CodeCoverageApi { } async getCoverageForEntity( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { const entity = encodeURIComponent(stringifyEntityRef(entityName)); return (await this.fetch( @@ -68,7 +73,7 @@ export class CodeCoverageRestApi implements CodeCoverageApi { } async getFileContentFromEntity( - entityName: EntityName, + entityName: CompoundEntityRef, filePath: string, ): Promise { const entity = encodeURIComponent(stringifyEntityRef(entityName)); @@ -78,7 +83,7 @@ export class CodeCoverageRestApi implements CodeCoverageApi { } async getCoverageHistoryForEntity( - entityName: EntityName, + entityName: CompoundEntityRef, limit?: number, ): Promise { const entity = encodeURIComponent(stringifyEntityRef(entityName)); diff --git a/plugins/code-coverage/src/types.ts b/plugins/code-coverage/src/types.ts index c4c622bfaf..b03025b7ad 100644 --- a/plugins/code-coverage/src/types.ts +++ b/plugins/code-coverage/src/types.ts @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; + +import { CompoundEntityRef } from '@backstage/catalog-model'; export type JsonCodeCoverage = { metadata: CoverageMetadata; - entity: EntityName; + entity: CompoundEntityRef; files: Array; }; export type JsonCoverageHistory = { - entity: EntityName; + entity: CompoundEntityRef; history: Array; }; diff --git a/plugins/fossa/src/components/FossaPage/FossaPage.tsx b/plugins/fossa/src/components/FossaPage/FossaPage.tsx index 90a36a6177..731852d4a7 100644 --- a/plugins/fossa/src/components/FossaPage/FossaPage.tsx +++ b/plugins/fossa/src/components/FossaPage/FossaPage.tsx @@ -16,7 +16,7 @@ import { Entity, - EntityName, + CompoundEntityRef, RELATION_OWNED_BY, } from '@backstage/catalog-model'; import { @@ -56,7 +56,7 @@ type FossaRow = { resolved: { name: string; ownedByRelationsTitle?: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; loading: boolean; details?: FindingSummary; }; diff --git a/plugins/jenkins-backend/api-report.md b/plugins/jenkins-backend/api-report.md index d437a5dd75..cbc3ef4325 100644 --- a/plugins/jenkins-backend/api-report.md +++ b/plugins/jenkins-backend/api-report.md @@ -4,8 +4,8 @@ ```ts import { CatalogApi } from '@backstage/catalog-client'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { EntityName } from '@backstage/catalog-model'; import express from 'express'; import { Logger as Logger_2 } from 'winston'; @@ -25,7 +25,7 @@ export class DefaultJenkinsInfoProvider implements JenkinsInfoProvider { }): DefaultJenkinsInfoProvider; // (undocumented) getInstance(opt: { - entityRef: EntityName; + entityRef: CompoundEntityRef; jobFullName?: string; }): Promise; // (undocumented) @@ -65,7 +65,7 @@ export interface JenkinsInfo { export interface JenkinsInfoProvider { // (undocumented) getInstance(options: { - entityRef: EntityName; + entityRef: CompoundEntityRef; jobFullName?: string; }): Promise; } diff --git a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts index 62299926df..33079dfc8f 100644 --- a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts +++ b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts @@ -15,7 +15,7 @@ */ import { CatalogApi } from '@backstage/catalog-client'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { DefaultJenkinsInfoProvider, @@ -163,7 +163,7 @@ describe('DefaultJenkinsInfoProvider', () => { getEntityByName: jest.fn(), } as any as jest.Mocked; - const entityRef: EntityName = { + const entityRef: CompoundEntityRef = { kind: 'Component', namespace: 'foo', name: 'bar', diff --git a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts index f822f475f1..528a1d038f 100644 --- a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts +++ b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts @@ -17,7 +17,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; @@ -27,7 +27,7 @@ export interface JenkinsInfoProvider { /** * The entity to get the info about. */ - entityRef: EntityName; + entityRef: CompoundEntityRef; /** * A specific job to get. This is only passed in when we know about a job name we are interested in. */ @@ -182,7 +182,7 @@ export class DefaultJenkinsInfoProvider implements JenkinsInfoProvider { } async getInstance(opt: { - entityRef: EntityName; + entityRef: CompoundEntityRef; jobFullName?: string; }): Promise { // load entity diff --git a/plugins/jenkins-backend/src/service/standaloneServer.ts b/plugins/jenkins-backend/src/service/standaloneServer.ts index 7f47d57ac3..f89f235667 100644 --- a/plugins/jenkins-backend/src/service/standaloneServer.ts +++ b/plugins/jenkins-backend/src/service/standaloneServer.ts @@ -18,7 +18,7 @@ import { createServiceBuilder } from '@backstage/backend-common'; import { Server } from 'http'; import { Logger } from 'winston'; import { createRouter } from './router'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { JenkinsInfo } from './jenkinsInfoProvider'; export interface ServerOptions { @@ -35,7 +35,9 @@ export async function startStandaloneServer( const router = await createRouter({ logger, jenkinsInfoProvider: { - async getInstance(_: { entityRef: EntityName }): Promise { + async getInstance(_: { + entityRef: CompoundEntityRef; + }): Promise { return { baseUrl: 'https://example.com/', jobFullName: 'build-foo' }; }, }, diff --git a/plugins/jenkins/api-report.md b/plugins/jenkins/api-report.md index 5db5fd0dfe..fc9de04a7a 100644 --- a/plugins/jenkins/api-report.md +++ b/plugins/jenkins/api-report.md @@ -7,9 +7,9 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; +import type { CompoundEntityRef } from '@backstage/catalog-model'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; -import type { EntityName } from '@backstage/catalog-model'; import { IdentityApi } from '@backstage/core-plugin-api'; import { InfoCardVariants } from '@backstage/core-components'; import { RouteRef } from '@backstage/core-plugin-api'; @@ -48,20 +48,20 @@ export const JENKINS_ANNOTATION = 'jenkins.io/job-full-name'; export interface JenkinsApi { // Warning: (ae-forgotten-export) The symbol "Build" needs to be exported by the entry point index.d.ts getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; // Warning: (ae-forgotten-export) The symbol "Project" needs to be exported by the entry point index.d.ts getProjects(options: { - entity: EntityName; + entity: CompoundEntityRef; filter: { branch?: string; }; }): Promise; // (undocumented) retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; @@ -82,20 +82,20 @@ export class JenkinsClient implements JenkinsApi { }); // (undocumented) getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; // (undocumented) getProjects(options: { - entity: EntityName; + entity: CompoundEntityRef; filter: { branch?: string; }; }): Promise; // (undocumented) retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; diff --git a/plugins/jenkins/src/api/JenkinsApi.ts b/plugins/jenkins/src/api/JenkinsApi.ts index dd169de78b..04bd313227 100644 --- a/plugins/jenkins/src/api/JenkinsApi.ts +++ b/plugins/jenkins/src/api/JenkinsApi.ts @@ -19,7 +19,7 @@ import { DiscoveryApi, IdentityApi, } from '@backstage/core-plugin-api'; -import type { EntityName } from '@backstage/catalog-model'; +import type { CompoundEntityRef } from '@backstage/catalog-model'; import { ResponseError } from '@backstage/errors'; export const jenkinsApiRef = createApiRef({ @@ -80,7 +80,7 @@ export interface JenkinsApi { */ getProjects(options: { /** the entity whose jobs should be retrieved. */ - entity: EntityName; + entity: CompoundEntityRef; /** a filter on jobs. Currently this just takes a branch (and assumes certain structures in jenkins) */ filter: { branch?: string }; }): Promise; @@ -93,13 +93,13 @@ export interface JenkinsApi { * TODO: abstract jobFullName (so we could support differentiating between the same named job on multiple instances). */ getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; @@ -118,7 +118,7 @@ export class JenkinsClient implements JenkinsApi { } async getProjects(options: { - entity: EntityName; + entity: CompoundEntityRef; filter: { branch?: string }; }): Promise { const { entity, filter } = options; @@ -157,7 +157,7 @@ export class JenkinsClient implements JenkinsApi { } async getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise { @@ -182,7 +182,7 @@ export class JenkinsClient implements JenkinsApi { } async retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise { diff --git a/plugins/jenkins/src/components/useBuildWithSteps.ts b/plugins/jenkins/src/components/useBuildWithSteps.ts index 2166b2a1a1..0910564384 100644 --- a/plugins/jenkins/src/components/useBuildWithSteps.ts +++ b/plugins/jenkins/src/components/useBuildWithSteps.ts @@ -19,7 +19,7 @@ import { jenkinsApiRef } from '../api'; import { useAsyncPolling } from './useAsyncPolling'; import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { useEntity } from '@backstage/plugin-catalog-react'; -import { getEntityName } from '@backstage/catalog-model'; +import { getCompoundEntityRef } from '@backstage/catalog-model'; const INTERVAL_AMOUNT = 1500; @@ -41,7 +41,7 @@ export function useBuildWithSteps({ const getBuildWithSteps = useCallback(async () => { try { - const entityName = await getEntityName(entity); + const entityName = await getCompoundEntityRef(entity); return api.getBuild({ entity: entityName, jobFullName, buildNumber }); } catch (e) { errorApi.post(e); diff --git a/plugins/jenkins/src/components/useBuilds.ts b/plugins/jenkins/src/components/useBuilds.ts index a4afeb351f..07d23cce5a 100644 --- a/plugins/jenkins/src/components/useBuilds.ts +++ b/plugins/jenkins/src/components/useBuilds.ts @@ -18,7 +18,7 @@ import useAsyncRetry from 'react-use/lib/useAsyncRetry'; import { jenkinsApiRef } from '../api'; import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { useEntity } from '@backstage/plugin-catalog-react'; -import { getEntityName } from '@backstage/catalog-model'; +import { getCompoundEntityRef } from '@backstage/catalog-model'; export enum ErrorType { CONNECTION_ERROR, @@ -33,7 +33,7 @@ export enum ErrorType { */ export function useBuilds({ branch }: { branch?: string } = {}) { const { entity } = useEntity(); - const entityName = getEntityName(entity); + const entityName = getCompoundEntityRef(entity); const api = useApi(jenkinsApiRef); const errorApi = useApi(errorApiRef); @@ -60,7 +60,7 @@ export function useBuilds({ branch }: { branch?: string } = {}) { } = useAsyncRetry(async () => { try { const build = await api.getProjects({ - entity: getEntityName(entity), + entity: getCompoundEntityRef(entity), filter: { branch }, }); diff --git a/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts b/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts index 5fcd89841e..d095b77b4c 100644 --- a/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts +++ b/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts @@ -16,7 +16,7 @@ import { Entity, - getEntityName, + getCompoundEntityRef, parseEntityRef, RELATION_OWNED_BY, RELATION_OWNER_OF, @@ -55,7 +55,7 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { _location: LocationSpec, emit: CatalogProcessorEmit, ): Promise { - const selfRef = getEntityName(entity); + const selfRef = getCompoundEntityRef(entity); if ( entity.apiVersion === 'scaffolder.backstage.io/v1beta3' && diff --git a/plugins/scaffolder-backend/src/service/helpers.ts b/plugins/scaffolder-backend/src/service/helpers.ts index e49c28cb74..e3aae0951b 100644 --- a/plugins/scaffolder-backend/src/service/helpers.ts +++ b/plugins/scaffolder-backend/src/service/helpers.ts @@ -20,7 +20,7 @@ import { ANNOTATION_LOCATION, parseLocationRef, ANNOTATION_SOURCE_LOCATION, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; @@ -91,7 +91,7 @@ export function getEntityBaseUrl(entity: Entity): string | undefined { * Returns the matching template, or throws a NotFoundError if no such template existed. */ export async function findTemplate(options: { - entityRef: EntityName; + entityRef: CompoundEntityRef; token?: string; catalogApi: CatalogApi; }): Promise { diff --git a/plugins/tech-insights-backend/src/service/router.ts b/plugins/tech-insights-backend/src/service/router.ts index b13802358f..b86677ab90 100644 --- a/plugins/tech-insights-backend/src/service/router.ts +++ b/plugins/tech-insights-backend/src/service/router.ts @@ -27,7 +27,7 @@ import { Logger } from 'winston'; import { DateTime } from 'luxon'; import { PersistenceContext } from './persistence/persistenceContext'; import { - EntityName, + CompoundEntityRef, parseEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; @@ -99,8 +99,10 @@ export async function createRouter< }); router.post('/checks/run', async (req, res) => { - const { checks, entities }: { checks: string[]; entities: EntityName[] } = - req.body; + const { + checks, + entities, + }: { checks: string[]; entities: CompoundEntityRef[] } = req.body; const tasks = entities.map(async entity => { const entityTriplet = typeof entity === 'string' ? entity : stringifyEntityRef(entity); diff --git a/plugins/tech-insights/api-report.md b/plugins/tech-insights/api-report.md index 79854e505f..deadd06010 100644 --- a/plugins/tech-insights/api-report.md +++ b/plugins/tech-insights/api-report.md @@ -9,7 +9,7 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; import { BulkCheckResponse } from '@backstage/plugin-tech-insights-common'; import { CheckResult } from '@backstage/plugin-tech-insights-common'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { default as React_2 } from 'react'; import { RouteRef } from '@backstage/core-plugin-api'; @@ -52,11 +52,14 @@ export interface TechInsightsApi { ) => CheckResultRenderer | undefined; // (undocumented) runBulkChecks( - entities: EntityName[], + entities: CompoundEntityRef[], checks?: Check[], ): Promise; // (undocumented) - runChecks(entityParams: EntityName, checks?: Check[]): Promise; + runChecks( + entityParams: CompoundEntityRef, + checks?: Check[], + ): Promise; } // @public diff --git a/plugins/tech-insights/src/api/TechInsightsApi.ts b/plugins/tech-insights/src/api/TechInsightsApi.ts index b1287a62dc..0a6bb937a6 100644 --- a/plugins/tech-insights/src/api/TechInsightsApi.ts +++ b/plugins/tech-insights/src/api/TechInsightsApi.ts @@ -21,7 +21,7 @@ import { } from '@backstage/plugin-tech-insights-common'; import { Check } from './types'; import { CheckResultRenderer } from '../components/CheckResultRenderer'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; /** * {@link @backstage/core-plugin-api#ApiRef} for the {@link TechInsightsApi} @@ -45,9 +45,12 @@ export interface TechInsightsApi { description?: string, ) => CheckResultRenderer | undefined; getAllChecks(): Promise; - runChecks(entityParams: EntityName, checks?: Check[]): Promise; + runChecks( + entityParams: CompoundEntityRef, + checks?: Check[], + ): Promise; runBulkChecks( - entities: EntityName[], + entities: CompoundEntityRef[], checks?: Check[], ): Promise; } diff --git a/plugins/tech-insights/src/api/TechInsightsClient.ts b/plugins/tech-insights/src/api/TechInsightsClient.ts index 5e29749854..5b9af9557f 100644 --- a/plugins/tech-insights/src/api/TechInsightsClient.ts +++ b/plugins/tech-insights/src/api/TechInsightsClient.ts @@ -22,7 +22,7 @@ import { import { Check } from './types'; import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; import { ResponseError } from '@backstage/errors'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { CheckResultRenderer, @@ -74,7 +74,7 @@ export class TechInsightsClient implements TechInsightsApi { } async runChecks( - entityParams: EntityName, + entityParams: CompoundEntityRef, checks?: Check[], ): Promise { const url = await this.discoveryApi.getBaseUrl('tech-insights'); @@ -102,7 +102,7 @@ export class TechInsightsClient implements TechInsightsApi { } async runBulkChecks( - entities: EntityName[], + entities: CompoundEntityRef[], checks?: Check[], ): Promise { const url = await this.discoveryApi.getBaseUrl('tech-insights'); diff --git a/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts b/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts index 678a4b0539..a1542df351 100644 --- a/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts +++ b/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts @@ -16,7 +16,7 @@ import { CachedEntityLoader } from './CachedEntityLoader'; import { CatalogClient } from '@backstage/catalog-client'; import { CacheClient } from '@backstage/backend-common'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; describe('CachedEntityLoader', () => { const catalog: jest.Mocked = { @@ -28,7 +28,7 @@ describe('CachedEntityLoader', () => { set: jest.fn(), } as any; - const entityName: EntityName = { + const entityName: CompoundEntityRef = { kind: 'component', namespace: 'default', name: 'test', diff --git a/plugins/techdocs-backend/src/service/CachedEntityLoader.ts b/plugins/techdocs-backend/src/service/CachedEntityLoader.ts index 81c6ec7431..cd771f7fff 100644 --- a/plugins/techdocs-backend/src/service/CachedEntityLoader.ts +++ b/plugins/techdocs-backend/src/service/CachedEntityLoader.ts @@ -17,7 +17,7 @@ import { CatalogClient } from '@backstage/catalog-client'; import { CacheClient } from '@backstage/backend-common'; import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; @@ -37,7 +37,7 @@ export class CachedEntityLoader { } async load( - entityName: EntityName, + entityName: CompoundEntityRef, token: string | undefined, ): Promise { const cacheKey = this.getCacheKey(entityName, token); @@ -66,7 +66,7 @@ export class CachedEntityLoader { } private getCacheKey( - entityName: EntityName, + entityName: CompoundEntityRef, token: string | undefined, ): string { const key = ['catalog', stringifyEntityRef(entityName)]; diff --git a/plugins/techdocs/api-report.md b/plugins/techdocs/api-report.md index 917c1d5d85..f7ef6e984a 100644 --- a/plugins/techdocs/api-report.md +++ b/plugins/techdocs/api-report.md @@ -7,11 +7,11 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { CSSProperties } from '@material-ui/styles'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { FetchApi } from '@backstage/core-plugin-api'; import { IdentityApi } from '@backstage/core-plugin-api'; import { PropsWithChildren } from 'react'; @@ -89,7 +89,7 @@ export type DocsTableRow = { resolved: { docsUrl: string; ownedByRelationsTitle: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; }; @@ -161,7 +161,7 @@ export const Reader: (props: ReaderProps) => JSX.Element; // @public export type ReaderProps = { - entityRef: EntityName; + entityRef: CompoundEntityRef; withSearch?: boolean; onReady?: () => void; }; @@ -187,9 +187,11 @@ export type TabsConfig = TabConfig[]; export interface TechDocsApi { getApiOrigin(): Promise; // (undocumented) - getEntityMetadata(entityId: EntityName): Promise; + getEntityMetadata( + entityId: CompoundEntityRef, + ): Promise; // (undocumented) - getTechDocsMetadata(entityId: EntityName): Promise; + getTechDocsMetadata(entityId: CompoundEntityRef): Promise; } // @public @@ -208,8 +210,10 @@ export class TechDocsClient implements TechDocsApi { discoveryApi: DiscoveryApi; // (undocumented) getApiOrigin(): Promise; - getEntityMetadata(entityId: EntityName): Promise; - getTechDocsMetadata(entityId: EntityName): Promise; + getEntityMetadata( + entityId: CompoundEntityRef, + ): Promise; + getTechDocsMetadata(entityId: CompoundEntityRef): Promise; } // @public @@ -297,7 +301,7 @@ export const TechDocsReaderPageHeader: ( // @public export type TechDocsReaderPageHeaderProps = PropsWithChildren<{ - entityRef: EntityName; + entityRef: CompoundEntityRef; entityMetadata?: TechDocsEntityMetadata; techDocsMetadata?: TechDocsMetadata; }>; @@ -315,7 +319,7 @@ export type TechDocsReaderPageRenderFunction = ({ }: { techdocsMetadataValue?: TechDocsMetadata | undefined; entityMetadataValue?: TechDocsEntityMetadata | undefined; - entityRef: EntityName; + entityRef: CompoundEntityRef; onReady: () => void; }) => JSX.Element; @@ -324,7 +328,7 @@ export const TechDocsSearch: (props: TechDocsSearchProps) => JSX.Element; // @public export type TechDocsSearchProps = { - entityId: EntityName; + entityId: CompoundEntityRef; debounceTime?: number; }; @@ -348,18 +352,18 @@ export interface TechDocsStorageApi { // (undocumented) getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise; // (undocumented) getBuilder(): Promise; // (undocumented) - getEntityDocs(entityId: EntityName, path: string): Promise; + getEntityDocs(entityId: CompoundEntityRef, path: string): Promise; // (undocumented) getStorageUrl(): Promise; // (undocumented) syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler?: (line: string) => void, ): Promise; } @@ -384,18 +388,18 @@ export class TechDocsStorageClient implements TechDocsStorageApi { // (undocumented) getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise; // (undocumented) getBuilder(): Promise; - getEntityDocs(entityId: EntityName, path: string): Promise; + getEntityDocs(entityId: CompoundEntityRef, path: string): Promise; // (undocumented) getStorageUrl(): Promise; // (undocumented) identityApi: IdentityApi; syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler?: (line: string) => void, ): Promise; } diff --git a/plugins/techdocs/dev/index.tsx b/plugins/techdocs/dev/index.tsx index 8dc0e5d30b..d08e914736 100644 --- a/plugins/techdocs/dev/index.tsx +++ b/plugins/techdocs/dev/index.tsx @@ -17,7 +17,7 @@ import { createDevApp } from '@backstage/dev-utils'; import { NotFoundError } from '@backstage/errors'; import React from 'react'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Reader, SyncResult, @@ -82,7 +82,10 @@ function createPage({ }); } - async syncEntityDocs(_: EntityName, logHandler?: (line: string) => void) { + async syncEntityDocs( + _: CompoundEntityRef, + logHandler?: (line: string) => void, + ) { if (syncDocsDelay) { for (let i = 0; i < 10; i++) { setTimeout( diff --git a/plugins/techdocs/src/api.ts b/plugins/techdocs/src/api.ts index 4566bbf028..b349431b0d 100644 --- a/plugins/techdocs/src/api.ts +++ b/plugins/techdocs/src/api.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { TechDocsEntityMetadata, TechDocsMetadata } from './types'; import { createApiRef } from '@backstage/core-plugin-api'; @@ -55,14 +55,14 @@ export interface TechDocsStorageApi { getApiOrigin(): Promise; getStorageUrl(): Promise; getBuilder(): Promise; - getEntityDocs(entityId: EntityName, path: string): Promise; + getEntityDocs(entityId: CompoundEntityRef, path: string): Promise; syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler?: (line: string) => void, ): Promise; getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise; } @@ -77,6 +77,8 @@ export interface TechDocsApi { * Set to techdocs.requestUrl as the URL for techdocs-backend API. */ getApiOrigin(): Promise; - getTechDocsMetadata(entityId: EntityName): Promise; - getEntityMetadata(entityId: EntityName): Promise; + getTechDocsMetadata(entityId: CompoundEntityRef): Promise; + getEntityMetadata( + entityId: CompoundEntityRef, + ): Promise; } diff --git a/plugins/techdocs/src/client.ts b/plugins/techdocs/src/client.ts index eac5da0b6b..f260f00547 100644 --- a/plugins/techdocs/src/client.ts +++ b/plugins/techdocs/src/client.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { DiscoveryApi, @@ -62,7 +62,9 @@ export class TechDocsClient implements TechDocsApi { * * @param entityId - Object containing entity data like name, namespace, etc. */ - async getTechDocsMetadata(entityId: EntityName): Promise { + async getTechDocsMetadata( + entityId: CompoundEntityRef, + ): Promise { const { kind, namespace, name } = entityId; const apiOrigin = await this.getApiOrigin(); @@ -84,7 +86,7 @@ export class TechDocsClient implements TechDocsApi { * @param entityId - Object containing entity data like name, namespace, etc. */ async getEntityMetadata( - entityId: EntityName, + entityId: CompoundEntityRef, ): Promise { const { kind, namespace, name } = entityId; @@ -149,7 +151,10 @@ export class TechDocsStorageClient implements TechDocsStorageApi { * @returns HTML content of the docs page as string * @throws Throws error when the page is not found. */ - async getEntityDocs(entityId: EntityName, path: string): Promise { + async getEntityDocs( + entityId: CompoundEntityRef, + path: string, + ): Promise { const { kind, namespace, name } = entityId; const storageUrl = await this.getStorageUrl(); @@ -190,7 +195,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi { * @throws Throws error on error from sync endpoint in Techdocs Backend */ async syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler: (line: string) => void = () => {}, ): Promise { const { kind, namespace, name } = entityId; @@ -243,7 +248,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi { async getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise { const { kind, namespace, name } = entityId; diff --git a/plugins/techdocs/src/home/components/Tables/types.ts b/plugins/techdocs/src/home/components/Tables/types.ts index c076d4c12d..6c06526d07 100644 --- a/plugins/techdocs/src/home/components/Tables/types.ts +++ b/plugins/techdocs/src/home/components/Tables/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; /** * Generic representing the metadata structure for a docs table row. @@ -26,6 +26,6 @@ export type DocsTableRow = { resolved: { docsUrl: string; ownedByRelationsTitle: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; }; diff --git a/plugins/techdocs/src/reader/components/Reader.tsx b/plugins/techdocs/src/reader/components/Reader.tsx index 4042585b6d..6aa5c99be4 100644 --- a/plugins/techdocs/src/reader/components/Reader.tsx +++ b/plugins/techdocs/src/reader/components/Reader.tsx @@ -34,7 +34,7 @@ import { alpha, } from '@material-ui/core'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { useApi, configApiRef } from '@backstage/core-plugin-api'; import { scmIntegrationsApiRef } from '@backstage/integration-react'; import { BackstageTheme } from '@backstage/theme'; @@ -68,7 +68,7 @@ import { useReaderState } from './useReaderState'; * @public */ export type ReaderProps = { - entityRef: EntityName; + entityRef: CompoundEntityRef; withSearch?: boolean; onReady?: () => void; }; @@ -95,7 +95,7 @@ const TechDocsReaderContext = createContext( const TechDocsReaderProvider = ({ children, entityRef, -}: PropsWithChildren<{ entityRef: EntityName }>) => { +}: PropsWithChildren<{ entityRef: CompoundEntityRef }>) => { const { '*': path } = useParams(); const { kind, namespace, name } = entityRef; const value = useReaderState(kind, namespace, name, path); @@ -116,7 +116,7 @@ const TechDocsReaderProvider = ({ * @internal */ export const withTechDocsReaderProvider = - (Component: ComponentType, entityRef: EntityName) => + (Component: ComponentType, entityRef: CompoundEntityRef) => (props: T) => ( @@ -157,7 +157,9 @@ const headings: TypographyHeadingsKeys[] = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']; * todo: Make public or stop exporting (see others: "altReaderExperiments") * @internal */ -export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { +export const useTechDocsReaderDom = ( + entityRef: CompoundEntityRef, +): Element | null => { const navigate = useNavigate(); const theme = useTheme(); const techdocsStorageApi = useApi(techdocsStorageApiRef); @@ -322,7 +324,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { --md-source-version-icon: url('data:image/svg+xml;charset=utf-8,'); --md-version-icon: url('data:image/svg+xml;charset=utf-8,'); } - + :host > * { /* CODE */ --md-code-fg-color: ${theme.palette.text.primary}; @@ -438,7 +440,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-main__inner { margin-top: 0; } - + .md-sidebar { height: calc(100% - 100px); position: fixed; @@ -450,13 +452,13 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-sidebar--secondary { right: ${theme.spacing(3)}px; } - + .md-content { max-width: calc(100% - 16rem * 2); margin-left: 16rem; margin-bottom: 50px; } - + .md-footer { position: fixed; bottom: 0px; @@ -471,7 +473,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-dialog { background-color: unset; } - + @media screen and (max-width: 76.1875em) { .md-nav { transition: none !important; @@ -567,7 +569,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { }), injectCss({ // Typeset - css: ` + css: ` .md-typeset { font-size: var(--md-typeset-font-size); } @@ -600,11 +602,11 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-typeset .md-content__button { color: var(--md-default-fg-color); } - + .md-typeset hr { border-bottom: 0.05rem dotted ${theme.palette.divider}; } - + .md-typeset details { font-size: var(--md-typeset-font-size) !important; } @@ -621,7 +623,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-typeset details[open] > summary:after { transform: rotate(90deg) translateX(-50%) !important; } - + .md-typeset blockquote { color: var(--md-default-fg-color--light); border-left: 0.2rem solid var(--md-default-fg-color--light); @@ -667,13 +669,13 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .highlight .md-clipboard:after { content: unset; } - + .highlight .nx { color: ${isDarkTheme ? '#ff53a3' : '#ec407a'}; } /* CODE HILITE */ - .codehilite .gd { + .codehilite .gd { background-color: ${ isDarkTheme ? 'rgba(248,81,73,0.65)' : '#fdd' }; diff --git a/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx b/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx index d03ea2ef61..184ab24b6d 100644 --- a/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx @@ -21,12 +21,12 @@ import useAsync from 'react-use/lib/useAsync'; import { techdocsApiRef } from '../../api'; import { LegacyTechDocsPage } from './LegacyTechDocsPage'; import { TechDocsEntityMetadata, TechDocsMetadata } from '../../types'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { useApi, useApp } from '@backstage/core-plugin-api'; import { Page } from '@backstage/core-components'; /** - * Helper function that gives the children of {@link TechDocsReaderPage} acccess to techdocs and entity metadata + * Helper function that gives the children of {@link TechDocsReaderPage} access to techdocs and entity metadata * * @public */ @@ -37,7 +37,7 @@ export type TechDocsReaderPageRenderFunction = ({ }: { techdocsMetadataValue?: TechDocsMetadata | undefined; entityMetadataValue?: TechDocsEntityMetadata | undefined; - entityRef: EntityName; + entityRef: CompoundEntityRef; onReady: () => void; }) => JSX.Element; diff --git a/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx b/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx index 928833efed..5867641c48 100644 --- a/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx @@ -19,7 +19,7 @@ import CodeIcon from '@material-ui/icons/Code'; import { useRouteRef } from '@backstage/core-plugin-api'; import { Header, HeaderLabel } from '@backstage/core-components'; -import { EntityName, RELATION_OWNED_BY } from '@backstage/catalog-model'; +import { CompoundEntityRef, RELATION_OWNED_BY } from '@backstage/catalog-model'; import { EntityRefLink, EntityRefLinks, @@ -35,7 +35,7 @@ import { TechDocsEntityMetadata, TechDocsMetadata } from '../../types'; * @public */ export type TechDocsReaderPageHeaderProps = PropsWithChildren<{ - entityRef: EntityName; + entityRef: CompoundEntityRef; entityMetadata?: TechDocsEntityMetadata; techDocsMetadata?: TechDocsMetadata; }>; diff --git a/plugins/techdocs/src/reader/components/useRawPage.ts b/plugins/techdocs/src/reader/components/useRawPage.ts index 61b04a5a82..311f8554a9 100644 --- a/plugins/techdocs/src/reader/components/useRawPage.ts +++ b/plugins/techdocs/src/reader/components/useRawPage.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import useAsyncRetry from 'react-use/lib/useAsyncRetry'; import { AsyncState } from 'react-use/lib/useAsync'; import { techdocsStorageApiRef } from '../../api'; @@ -22,7 +22,7 @@ import { useApi } from '@backstage/core-plugin-api'; export type RawPage = { content: string; path: string; - entityId: EntityName; + entityId: CompoundEntityRef; }; export function useRawPage( diff --git a/plugins/techdocs/src/reader/transformers/addBaseUrl.ts b/plugins/techdocs/src/reader/transformers/addBaseUrl.ts index f3906b7b91..2e3b853e42 100644 --- a/plugins/techdocs/src/reader/transformers/addBaseUrl.ts +++ b/plugins/techdocs/src/reader/transformers/addBaseUrl.ts @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { TechDocsStorageApi } from '../../api'; import type { Transformer } from './transformer'; type AddBaseUrlOptions = { techdocsStorageApi: TechDocsStorageApi; - entityId: EntityName; + entityId: CompoundEntityRef; path: string; }; diff --git a/plugins/techdocs/src/search/components/TechDocsSearch.tsx b/plugins/techdocs/src/search/components/TechDocsSearch.tsx index c817fb3217..30cc692b7c 100644 --- a/plugins/techdocs/src/search/components/TechDocsSearch.tsx +++ b/plugins/techdocs/src/search/components/TechDocsSearch.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { SearchContextProvider, useSearch } from '@backstage/plugin-search'; import { makeStyles, @@ -42,7 +42,7 @@ const useStyles = makeStyles({ * @public */ export type TechDocsSearchProps = { - entityId: EntityName; + entityId: CompoundEntityRef; debounceTime?: number; }; diff --git a/plugins/todo-backend/api-report.md b/plugins/todo-backend/api-report.md index 71094e03f8..f3f2af3fb9 100644 --- a/plugins/todo-backend/api-report.md +++ b/plugins/todo-backend/api-report.md @@ -4,8 +4,8 @@ ```ts import { CatalogApi } from '@backstage/catalog-client'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { EntityName } from '@backstage/catalog-model'; import express from 'express'; import { Logger as Logger_2 } from 'winston'; import { ScmIntegrations } from '@backstage/integration'; @@ -19,7 +19,7 @@ export function createTodoParser(options?: TodoParserOptions): TodoParser; // @public (undocumented) export type ListTodosRequest = { - entity?: EntityName; + entity?: CompoundEntityRef; offset?: number; limit?: number; orderBy?: { diff --git a/plugins/todo-backend/src/service/router.ts b/plugins/todo-backend/src/service/router.ts index a0e6a6ec94..33356a83d6 100644 --- a/plugins/todo-backend/src/service/router.ts +++ b/plugins/todo-backend/src/service/router.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName, parseEntityRef } from '@backstage/catalog-model'; +import { CompoundEntityRef, parseEntityRef } from '@backstage/catalog-model'; import { InputError } from '@backstage/errors'; import express from 'express'; import Router from 'express-promise-router'; @@ -52,7 +52,7 @@ export async function createRouter( if (entityRef && typeof entityRef !== 'string') { throw new InputError(`entity query must be a string`); } - let entity: EntityName | undefined = undefined; + let entity: CompoundEntityRef | undefined = undefined; if (entityRef) { try { entity = parseEntityRef(entityRef); diff --git a/plugins/todo-backend/src/service/types.ts b/plugins/todo-backend/src/service/types.ts index 90ee6bc913..88d2f2b340 100644 --- a/plugins/todo-backend/src/service/types.ts +++ b/plugins/todo-backend/src/service/types.ts @@ -14,12 +14,12 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { TodoItem } from '../lib'; /** @public */ export type ListTodosRequest = { - entity?: EntityName; + entity?: CompoundEntityRef; offset?: number; limit?: number; orderBy?: { From 5b7343be6780942b18541ffba776dc199b925ab2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Mar 2022 04:07:37 +0000 Subject: [PATCH 124/150] chore(deps): bump ldapjs from 2.3.1 to 2.3.2 Bumps [ldapjs](https://github.com/ldapjs/node-ldapjs) from 2.3.1 to 2.3.2. - [Release notes](https://github.com/ldapjs/node-ldapjs/releases) - [Changelog](https://github.com/ldapjs/node-ldapjs/blob/master/CHANGES.md) - [Commits](https://github.com/ldapjs/node-ldapjs/compare/v2.3.1...v2.3.2) --- updated-dependencies: - dependency-name: ldapjs dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index c2f299faa3..13371a2df3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16251,9 +16251,9 @@ ldap-filter@^0.3.3: assert-plus "^1.0.0" ldapjs@^2.2.0: - version "2.3.1" - resolved "https://registry.npmjs.org/ldapjs/-/ldapjs-2.3.1.tgz#04136815fb1f21d692ac87fab5961a04d86e8b04" - integrity sha512-kf0tHHLrpwKaBAQOhYHXgdeh2PkFuCCxWgLb1MRn67ZQVo787D2pij3mmHVZx193GIdM8xcfi8HF6AIYYnj0fQ== + version "2.3.2" + resolved "https://registry.npmjs.org/ldapjs/-/ldapjs-2.3.2.tgz#a599d081519f70462941cc33a50e9354c32f35b7" + integrity sha512-FU+GR/qbQ96WUZ2DUb7FzaEybYvv3240wTVPcbsdELB3o4cK92zGVjntsh68siVkLeCmlCcsd/cIQzyGXSS7LA== dependencies: abstract-logging "^2.0.0" asn1 "^0.2.4" From 5617b2f815f71cf1adf9f5f297f860009091c656 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Mar 2022 04:08:04 +0000 Subject: [PATCH 125/150] chore(deps): bump eslint-config-prettier from 8.3.0 to 8.5.0 Bumps [eslint-config-prettier](https://github.com/prettier/eslint-config-prettier) from 8.3.0 to 8.5.0. - [Release notes](https://github.com/prettier/eslint-config-prettier/releases) - [Changelog](https://github.com/prettier/eslint-config-prettier/blob/main/CHANGELOG.md) - [Commits](https://github.com/prettier/eslint-config-prettier/compare/v8.3.0...v8.5.0) --- updated-dependencies: - dependency-name: eslint-config-prettier dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index c2f299faa3..63ddc00adf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11548,9 +11548,9 @@ escodegen@^2.0.0: source-map "~0.6.1" eslint-config-prettier@^8.3.0: - version "8.3.0" - resolved "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.3.0.tgz#f7471b20b6fe8a9a9254cc684454202886a2dd7a" - integrity sha512-BgZuLUSeKzvlL/VUjx/Yb787VQ26RU3gGjA3iiFvdsp/2bMfVIWUVP7tjxtjS0e+HP409cPlPvNkQloz8C91ew== + version "8.5.0" + resolved "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.5.0.tgz#5a81680ec934beca02c7b1a61cf8ca34b66feab1" + integrity sha512-obmWKLUNCnhtQRKc+tmnYuQl0pFU1ibYJQ5BGhTVB08bHe9wC8qUeG7c08dj9XX+AuPj1YSGSQIHl1pnDHZR0Q== eslint-formatter-friendly@^7.0.0: version "7.0.0" From bb2ba5f10d84477eee9807d5140a314b1236e474 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 3 Mar 2022 10:02:14 +0100 Subject: [PATCH 126/150] remove deprecated catalog-client request/response types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/brown-dryers-serve.md | 10 +++++ packages/catalog-client/api-report.md | 12 ------ .../catalog-client/src/types/deprecated.ts | 43 ------------------- packages/catalog-client/src/types/index.ts | 1 - 4 files changed, 10 insertions(+), 56 deletions(-) create mode 100644 .changeset/brown-dryers-serve.md delete mode 100644 packages/catalog-client/src/types/deprecated.ts diff --git a/.changeset/brown-dryers-serve.md b/.changeset/brown-dryers-serve.md new file mode 100644 index 0000000000..40c02538b8 --- /dev/null +++ b/.changeset/brown-dryers-serve.md @@ -0,0 +1,10 @@ +--- +'@backstage/catalog-client': minor +--- + +**BREAKING**: Removed the old deprecated request/response types: + +- `CatalogEntitiesRequest` - please use `GetEntitiesRequest` instead +- `CatalogEntityAncestorsRequest` - please use `GetEntityAncestorsRequest` instead +- `CatalogEntityAncestorsResponse` - please use `GetEntityAncestorsResponse` instead +- `CatalogListResponse` - please use `GetEntitiesResponse` instead diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index ce6397f51e..ef19effe90 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -130,18 +130,6 @@ export class CatalogClient implements CatalogApi { ): Promise; } -// @public @deprecated (undocumented) -export type CatalogEntitiesRequest = GetEntitiesRequest; - -// @public @deprecated (undocumented) -export type CatalogEntityAncestorsRequest = GetEntityAncestorsRequest; - -// @public @deprecated (undocumented) -export type CatalogEntityAncestorsResponse = GetEntityAncestorsResponse; - -// @public @deprecated (undocumented) -export type CatalogListResponse<_Entity> = GetEntitiesResponse; - // @public export interface CatalogRequestOptions { // (undocumented) diff --git a/packages/catalog-client/src/types/deprecated.ts b/packages/catalog-client/src/types/deprecated.ts deleted file mode 100644 index 0b145f3024..0000000000 --- a/packages/catalog-client/src/types/deprecated.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - GetEntitiesRequest, - GetEntitiesResponse, - GetEntityAncestorsRequest, - GetEntityAncestorsResponse, -} from './api'; - -/** - * @public - * @deprecated use GetEntitiesRequest instead - */ -export type CatalogEntitiesRequest = GetEntitiesRequest; -/** - * @public - * @deprecated use GetEntitiesResponse instead - */ -export type CatalogListResponse<_Entity> = GetEntitiesResponse; -/** - * @public - * @deprecated use GetEntityAncestorsRequest instead - */ -export type CatalogEntityAncestorsRequest = GetEntityAncestorsRequest; -/** - * @public - * @deprecated use GetEntityAncestorsResponse instead - */ -export type CatalogEntityAncestorsResponse = GetEntityAncestorsResponse; diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 39c8962b68..8bf4e34da2 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -28,5 +28,4 @@ export type { GetEntityFacetsRequest, GetEntityFacetsResponse, } from './api'; -export * from './deprecated'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; From 880967948080d2bb56376be17265cc6ae70e41f7 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 10:21:46 +0100 Subject: [PATCH 127/150] chore: renaming refreshInterval to processingInterval Signed-off-by: blam --- .../DefaultProcessingDatabase.test.ts | 4 +- .../catalog-backend/src/processing/index.ts | 10 ++++- .../catalog-backend/src/processing/refresh.ts | 23 ++++++++++ .../src/service/CatalogBuilder.ts | 42 ++++++++++++++++--- 4 files changed, 69 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 729be442aa..c7cd6b7026 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -28,7 +28,7 @@ import { DbRefreshStateRow, DbRelationsRow, } from './tables'; -import { createRandomRefreshInterval } from '../processing/refresh'; +import { createRandomProcessingInterval } from '../processing/refresh'; import { timestampToDateTime } from './conversion'; import { generateStableHash } from './util'; @@ -49,7 +49,7 @@ describe('Default Processing Database', () => { db: new DefaultProcessingDatabase({ database: knex, logger, - refreshInterval: createRandomRefreshInterval({ + refreshInterval: createRandomProcessingInterval({ minSeconds: 100, maxSeconds: 150, }), diff --git a/plugins/catalog-backend/src/processing/index.ts b/plugins/catalog-backend/src/processing/index.ts index dc8b07eceb..aec6a5d71c 100644 --- a/plugins/catalog-backend/src/processing/index.ts +++ b/plugins/catalog-backend/src/processing/index.ts @@ -23,5 +23,11 @@ export type { } from './types'; export { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; -export { createRandomRefreshInterval } from './refresh'; -export type { RefreshIntervalFunction } from './refresh'; +export { + createRandomRefreshInterval, + createRandomProcessingInterval, +} from './refresh'; +export type { + RefreshIntervalFunction, + ProcessingIntervalFunction, +} from './refresh'; diff --git a/plugins/catalog-backend/src/processing/refresh.ts b/plugins/catalog-backend/src/processing/refresh.ts index 3e03e3add8..941c338912 100644 --- a/plugins/catalog-backend/src/processing/refresh.ts +++ b/plugins/catalog-backend/src/processing/refresh.ts @@ -16,13 +16,21 @@ /** * Function that returns the catalog refresh interval in seconds. + * @deprecated use {@link ProcessingIntervalFunction} instead * @public */ export type RefreshIntervalFunction = () => number; +/** + * Function that returns the catalog processing interval in seconds. + * @public + */ +export type ProcessingIntervalFunction = () => number; + /** * Creates a function that returns a random refresh interval between minSeconds and maxSeconds. * @returns A {@link RefreshIntervalFunction} that provides the next refresh interval + * @deprecated use {@link createRandomProcessingInterval} instead * @public */ export function createRandomRefreshInterval(options: { @@ -34,3 +42,18 @@ export function createRandomRefreshInterval(options: { return Math.random() * (maxSeconds - minSeconds) + minSeconds; }; } + +/** + * Creates a function that returns a random processing interval between minSeconds and maxSeconds. + * @returns A {@link ProcessingIntervalFunction} that provides the next processing interval + * @public + */ +export function createRandomProcessingInterval(options: { + minSeconds: number; + maxSeconds: number; +}): ProcessingIntervalFunction { + const { minSeconds, maxSeconds } = options; + return () => { + return Math.random() * (maxSeconds - minSeconds) + minSeconds; + }; +} diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index f62377decb..ce55bc5937 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -76,8 +76,9 @@ import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; import { Stitcher } from '../stitching/Stitcher'; import { - createRandomRefreshInterval, + createRandomProcessingInterval, RefreshIntervalFunction, + ProcessingIntervalFunction, } from '../processing/refresh'; import { createRouter } from './createRouter'; import { DefaultRefreshService } from './DefaultRefreshService'; @@ -139,8 +140,8 @@ export class CatalogBuilder { private processors: CatalogProcessor[]; private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; - private refreshInterval: RefreshIntervalFunction = - createRandomRefreshInterval({ + private processingInterval: ProcessingIntervalFunction = + createRandomProcessingInterval({ minSeconds: 100, maxSeconds: 150, }); @@ -192,9 +193,25 @@ export class CatalogBuilder { * Seconds provided will be multiplied by 1.5 * The default refresh duration is 100-150 seconds. * setting this too low will potentially deplete request quotas to upstream services. + * + * @deprecated use {@link CatalogBuilder#setProcessingIntervalSeconds} instead */ setRefreshIntervalSeconds(seconds: number): CatalogBuilder { - this.refreshInterval = createRandomRefreshInterval({ + this.processingInterval = createRandomProcessingInterval({ + minSeconds: seconds, + maxSeconds: seconds * 1.5, + }); + return this; + } + + /** + * Processing interval determines how often entities should be processed. + * Seconds provided will be multiplied by 1.5 + * The default processing duration is 100-150 seconds. + * setting this too low will potentially deplete request quotas to upstream services. + */ + setProcessingIntervalSeconds(seconds: number): CatalogBuilder { + this.processingInterval = createRandomProcessingInterval({ minSeconds: seconds, maxSeconds: seconds * 1.5, }); @@ -204,9 +221,22 @@ export class CatalogBuilder { /** * Overwrites the default refresh interval function used to spread * entity updates in the catalog. + * + * @deprecated use {@link CatalogBuilder#setProcessingInterval} instead */ setRefreshInterval(refreshInterval: RefreshIntervalFunction): CatalogBuilder { - this.refreshInterval = refreshInterval; + this.processingInterval = refreshInterval; + return this; + } + + /** + * Overwrites the default processing interval function used to spread + * entity updates in the catalog. + */ + setProcessingInterval( + processingInterval: ProcessingIntervalFunction, + ): CatalogBuilder { + this.processingInterval = processingInterval; return this; } @@ -396,7 +426,7 @@ export class CatalogBuilder { const processingDatabase = new DefaultProcessingDatabase({ database: dbClient, logger, - refreshInterval: this.refreshInterval, + refreshInterval: this.processingInterval, }); const integrations = ScmIntegrations.fromConfig(config); const rulesEnforcer = DefaultCatalogRulesEnforcer.fromConfig(config); From b753d22a564249f547c24f601d782acf0c98a82f Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 10:26:49 +0100 Subject: [PATCH 128/150] chore: added changeset and added logging Signed-off-by: blam --- .changeset/wild-dolphins-lick.md | 7 +++++++ plugins/catalog-backend/src/service/CatalogBuilder.ts | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 .changeset/wild-dolphins-lick.md diff --git a/.changeset/wild-dolphins-lick.md b/.changeset/wild-dolphins-lick.md new file mode 100644 index 0000000000..cdd5827b00 --- /dev/null +++ b/.changeset/wild-dolphins-lick.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +**DEPRECATION**: Deprecated the `RefreshIntervalFunction` and `createRandomRefreshInterval` in favour of the `ProcessingIntervalFunction` and `createRandomProcessingInterval` type and method respectively. Please migrate to use the new names. + +**DEPRECATION**: Deprecated the `setRefreshInterval` and `setRefreshIntervalSeconds` methods on the `CatalogBuilder` for the new `setProcessingInterval` and `setProcessingIntervalSeconds` methods. Please migrate to use the new names. diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index ce55bc5937..3d7de6ba95 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -196,7 +196,10 @@ export class CatalogBuilder { * * @deprecated use {@link CatalogBuilder#setProcessingIntervalSeconds} instead */ - setRefreshIntervalSeconds(seconds: number): CatalogBuilder { + setRefreshIntervalSecon0ds(seconds: number): CatalogBuilder { + this.env.logger.warn( + '[DEPRECATION] - CatalogBuilder.setRefreshIntervalSeconds is deprecated. Use CatalogBuilder.setProcessingIntervalSeconds instead.', + ); this.processingInterval = createRandomProcessingInterval({ minSeconds: seconds, maxSeconds: seconds * 1.5, @@ -225,6 +228,9 @@ export class CatalogBuilder { * @deprecated use {@link CatalogBuilder#setProcessingInterval} instead */ setRefreshInterval(refreshInterval: RefreshIntervalFunction): CatalogBuilder { + this.env.logger.warn( + '[DEPRECATION] - CatalogBuilder.setRefreshInterval is deprecated. Use CatalogBuilder.setProcessingInterval instead.', + ); this.processingInterval = refreshInterval; return this; } From db5bf53ffe145b882b69332c0d5228d4eefd63e3 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 10:28:42 +0100 Subject: [PATCH 129/150] chore: updating api-report with deprecations Signed-off-by: blam --- plugins/catalog-backend/api-report.md | 19 +++++++++++++++++-- .../src/service/CatalogBuilder.ts | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index ee7b419a71..a0e5678e97 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -218,8 +218,14 @@ export class CatalogBuilder { key: string, resolver: PlaceholderResolver, ): CatalogBuilder; + setProcessingInterval( + processingInterval: ProcessingIntervalFunction, + ): CatalogBuilder; + setProcessingIntervalSeconds(seconds: number): CatalogBuilder; + // @deprecated setRefreshInterval(refreshInterval: RefreshIntervalFunction): CatalogBuilder; - setRefreshIntervalSeconds(seconds: number): CatalogBuilder; + // @deprecated + setRefreshIntervalSecon0ds(seconds: number): CatalogBuilder; } // @alpha @@ -408,6 +414,12 @@ export const createCatalogPolicyDecision: ( ) => ConditionalPolicyDecision; // @public +export function createRandomProcessingInterval(options: { + minSeconds: number; + maxSeconds: number; +}): ProcessingIntervalFunction; + +// @public @deprecated export function createRandomRefreshInterval(options: { minSeconds: number; maxSeconds: number; @@ -989,6 +1001,9 @@ export type PlaceholderResolverResolveUrl = ( base: string, ) => string; +// @public +export type ProcessingIntervalFunction = () => number; + // @public export const processingResult: Readonly<{ readonly notFoundError: ( @@ -1023,7 +1038,7 @@ export type RecursivePartial = { : T[P]; }; -// @public +// @public @deprecated export type RefreshIntervalFunction = () => number; // @public diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 3d7de6ba95..179233cb28 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -210,7 +210,7 @@ export class CatalogBuilder { /** * Processing interval determines how often entities should be processed. * Seconds provided will be multiplied by 1.5 - * The default processing duration is 100-150 seconds. + * The default processing interval is 100-150 seconds. * setting this too low will potentially deplete request quotas to upstream services. */ setProcessingIntervalSeconds(seconds: number): CatalogBuilder { From 11bf36fcbc7b9043843cc7263fe666201560beae Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 10:39:39 +0100 Subject: [PATCH 130/150] chore: move some more things to alpha Signed-off-by: blam --- .changeset/sharp-spies-study.md | 5 +++++ .../catalog-backend/src/permissions/rules/hasAnnotation.ts | 2 +- plugins/catalog-backend/src/permissions/rules/hasLabel.ts | 2 +- plugins/catalog-backend/src/permissions/rules/hasSpec.ts | 2 +- .../catalog-backend/src/permissions/rules/isEntityKind.ts | 2 +- .../catalog-backend/src/permissions/rules/isEntityOwner.ts | 2 +- 6 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 .changeset/sharp-spies-study.md diff --git a/.changeset/sharp-spies-study.md b/.changeset/sharp-spies-study.md new file mode 100644 index 0000000000..3027aa1bd8 --- /dev/null +++ b/.changeset/sharp-spies-study.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Moved some more permissions things in the `catalog-backend` to `@alpha` diff --git a/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts b/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts index 81ebd52789..c6c5951153 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts @@ -21,7 +21,7 @@ import { createCatalogPermissionRule } from './util'; * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for the presence of an annotation on a given entity. * - * @public + * @alpha */ export const hasAnnotation = createCatalogPermissionRule({ name: 'HAS_ANNOTATION', diff --git a/plugins/catalog-backend/src/permissions/rules/hasLabel.ts b/plugins/catalog-backend/src/permissions/rules/hasLabel.ts index 04b00d68fa..7296eecd89 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasLabel.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasLabel.ts @@ -20,7 +20,7 @@ import { createCatalogPermissionRule } from './util'; /** * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for entities with a specified label in its metadata. - * @public + * @alpha */ export const hasLabel = createCatalogPermissionRule({ name: 'HAS_LABEL', diff --git a/plugins/catalog-backend/src/permissions/rules/hasSpec.ts b/plugins/catalog-backend/src/permissions/rules/hasSpec.ts index 891cf1d58c..63ae67d01c 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasSpec.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasSpec.ts @@ -23,6 +23,6 @@ import { createPropertyRule } from './createPropertyRule'; * * The key argument to the `apply` and `toQuery` methods can be nested, such as * 'field.nestedfield'. - * @public + * @alpha */ export const hasSpec = createPropertyRule('spec'); diff --git a/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts b/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts index 6356c94dc4..c330c903d9 100644 --- a/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts +++ b/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts @@ -20,7 +20,7 @@ import { createCatalogPermissionRule } from './util'; /** * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for entities with a specified kind. - * @public + * @alpha */ export const isEntityKind = createCatalogPermissionRule({ name: 'IS_ENTITY_KIND', diff --git a/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts b/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts index a6dda13c29..23f118abc4 100644 --- a/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts +++ b/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts @@ -21,7 +21,7 @@ import { createCatalogPermissionRule } from './util'; * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for entities with a specified owner. * - * @public + * @alpha */ export const isEntityOwner = createCatalogPermissionRule({ name: 'IS_ENTITY_OWNER', From 9c238ca4c6e5e6c8a70b95d6bad01f9d0c09ae32 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 10:53:27 +0100 Subject: [PATCH 131/150] chore: moving isOwnerOf to alpha and documentation the limitations Signed-off-by: blam --- plugins/catalog-react/api-report.md | 4 ++-- plugins/catalog-react/src/utils/isOwnerOf.ts | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 75414e3d59..b482bed9a6 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -452,8 +452,8 @@ export function InspectEntityDialog(props: { onClose: () => void; }): JSX.Element | null; -// @public -export function isOwnerOf(owner: Entity, owned: Entity): boolean; +// @alpha +export function isOwnerOf(owner: Entity, entity: Entity): boolean; // @public @deprecated export function loadCatalogOwnerRefs( diff --git a/plugins/catalog-react/src/utils/isOwnerOf.ts b/plugins/catalog-react/src/utils/isOwnerOf.ts index f1ba0fa35f..2b38ace247 100644 --- a/plugins/catalog-react/src/utils/isOwnerOf.ts +++ b/plugins/catalog-react/src/utils/isOwnerOf.ts @@ -24,10 +24,15 @@ import { import { getEntityRelations } from './getEntityRelations'; /** - * Get the related entity references. - * @public + * Returns true if the `owner` argument is a direct owner on the `entity` argument. + * + * @alpha + * @remarks + * + * Note that this ownership is not the same as using the claims in the auth-resolver, it only will take into account ownership as expressed by direct entity relations. + * It doesn't know anything about the additional groups that a user might belong to which the claims contain. */ -export function isOwnerOf(owner: Entity, owned: Entity) { +export function isOwnerOf(owner: Entity, entity: Entity) { const possibleOwners = new Set( [ ...getEntityRelations(owner, RELATION_MEMBER_OF, { kind: 'group' }), @@ -35,7 +40,7 @@ export function isOwnerOf(owner: Entity, owned: Entity) { ].map(stringifyEntityRef), ); - const owners = getEntityRelations(owned, RELATION_OWNED_BY).map( + const owners = getEntityRelations(entity, RELATION_OWNED_BY).map( stringifyEntityRef, ); From 72431d7bedb26787c5b4cf22191fc5673fd0fd0e Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 10:55:08 +0100 Subject: [PATCH 132/150] chore: added changeset Signed-off-by: blam --- .changeset/polite-houses-wink.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/polite-houses-wink.md diff --git a/.changeset/polite-houses-wink.md b/.changeset/polite-houses-wink.md new file mode 100644 index 0000000000..4adceef6ee --- /dev/null +++ b/.changeset/polite-houses-wink.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +- Moving `isOwnerOf` to `@alpha` and documenting the limitations of this function with regards to only supporting direct relations. From 1de7dd85b00185472a07edf9b03d5fe2c36b39e4 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 11:08:57 +0100 Subject: [PATCH 133/150] chore: moving the CatalogEntityDocument to catalog-common Signed-off-by: blam --- plugins/catalog-backend/api-report.md | 16 ++-------- .../search/DefaultCatalogCollatorFactory.ts | 20 +++++-------- plugins/catalog-common/api-report.md | 15 ++++++++++ plugins/catalog-common/package.json | 3 +- plugins/catalog-common/src/index.ts | 2 ++ .../src/search/CatalogEntityDocument.ts | 29 +++++++++++++++++++ plugins/catalog-common/src/search/index.ts | 16 ++++++++++ 7 files changed, 74 insertions(+), 27 deletions(-) create mode 100644 plugins/catalog-common/src/search/CatalogEntityDocument.ts create mode 100644 plugins/catalog-common/src/search/index.ts diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index ee7b419a71..9dcb2db752 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -7,6 +7,7 @@ import { BitbucketIntegration } from '@backstage/integration'; import { CatalogApi } from '@backstage/catalog-client'; +import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConditionalPolicyDecision } from '@backstage/plugin-permission-node'; import { Conditions } from '@backstage/plugin-permission-node'; @@ -18,7 +19,6 @@ import express from 'express'; import { GetEntitiesRequest } from '@backstage/catalog-client'; import { GithubCredentialsProvider } from '@backstage/integration'; import { GitHubIntegrationConfig } from '@backstage/integration'; -import { IndexableDocument } from '@backstage/search-common'; import { JsonObject } from '@backstage/types'; import { JsonValue } from '@backstage/types'; import { Location as Location_2 } from '@backstage/catalog-client'; @@ -248,19 +248,7 @@ export const catalogConditions: Conditions<{ >; }>; -// @public (undocumented) -export interface CatalogEntityDocument extends IndexableDocument { - // (undocumented) - componentType: string; - // (undocumented) - kind: string; - // (undocumented) - lifecycle: string; - // (undocumented) - namespace: string; - // (undocumented) - owner: string; -} +export { CatalogEntityDocument }; // @public (undocumented) export type CatalogEnvironment = { diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts index 99377b2ca0..811ac10210 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -29,21 +29,17 @@ import { UserEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; +import { DocumentCollatorFactory } from '@backstage/search-common'; import { - DocumentCollatorFactory, - IndexableDocument, -} from '@backstage/search-common'; -import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; + catalogEntityReadPermission, + CatalogEntityDocument, +} from '@backstage/plugin-catalog-common'; import { Readable } from 'stream'; -/** @public */ -export interface CatalogEntityDocument extends IndexableDocument { - componentType: string; - namespace: string; - kind: string; - lifecycle: string; - owner: string; -} +/** + * @deprecated import from `@backstage/plugin-catalog-common` instead + */ +export type { CatalogEntityDocument }; /** @public */ export type DefaultCatalogCollatorFactoryOptions = { diff --git a/plugins/catalog-common/api-report.md b/plugins/catalog-common/api-report.md index 57f8c3983e..8ae9571d4c 100644 --- a/plugins/catalog-common/api-report.md +++ b/plugins/catalog-common/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { IndexableDocument } from '@backstage/search-common'; import { Permission } from '@backstage/plugin-permission-common'; // @alpha @@ -11,6 +12,20 @@ export const catalogEntityCreatePermission: Permission; // @alpha export const catalogEntityDeletePermission: Permission; +// @public +export interface CatalogEntityDocument extends IndexableDocument { + // (undocumented) + componentType: string; + // (undocumented) + kind: string; + // (undocumented) + lifecycle: string; + // (undocumented) + namespace: string; + // (undocumented) + owner: string; +} + // @alpha export const catalogEntityReadPermission: Permission; diff --git a/plugins/catalog-common/package.json b/plugins/catalog-common/package.json index acf4ed4aaa..bc0d3750ba 100644 --- a/plugins/catalog-common/package.json +++ b/plugins/catalog-common/package.json @@ -34,7 +34,8 @@ "clean": "backstage-cli package clean" }, "dependencies": { - "@backstage/plugin-permission-common": "^0.5.1" + "@backstage/plugin-permission-common": "^0.5.1", + "@backstage/search-common": "^0.2.4" }, "devDependencies": { "@backstage/cli": "^0.14.0" diff --git a/plugins/catalog-common/src/index.ts b/plugins/catalog-common/src/index.ts index 9eddf37452..08184eb871 100644 --- a/plugins/catalog-common/src/index.ts +++ b/plugins/catalog-common/src/index.ts @@ -31,3 +31,5 @@ export { catalogLocationCreatePermission, catalogLocationDeletePermission, } from './permissions'; + +export * from './search'; diff --git a/plugins/catalog-common/src/search/CatalogEntityDocument.ts b/plugins/catalog-common/src/search/CatalogEntityDocument.ts new file mode 100644 index 0000000000..e48cde431a --- /dev/null +++ b/plugins/catalog-common/src/search/CatalogEntityDocument.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { IndexableDocument } from '@backstage/search-common'; + +/** + * The Document format for an Entity in the Catalog for search + * + * @public + */ +export interface CatalogEntityDocument extends IndexableDocument { + componentType: string; + namespace: string; + kind: string; + lifecycle: string; + owner: string; +} diff --git a/plugins/catalog-common/src/search/index.ts b/plugins/catalog-common/src/search/index.ts new file mode 100644 index 0000000000..82e713aba0 --- /dev/null +++ b/plugins/catalog-common/src/search/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export type { CatalogEntityDocument } from './CatalogEntityDocument'; From ab7b6cb7b1116ea5376e997ba2bfadb272f6dc5b Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 11:13:15 +0100 Subject: [PATCH 134/150] chore: reworking exports Signed-off-by: blam --- .changeset/small-hornets-dress.md | 6 ++++++ .../src/search/DefaultCatalogCollator.ts | 11 ++++------- plugins/catalog-backend/src/search/index.ts | 4 +++- 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 .changeset/small-hornets-dress.md diff --git a/.changeset/small-hornets-dress.md b/.changeset/small-hornets-dress.md new file mode 100644 index 0000000000..6d6817977d --- /dev/null +++ b/.changeset/small-hornets-dress.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-catalog-common': patch +--- + +Moved the `CatalogEntityDocument` to `@backstage/plugin-catalog-common` and deprecated the export from `@backstage/plugin-catalog-backend` diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts index c59318d0af..a24c6d4f67 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts @@ -29,14 +29,11 @@ import { CatalogClient, GetEntitiesRequest, } from '@backstage/catalog-client'; -import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; -import { CatalogEntityDocument } from './DefaultCatalogCollatorFactory'; +import { + catalogEntityReadPermission, + CatalogEntityDocument, +} from '@backstage/plugin-catalog-common'; -/** - * @public - * @deprecated Upgrade to a more recent `@backstage/search-backend-node` and - * use `DefaultCatalogCollatorFactory` instead. - */ export class DefaultCatalogCollator { protected discovery: PluginEndpointDiscovery; protected locationTemplate: string; diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index 93ff0b8b32..bd54e33e59 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -16,7 +16,9 @@ export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; -export type { CatalogEntityDocument } from './DefaultCatalogCollatorFactory'; + +/** @public @deprecated use the export from `plugin-catalog-common` instead */ +export type { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; /** * todo(backstage/techdocs-core): stop exporting this in a future release. From 8b3678e120b3acd105b78bd29d6d9b2f0b3a3368 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 11:28:20 +0100 Subject: [PATCH 135/150] chore: reworking the api-report Signed-off-by: blam --- plugins/catalog-backend/api-report.md | 2 +- plugins/catalog-backend/src/search/DefaultCatalogCollator.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 9dcb2db752..ad07b002ea 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -404,7 +404,7 @@ export function createRandomRefreshInterval(options: { // @public export function createRouter(options: RouterOptions): Promise; -// @public @deprecated (undocumented) +// @public export class DefaultCatalogCollator { constructor(options: { discovery: PluginEndpointDiscovery; diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts index a24c6d4f67..a577147082 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts @@ -34,6 +34,10 @@ import { CatalogEntityDocument, } from '@backstage/plugin-catalog-common'; +/** + * The DefaultCatalogCollator for Search + * @public + */ export class DefaultCatalogCollator { protected discovery: PluginEndpointDiscovery; protected locationTemplate: string; From 633019509ac78b80ab2d7fcfc339857eab710d9b Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 11:38:26 +0100 Subject: [PATCH 136/150] chore: reworking some more things Signed-off-by: blam --- plugins/catalog-backend/src/search/DefaultCatalogCollator.ts | 1 + .../src/search/DefaultCatalogCollatorFactory.ts | 1 + plugins/catalog-common/api-report.md | 4 +++- plugins/catalog-common/src/search/CatalogEntityDocument.ts | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts index a577147082..7700048b14 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts @@ -126,6 +126,7 @@ export class DefaultCatalogCollator { }), text: this.getDocumentText(entity), componentType: entity.spec?.type?.toString() || 'other', + type: entity.spec?.type?.toString() || 'other', namespace: entity.metadata.namespace || 'default', kind: entity.kind, lifecycle: (entity.spec?.lifecycle as string) || '', diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts index 811ac10210..ca6395b60a 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -155,6 +155,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { }), text: this.getDocumentText(entity), componentType: entity.spec?.type?.toString() || 'other', + type: entity.spec?.type?.toString() || 'other', namespace: entity.metadata.namespace || 'default', kind: entity.kind, lifecycle: (entity.spec?.lifecycle as string) || '', diff --git a/plugins/catalog-common/api-report.md b/plugins/catalog-common/api-report.md index 8ae9571d4c..a73cae9994 100644 --- a/plugins/catalog-common/api-report.md +++ b/plugins/catalog-common/api-report.md @@ -14,7 +14,7 @@ export const catalogEntityDeletePermission: Permission; // @public export interface CatalogEntityDocument extends IndexableDocument { - // (undocumented) + // @deprecated (undocumented) componentType: string; // (undocumented) kind: string; @@ -24,6 +24,8 @@ export interface CatalogEntityDocument extends IndexableDocument { namespace: string; // (undocumented) owner: string; + // (undocumented) + type: string; } // @alpha diff --git a/plugins/catalog-common/src/search/CatalogEntityDocument.ts b/plugins/catalog-common/src/search/CatalogEntityDocument.ts index e48cde431a..005bc79c92 100644 --- a/plugins/catalog-common/src/search/CatalogEntityDocument.ts +++ b/plugins/catalog-common/src/search/CatalogEntityDocument.ts @@ -21,7 +21,9 @@ import { IndexableDocument } from '@backstage/search-common'; * @public */ export interface CatalogEntityDocument extends IndexableDocument { + /** @deprecated use `type` as well, as `componentType` will be removed after a few releases but we dont want to break indexing */ componentType: string; + type: string; namespace: string; kind: string; lifecycle: string; From 50b09a5fffb1bfa4540014ba11fdf4c3d7609e8d Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 11:43:31 +0100 Subject: [PATCH 137/150] chore: fix Signed-off-by: blam --- plugins/catalog-backend/api-report.md | 2 +- plugins/catalog-backend/src/search/DefaultCatalogCollator.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index ad07b002ea..9dcb2db752 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -404,7 +404,7 @@ export function createRandomRefreshInterval(options: { // @public export function createRouter(options: RouterOptions): Promise; -// @public +// @public @deprecated (undocumented) export class DefaultCatalogCollator { constructor(options: { discovery: PluginEndpointDiscovery; diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts index 7700048b14..ac9a33bf62 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts @@ -35,8 +35,9 @@ import { } from '@backstage/plugin-catalog-common'; /** - * The DefaultCatalogCollator for Search * @public + * @deprecated Upgrade to a more recent `@backstage/search-backend-node` and + * use `DefaultCatalogCollatorFactory` instead. */ export class DefaultCatalogCollator { protected discovery: PluginEndpointDiscovery; From ac2365b451439c25ce566e6f3e068dc5ef6b6624 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 11:46:27 +0100 Subject: [PATCH 138/150] chore: -0 Signed-off-by: blam --- plugins/catalog-backend/api-report.md | 2 +- plugins/catalog-backend/src/service/CatalogBuilder.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index a0e5678e97..a1125ec529 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -225,7 +225,7 @@ export class CatalogBuilder { // @deprecated setRefreshInterval(refreshInterval: RefreshIntervalFunction): CatalogBuilder; // @deprecated - setRefreshIntervalSecon0ds(seconds: number): CatalogBuilder; + setRefreshIntervalSeconds(seconds: number): CatalogBuilder; } // @alpha diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 179233cb28..b53335c469 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -196,7 +196,7 @@ export class CatalogBuilder { * * @deprecated use {@link CatalogBuilder#setProcessingIntervalSeconds} instead */ - setRefreshIntervalSecon0ds(seconds: number): CatalogBuilder { + setRefreshIntervalSeconds(seconds: number): CatalogBuilder { this.env.logger.warn( '[DEPRECATION] - CatalogBuilder.setRefreshIntervalSeconds is deprecated. Use CatalogBuilder.setProcessingIntervalSeconds instead.', ); From debfcd9515ffdce61648937310702d5388bf975a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 3 Mar 2022 11:50:46 +0100 Subject: [PATCH 139/150] move @types/json-schema to a dev dep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/beige-snails-drum.md | 5 +++++ packages/catalog-model/package.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/beige-snails-drum.md diff --git a/.changeset/beige-snails-drum.md b/.changeset/beige-snails-drum.md new file mode 100644 index 0000000000..3476c9383b --- /dev/null +++ b/.changeset/beige-snails-drum.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-model': patch +--- + +Move `@types/json-schema` to be a dev dependency diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json index 70ed0fbe4d..f9781b14a2 100644 --- a/packages/catalog-model/package.json +++ b/packages/catalog-model/package.json @@ -37,7 +37,6 @@ "@backstage/config": "^0.1.15", "@backstage/errors": "^0.2.2", "@backstage/types": "^0.1.3", - "@types/json-schema": "^7.0.5", "ajv": "^7.0.3", "json-schema": "^0.4.0", "lodash": "^4.17.21", @@ -46,6 +45,7 @@ "devDependencies": { "@backstage/cli": "^0.14.1", "@types/jest": "^26.0.7", + "@types/json-schema": "^7.0.5", "@types/lodash": "^4.14.151", "yaml": "^1.9.2" }, From a52f69987a5aa958e8da1aa214629f1d50c56be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 3 Mar 2022 10:32:41 +0100 Subject: [PATCH 140/150] getEntityByName -> getEntityByRef MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/rotten-windows-worry.md | 5 ++ packages/catalog-client/api-report.md | 9 ++++ .../catalog-client/src/CatalogClient.test.ts | 54 +++++++++++++++++++ packages/catalog-client/src/CatalogClient.ts | 21 ++++++++ packages/catalog-client/src/types/api.ts | 14 +++++ 5 files changed, 103 insertions(+) create mode 100644 .changeset/rotten-windows-worry.md diff --git a/.changeset/rotten-windows-worry.md b/.changeset/rotten-windows-worry.md new file mode 100644 index 0000000000..a1ab353ce3 --- /dev/null +++ b/.changeset/rotten-windows-worry.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': patch +--- + +**DEPRECATION**: Deprecated `getEntityByName` from `CatalogApi` and added `getEntityByRef` instead, which accepts both string and compound ref forms. diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index ef19effe90..9bbb24463b 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -38,10 +38,15 @@ export interface CatalogApi { request: GetEntityAncestorsRequest, options?: CatalogRequestOptions, ): Promise; + // @deprecated getEntityByName( name: CompoundEntityRef, options?: CatalogRequestOptions, ): Promise; + getEntityByRef( + entityRef: string | CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise; getEntityFacets( request: GetEntityFacetsRequest, options?: CatalogRequestOptions, @@ -94,6 +99,10 @@ export class CatalogClient implements CatalogApi { compoundName: CompoundEntityRef, options?: CatalogRequestOptions, ): Promise; + getEntityByRef( + entityRef: string | CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise; getEntityFacets( request: GetEntityFacetsRequest, options?: CatalogRequestOptions, diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 0035691f2c..0b838eb4c2 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -195,6 +195,60 @@ describe('CatalogClient', () => { }); }); + describe('getEntityByRef', () => { + const existingEntity: Entity = { + apiVersion: 'v1', + kind: 'CustomKind', + metadata: { + namespace: 'default', + name: 'exists', + }, + }; + + beforeEach(() => { + server.use( + rest.get( + `${mockBaseUrl}/entities/by-name/customkind/default/exists`, + (_, res, ctx) => { + return res(ctx.json(existingEntity)); + }, + ), + rest.get( + `${mockBaseUrl}/entities/by-name/customkind/default/missing`, + (_, res, ctx) => { + return res(ctx.status(404)); + }, + ), + ); + }); + + it('finds by string and compound', async () => { + await expect( + client.getEntityByRef('customkind:default/exists'), + ).resolves.toEqual(existingEntity); + await expect( + client.getEntityByRef({ + kind: 'CustomKind', + namespace: 'default', + name: 'exists', + }), + ).resolves.toEqual(existingEntity); + }); + + it('returns undefined for 404s', async () => { + await expect( + client.getEntityByRef('customkind:default/missing'), + ).resolves.toBeUndefined(); + await expect( + client.getEntityByRef({ + kind: 'CustomKind', + namespace: 'default', + name: 'missing', + }), + ).resolves.toBeUndefined(); + }); + }); + describe('getLocationById', () => { const defaultResponse = { data: { diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index da580ae09c..f2bc3ed8f6 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -170,6 +170,27 @@ export class CatalogClient implements CatalogApi { return { items: entities.sort(refCompare) }; } + /** + * {@inheritdoc CatalogApi.getEntityByRef} + */ + async getEntityByRef( + entityRef: string | CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise { + const { kind, namespace, name } = parseEntityRef(entityRef); + return this.requestOptional( + 'GET', + `/entities/by-name/${encodeURIComponent(kind)}/${encodeURIComponent( + namespace, + )}/${encodeURIComponent(name)}`, + options, + ); + } + + // NOTE(freben): When we deprecate getEntityByName from the interface, we may + // still want to leave this implementation in place for quite some time + // longer, to minimize the risk for breakages. Suggested date for removal: + // August 2022 /** * {@inheritdoc CatalogApi.getEntityByName} */ diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 0ee2b57ffb..26af3889e2 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -306,6 +306,20 @@ export interface CatalogApi { * Gets a single entity from the catalog by its ref (kind, namespace, name) * triplet. * + * @param entityRef - A complete entity ref, either on string or compound form + * @param options - Additional options + * @returns The matching entity, or undefined if there was no entity with that ref + */ + getEntityByRef( + entityRef: string | CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise; + + /** + * Gets a single entity from the catalog by its ref (kind, namespace, name) + * triplet. + * + * @deprecated Use getEntityRef instead * @param name - A complete entity ref * @param options - Additional options */ From 0ca28cc2c702fa403b263bd5750156d54ffabc0e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 3 Mar 2022 12:41:26 +0100 Subject: [PATCH 141/150] Update polite-houses-wink.md Signed-off-by: Patrik Oldsberg --- .changeset/polite-houses-wink.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/polite-houses-wink.md b/.changeset/polite-houses-wink.md index 4adceef6ee..3e177f5383 100644 --- a/.changeset/polite-houses-wink.md +++ b/.changeset/polite-houses-wink.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-react': patch --- -- Moving `isOwnerOf` to `@alpha` and documenting the limitations of this function with regards to only supporting direct relations. +- **BREAKING**: The `isOwnerOf` function has been marked as `@alpha` and is now only available via the `@backstage/plugin-catalog-react/alpha` import. The limitations of this function with regards to only supporting direct relations have also been documented. From c820a49426647212298e78eaf286a5e2e4232e05 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 25 Feb 2022 21:47:07 +0100 Subject: [PATCH 142/150] feat(msgraph): add `groupExpand` config option Add `groupExpand` allowing to use the `$expand` query parameter by the Microsoft Graph API to expand a single relationship. Relates-to: issue #9819 Relates-to: PR #9826 Signed-off-by: Patrick Jungermann --- .changeset/fluffy-trees-occur.md | 5 + .../catalog-backend-module-msgraph/README.md | 6 ++ .../api-report.md | 2 + .../src/microsoftGraph/config.test.ts | 2 + .../src/microsoftGraph/config.ts | 8 ++ .../src/microsoftGraph/read.test.ts | 98 ++++++++++++++++++- .../src/microsoftGraph/read.ts | 13 ++- .../MicrosoftGraphOrgReaderProcessor.ts | 1 + 8 files changed, 130 insertions(+), 5 deletions(-) create mode 100644 .changeset/fluffy-trees-occur.md diff --git a/.changeset/fluffy-trees-occur.md b/.changeset/fluffy-trees-occur.md new file mode 100644 index 0000000000..6e41a61069 --- /dev/null +++ b/.changeset/fluffy-trees-occur.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +add config option `groupExpand` to allow expanding a single relationship diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index daaf1ec225..a253359471 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -51,6 +51,12 @@ catalog: # This and userFilter are mutually exclusive, only one can be specified # See https://docs.microsoft.com/en-us/graph/search-query-parameter userGroupMemberFilter: "displayName eq 'Backstage Users'" + # Optional parameter to include the expanded resource or collection referenced + # by a single relationship (navigation property) in your results. + # Only one relationship can be expanded in a single request. + # See https://docs.microsoft.com/en-us/graph/query-parameters#expand-parameter + # Can be combined with userGroupMember[...] instead of userFilter. + groupExpand: member # Optional search for users, use group membership to get users. # (Search for groups and fetch their members.) # This and userFilter are mutually exclusive, only one can be specified diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index 574b948dea..8df06f54b1 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -164,6 +164,7 @@ export type MicrosoftGraphProviderConfig = { userExpand?: string; userGroupMemberFilter?: string; userGroupMemberSearch?: string; + groupExpand?: string; groupFilter?: string; groupSearch?: string; }; @@ -198,6 +199,7 @@ export function readMicrosoftGraphOrg( userFilter?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; + groupExpand?: string; groupSearch?: string; groupFilter?: string; userTransformer?: UserTransformer; diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts index 700e5cd8fe..cfc5c1cbb0 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts @@ -55,6 +55,7 @@ describe('readMicrosoftGraphConfig', () => { authority: 'https://login.example.com/', userExpand: 'manager', userFilter: 'accountEnabled eq true', + groupExpand: 'member', groupFilter: 'securityEnabled eq false', }, ], @@ -69,6 +70,7 @@ describe('readMicrosoftGraphConfig', () => { authority: 'https://login.example.com', userExpand: 'manager', userFilter: 'accountEnabled eq true', + groupExpand: 'member', groupFilter: 'securityEnabled eq false', }, ]; diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index c2789d767a..b7fbcfb8a9 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -70,6 +70,12 @@ export type MicrosoftGraphProviderConfig = { * E.g. "\"displayName:-team\"" would only match groups which contain '-team' */ userGroupMemberSearch?: string; + /** + * The "expand" argument to apply to groups. + * + * E.g. "member" + */ + groupExpand?: string; /** * The filter to apply to extract groups. * @@ -115,6 +121,7 @@ export function readMicrosoftGraphConfig( const userGroupMemberSearch = providerConfig.getOptionalString( 'userGroupMemberSearch', ); + const groupExpand = providerConfig.getOptionalString('groupExpand'); const groupFilter = providerConfig.getOptionalString('groupFilter'); const groupSearch = providerConfig.getOptionalString('groupSearch'); @@ -139,6 +146,7 @@ export function readMicrosoftGraphConfig( userFilter, userGroupMemberFilter, userGroupMemberSearch, + groupExpand, groupFilter, groupSearch, }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts index dba3741c73..e3d6897d66 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts @@ -236,7 +236,7 @@ describe('read microsoft graph', () => { expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); - it('should read users with userExpand and custom transformer', async () => { + it('should read users with userExpand, groupExpand and custom transformer', async () => { async function* getExampleGroups() { yield { id: 'groupid', @@ -272,6 +272,7 @@ describe('read microsoft graph', () => { const { users } = await readMicrosoftGraphUsersInGroups(client, { userExpand: 'manager', userGroupMemberFilter: 'securityEnabled eq true', + groupExpand: 'member', transformer: async () => ({ apiVersion: 'backstage.io/v1alpha1', kind: 'User', @@ -292,6 +293,7 @@ describe('read microsoft graph', () => { expect(client.getGroups).toBeCalledTimes(1); expect(client.getGroups).toBeCalledWith({ + expand: 'member', filter: 'securityEnabled eq true', }); expect(client.getGroupMembers).toBeCalledTimes(1); @@ -453,6 +455,100 @@ describe('read microsoft graph', () => { // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); }); + it('should read groups with groupExpand', async () => { + async function* getExampleGroups() { + yield { + id: 'groupid', + displayName: 'Group Name', + description: 'Group Description', + mail: 'group@example.com', + }; + } + + async function* getExampleGroupMembers(): AsyncIterable { + yield { + '@odata.type': '#microsoft.graph.group', + id: 'childgroupid', + }; + yield { + '@odata.type': '#microsoft.graph.user', + id: 'userid', + }; + } + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementation(getExampleGroupMembers); + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', + }); + client.getGroupPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { groups, groupMember, groupMemberOf, rootGroup } = + await readMicrosoftGraphGroups(client, 'tenantid', { + groupExpand: 'member', + groupFilter: 'securityEnabled eq false', + }); + + const expectedRootGroup = group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenantid', + }, + name: 'organization_name', + description: 'Organization Name', + }, + spec: { + type: 'root', + profile: { + displayName: 'Organization Name', + }, + children: [], + }, + }); + expect(groups).toEqual([ + expectedRootGroup, + group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'groupid', + }, + name: 'group_name', + description: 'Group Description', + }, + spec: { + type: 'team', + profile: { + displayName: 'Group Name', + email: 'group@example.com', + // TODO: Loading groups photos doesn't work right now as Microsoft + // Graph doesn't allows this yet + /* picture: 'data:image/jpeg;base64,...',*/ + }, + children: [], + }, + }), + ]); + expect(rootGroup).toEqual(expectedRootGroup); + expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid'])); + expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid'])); + expect(groupMember.get('organization_name')).toEqual(new Set()); + + expect(client.getGroups).toBeCalledTimes(1); + expect(client.getGroups).toBeCalledWith({ + expand: 'member', + filter: 'securityEnabled eq false', + }); + expect(client.getGroupMembers).toBeCalledTimes(1); + expect(client.getGroupMembers).toBeCalledWith('groupid'); + // TODO: Loading groups photos doesn't work right now as Microsoft Graph + // doesn't allows this yet + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1); + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); + }); + it('should read security groups', async () => { async function* getExampleGroups() { yield { diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index 333eaa6b0a..ed6ca76e68 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -140,6 +140,7 @@ export async function readMicrosoftGraphUsersInGroups( userExpand?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; + groupExpand?: string; transformer?: UserTransformer; logger: Logger; }, @@ -150,15 +151,16 @@ export async function readMicrosoftGraphUsersInGroups( const limiter = limiterFactory(10); - const transformer = options?.transformer ?? defaultUserTransformer; + const transformer = options.transformer ?? defaultUserTransformer; const userGroupMemberPromises: Promise[] = []; const userPromises: Promise[] = []; const groupMemberUsers: Set = new Set(); for await (const group of client.getGroups({ - search: options?.userGroupMemberSearch, - filter: options?.userGroupMemberFilter, + expand: options.groupExpand, + search: options.userGroupMemberSearch, + filter: options.userGroupMemberFilter, })) { // Process all groups in parallel, otherwise it can take quite some time userGroupMemberPromises.push( @@ -329,8 +331,9 @@ export async function readMicrosoftGraphGroups( client: MicrosoftGraphClient, tenantId: string, options?: { - groupSearch?: string; + groupExpand?: string; groupFilter?: string; + groupSearch?: string; groupTransformer?: GroupTransformer; organizationTransformer?: OrganizationTransformer; }, @@ -357,6 +360,7 @@ export async function readMicrosoftGraphGroups( const promises: Promise[] = []; for await (const group of client.getGroups({ + expand: options?.groupExpand, search: options?.groupSearch, filter: options?.groupFilter, })) { @@ -513,6 +517,7 @@ export async function readMicrosoftGraphOrg( userFilter?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; + groupExpand?: string; groupSearch?: string; groupFilter?: string; userTransformer?: UserTransformer; diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 942bde19eb..9037bd8728 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -109,6 +109,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { userFilter: provider.userFilter, userGroupMemberFilter: provider.userGroupMemberFilter, userGroupMemberSearch: provider.userGroupMemberSearch, + groupExpand: provider.groupExpand, groupFilter: provider.groupFilter, groupSearch: provider.groupSearch, userTransformer: this.userTransformer, From c870c3bb99faeecc39cd918c34c310fd1d1582d9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 3 Mar 2022 12:44:42 +0100 Subject: [PATCH 143/150] changesets: removed changeset catalog permission alpha tweaks Signed-off-by: Patrik Oldsberg --- .changeset/sharp-spies-study.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/sharp-spies-study.md diff --git a/.changeset/sharp-spies-study.md b/.changeset/sharp-spies-study.md deleted file mode 100644 index 3027aa1bd8..0000000000 --- a/.changeset/sharp-spies-study.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog-backend': patch ---- - -Moved some more permissions things in the `catalog-backend` to `@alpha` From 5da4d0cfbd9feccdca1be3e2f7129a3c6edd4a6e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 3 Mar 2022 12:46:47 +0100 Subject: [PATCH 144/150] catalog-backend: also mark hasMetadata as alpha Signed-off-by: Patrik Oldsberg --- plugins/catalog-backend/src/permissions/rules/hasMetadata.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts b/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts index f5f25a5ecf..cf14faea14 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts @@ -23,6 +23,6 @@ import { createPropertyRule } from './createPropertyRule'; * * The key argument to the `apply` and `toQuery` methods can be nested, such as * 'field.nestedfield'. - * @public + * @alpha */ export const hasMetadata = createPropertyRule('metadata'); From 8a687faa7bfb2616345a55a4e06e4dd618298cb4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 3 Mar 2022 12:51:15 +0100 Subject: [PATCH 145/150] Update tasty-poems-raise.md Signed-off-by: Patrik Oldsberg --- .changeset/tasty-poems-raise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/tasty-poems-raise.md b/.changeset/tasty-poems-raise.md index 47556cb3ad..fbd81423b6 100644 --- a/.changeset/tasty-poems-raise.md +++ b/.changeset/tasty-poems-raise.md @@ -2,4 +2,4 @@ '@backstage/plugin-auth-backend': minor --- -Added validation to TokenFactory.issueToken that ensure any sub claim given is a valid entityRef. This will affect any custom resolver functions given to auth providers. +**BREAKING**: The `TokenFactory.issueToken` used by custom sign-in resolvers now ensures that the sub claim given is a full entity reference of the format `:/`. Any existing custom sign-in resolver functions that do not supply a full entity reference must be updated. From 899f196af5df1b0bc49e4135bdf07a0583faecc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 3 Mar 2022 11:26:42 +0100 Subject: [PATCH 146/150] update to use getEntityByRef MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/gentle-icons-vanish.md | 20 ++++++++++++ plugins/api-docs/dev/index.tsx | 4 +-- .../DefaultApiExplorerPage.test.tsx | 10 ++++-- .../lib/catalog/CatalogIdentityClient.test.ts | 1 + .../badges-backend/src/service/router.test.ts | 17 +++++----- plugins/badges-backend/src/service/router.ts | 4 +-- plugins/catalog-graph/dev/index.tsx | 8 +++-- .../CatalogGraphCard.test.tsx | 5 +-- .../CatalogGraphPage.test.tsx | 7 +++-- .../EntityRelationsGraph.test.tsx | 31 ++++++++++--------- .../useEntityStore.test.ts | 17 +++++----- .../EntityRelationsGraph/useEntityStore.ts | 6 ++-- .../src/api/CatalogImportClient.test.ts | 1 + .../StepPrepareCreatePullRequest.test.tsx | 1 + .../UserListPicker/UserListPicker.test.tsx | 2 +- plugins/catalog-react/src/hooks/useEntity.tsx | 2 +- .../src/hooks/useEntityListProvider.test.tsx | 2 +- .../src/hooks/useEntityOwnership.test.tsx | 14 ++++----- .../src/hooks/useEntityOwnership.ts | 2 +- plugins/catalog-react/src/hooks/useOwnUser.ts | 7 ++++- .../CatalogEntityPage/useEntityFromUrl.ts | 2 +- .../CatalogPage/DefaultCatalogPage.test.tsx | 5 +-- .../src/service/router.ts | 21 +++++-------- .../DefaultExplorePage.test.tsx | 1 + .../DomainExplorerContent.test.tsx | 1 + .../GroupsExplorerContent.test.tsx | 1 + .../components/FossaPage/FossaPage.test.tsx | 1 + plugins/jenkins-backend/README.md | 2 +- .../src/service/jenkinsInfoProvider.test.ts | 20 ++++++------ .../src/service/jenkinsInfoProvider.ts | 2 +- plugins/rollbar/src/hooks/useCatalogEntity.ts | 2 +- .../scaffolder-backend/src/service/helpers.ts | 7 +++-- .../src/service/router.test.ts | 2 +- .../src/service/CachedEntityLoader.test.ts | 12 +++---- .../src/service/CachedEntityLoader.ts | 6 ++-- .../components/DefaultTechDocsHome.test.tsx | 2 +- .../components/LegacyTechDocsHome.test.tsx | 2 +- .../components/TechDocsCustomHome.test.tsx | 2 +- .../home/components/TechDocsCustomHome.tsx | 7 ++++- .../src/service/TodoReaderService.test.ts | 7 +++-- .../src/service/TodoReaderService.ts | 2 +- 41 files changed, 157 insertions(+), 111 deletions(-) create mode 100644 .changeset/gentle-icons-vanish.md diff --git a/.changeset/gentle-icons-vanish.md b/.changeset/gentle-icons-vanish.md new file mode 100644 index 0000000000..27ea78863b --- /dev/null +++ b/.changeset/gentle-icons-vanish.md @@ -0,0 +1,20 @@ +--- +'@backstage/plugin-api-docs': patch +'@backstage/plugin-auth-backend': patch +'@backstage/plugin-badges-backend': patch +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-graph': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-code-coverage-backend': patch +'@backstage/plugin-explore': patch +'@backstage/plugin-fossa': patch +'@backstage/plugin-jenkins-backend': patch +'@backstage/plugin-rollbar': patch +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs-backend': patch +'@backstage/plugin-todo-backend': patch +--- + +Use `getEntityByRef` instead of `getEntityByName` in the catalog client diff --git a/plugins/api-docs/dev/index.tsx b/plugins/api-docs/dev/index.tsx index 633a285b98..9975e91e98 100644 --- a/plugins/api-docs/dev/index.tsx +++ b/plugins/api-docs/dev/index.tsx @@ -53,8 +53,8 @@ createDevApp() items: mockEntities.slice(), }; }, - async getEntityByName(name: string) { - return mockEntities.find(e => e.metadata.name === name); + async getEntityByRef(ref: string) { + return mockEntities.find(e => e.metadata.name === ref); }, } as unknown as typeof catalogApiRef.T), }) diff --git a/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx b/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx index 5924af3ef3..bc413661a8 100644 --- a/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx +++ b/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx @@ -14,7 +14,11 @@ * limitations under the License. */ -import { Entity, RELATION_MEMBER_OF } from '@backstage/catalog-model'; +import { + Entity, + parseEntityRef, + RELATION_MEMBER_OF, +} from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/core-app-api'; import { TableColumn, TableProps } from '@backstage/core-components'; import { @@ -60,11 +64,11 @@ describe('DefaultApiExplorerPage', () => { }), getLocationByRef: () => Promise.resolve({ id: 'id', type: 'url', target: 'url' }), - getEntityByName: async entityName => { + getEntityByRef: async entityRef => { return { apiVersion: 'backstage.io/v1alpha1', kind: 'User', - metadata: { name: entityName.name }, + metadata: { name: parseEntityRef(entityRef).name }, relations: [ { type: RELATION_MEMBER_OF, diff --git a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts index d2cee7bd89..08c46f2c21 100644 --- a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts +++ b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts @@ -26,6 +26,7 @@ import { CatalogIdentityClient } from './CatalogIdentityClient'; describe('CatalogIdentityClient', () => { const catalogApi: jest.Mocked = { getLocationById: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getEntities: jest.fn(), addLocation: jest.fn(), diff --git a/plugins/badges-backend/src/service/router.test.ts b/plugins/badges-backend/src/service/router.test.ts index 56dbfb0655..c492449cb3 100644 --- a/plugins/badges-backend/src/service/router.test.ts +++ b/plugins/badges-backend/src/service/router.test.ts @@ -60,6 +60,7 @@ describe('createRouter', () => { catalog = { addLocation: jest.fn(), getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), @@ -103,7 +104,7 @@ describe('createRouter', () => { describe('GET /entity/:namespace/:kind/:name/badge-specs', () => { it('returns all badge specs for entity', async () => { - catalog.getEntityByName.mockResolvedValueOnce(entity); + catalog.getEntityByRef.mockResolvedValueOnce(entity); badgeBuilder.getBadges.mockResolvedValueOnce([{ id: badge.id }]); badgeBuilder.createBadgeJson.mockResolvedValueOnce(badge); @@ -115,8 +116,8 @@ describe('createRouter', () => { expect(response.status).toEqual(200); expect(response.text).toEqual(JSON.stringify([badge], null, 2)); - expect(catalog.getEntityByName).toHaveBeenCalledTimes(1); - expect(catalog.getEntityByName).toHaveBeenCalledWith( + expect(catalog.getEntityByRef).toHaveBeenCalledTimes(1); + expect(catalog.getEntityByRef).toHaveBeenCalledWith( { namespace: 'default', kind: 'service', @@ -142,7 +143,7 @@ describe('createRouter', () => { describe('GET /entity/:namespace/:kind/:name/badge/test-badge', () => { it('returns badge for entity', async () => { - catalog.getEntityByName.mockResolvedValueOnce(entity); + catalog.getEntityByRef.mockResolvedValueOnce(entity); const image = '...'; badgeBuilder.createBadgeSvg.mockResolvedValueOnce(image); @@ -154,8 +155,8 @@ describe('createRouter', () => { expect(response.status).toEqual(200); expect(response.body).toEqual(Buffer.from(image)); - expect(catalog.getEntityByName).toHaveBeenCalledTimes(1); - expect(catalog.getEntityByName).toHaveBeenCalledWith( + expect(catalog.getEntityByRef).toHaveBeenCalledTimes(1); + expect(catalog.getEntityByRef).toHaveBeenCalledWith( { namespace: 'default', kind: 'service', @@ -179,7 +180,7 @@ describe('createRouter', () => { }); it('returns badge spec for entity', async () => { - catalog.getEntityByName.mockResolvedValueOnce(entity); + catalog.getEntityByRef.mockResolvedValueOnce(entity); badgeBuilder.createBadgeJson.mockResolvedValueOnce(badge); const url = '/entity/default/service/test/badge/test-badge?format=json'; @@ -192,7 +193,7 @@ describe('createRouter', () => { describe('Errors', () => { it('returns 404 for unknown entities', async () => { - catalog.getEntityByName.mockResolvedValue(undefined); + catalog.getEntityByRef.mockResolvedValue(undefined); async function testUrl(url: string) { const response = await request(app).get(url); expect(response.status).toEqual(404); diff --git a/plugins/badges-backend/src/service/router.ts b/plugins/badges-backend/src/service/router.ts index 196ddf36da..b7211a0aad 100644 --- a/plugins/badges-backend/src/service/router.ts +++ b/plugins/badges-backend/src/service/router.ts @@ -46,7 +46,7 @@ export async function createRouter( router.get('/entity/:namespace/:kind/:name/badge-specs', async (req, res) => { const { namespace, kind, name } = req.params; - const entity = await catalog.getEntityByName( + const entity = await catalog.getEntityByRef( { namespace, kind, name }, { token: getBearerToken(req.headers.authorization), @@ -84,7 +84,7 @@ export async function createRouter( '/entity/:namespace/:kind/:name/badge/:badgeId', async (req, res) => { const { namespace, kind, name, badgeId } = req.params; - const entity = await catalog.getEntityByName( + const entity = await catalog.getEntityByRef( { namespace, kind, name }, { token: getBearerToken(req.headers.authorization), diff --git a/plugins/catalog-graph/dev/index.tsx b/plugins/catalog-graph/dev/index.tsx index 7413634497..7d772702a1 100644 --- a/plugins/catalog-graph/dev/index.tsx +++ b/plugins/catalog-graph/dev/index.tsx @@ -139,10 +139,12 @@ createDevApp() deps: {}, factory() { return { - async getEntityByName( - name: CompoundEntityRef, + async getEntityByRef( + ref: string | CompoundEntityRef, ): Promise { - return entities[stringifyEntityRef(name)]; + return entities[ + typeof ref === 'string' ? ref : stringifyEntityRef(ref) + ]; }, async getEntities(): Promise { return { items: Object.values(entities) }; diff --git a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx index 7cebdee692..858fca1a83 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx @@ -57,7 +57,8 @@ describe('', () => { }; catalog = { getEntities: jest.fn(), - getEntityByName: jest.fn(async _ => ({ ...entity, relations: [] })), + getEntityByRef: jest.fn(async _ => ({ ...entity, relations: [] })), + getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), getLocationByRef: jest.fn(), @@ -88,7 +89,7 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(1); + expect(catalog.getEntityByRef).toBeCalledTimes(1); }); test('renders with custom title', async () => { diff --git a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx index 2f686f8f62..0bf6469123 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx @@ -88,7 +88,10 @@ describe('', () => { }; catalog = { getEntities: jest.fn(), - getEntityByName: jest.fn(async n => (n.name === 'e' ? entityE : entityC)), + getEntityByRef: jest.fn(async (n: any) => + n === 'b:d/e' ? entityE : entityC, + ), + getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), getLocationByRef: jest.fn(), @@ -128,7 +131,7 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findByText('b:d/e')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(2); - expect(catalog.getEntityByName).toBeCalledTimes(2); + expect(catalog.getEntityByRef).toBeCalledTimes(2); }); test('should toggle filters', async () => { diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx index 36dc816444..58ff81a91f 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Entity, RELATION_HAS_PART, RELATION_OWNED_BY, RELATION_OWNER_OF, RELATION_PART_OF, - stringifyEntityRef, } from '@backstage/catalog-model'; import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog-react'; import { renderInTestApp, TestApiProvider } from '@backstage/test-utils'; @@ -155,7 +155,8 @@ describe('', () => { }; catalog = { getEntities: jest.fn(), - getEntityByName: jest.fn(async n => entities[stringifyEntityRef(n)]), + getEntityByRef: jest.fn(async n => entities[n as string]), + getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), getLocationByRef: jest.fn(), @@ -178,7 +179,7 @@ describe('', () => { }); test('renders a single node without exploding', async () => { - catalog.getEntityByName.mockResolvedValue({ + catalog.getEntityByRef.mockResolvedValue({ apiVersion: 'a', kind: 'b', metadata: { @@ -198,11 +199,11 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(1); + expect(catalog.getEntityByRef).toBeCalledTimes(1); }); test('renders a progress indicator while loading', async () => { - catalog.getEntityByName.mockImplementation(() => new Promise(() => {})); + catalog.getEntityByRef.mockImplementation(() => new Promise(() => {})); const { findByRole } = await renderInTestApp( @@ -213,12 +214,12 @@ describe('', () => { ); expect(await findByRole('progressbar')).toBeInTheDocument(); - expect(catalog.getEntityByName).toBeCalledTimes(1); + expect(catalog.getEntityByRef).toBeCalledTimes(1); }); test('does not explode if an entity is missing', async () => { - catalog.getEntityByName.mockImplementation(async n => { - if (n.name === 'c') { + catalog.getEntityByRef.mockImplementation(async (n: any) => { + if (n === 'b:d/c') { return { apiVersion: 'a', kind: 'b', @@ -253,7 +254,7 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(2); + expect(catalog.getEntityByRef).toBeCalledTimes(2); }); test('renders at max depth of one', async () => { @@ -276,7 +277,7 @@ describe('', () => { expect(await findAllByText('hasPart')).toHaveLength(1); expect(await findAllByTestId('label')).toHaveLength(2); - expect(catalog.getEntityByName).toBeCalledTimes(3); + expect(catalog.getEntityByRef).toBeCalledTimes(3); }); test('renders simplied graph at full depth', async () => { @@ -301,7 +302,7 @@ describe('', () => { expect(await findAllByText('hasPart')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(3); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders full graph at full depth', async () => { @@ -328,7 +329,7 @@ describe('', () => { expect(await findAllByText('partOf')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(8); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders full graph at full depth with merged relations', async () => { @@ -353,7 +354,7 @@ describe('', () => { expect(await findAllByText('hasPart')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(4); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders a graph with multiple root nodes', async () => { @@ -379,7 +380,7 @@ describe('', () => { expect(await findAllByText('partOf')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(3); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders a graph with filtered kinds and relations', async () => { @@ -401,7 +402,7 @@ describe('', () => { expect(await findAllByText('ownerOf')).toHaveLength(1); expect(await findAllByTestId('label')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(2); + expect(catalog.getEntityByRef).toBeCalledTimes(2); }); test('handle clicks on a node', async () => { diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts index a5dc82eab9..5349b64a72 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts @@ -29,6 +29,7 @@ describe('useEntityStore', () => { beforeEach(() => { catalogApi = { getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), @@ -64,7 +65,7 @@ describe('useEntityStore', () => { }, }; - catalogApi.getEntityByName.mockResolvedValue(entity); + catalogApi.getEntityByRef.mockResolvedValue(entity); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -84,7 +85,7 @@ describe('useEntityStore', () => { test('handles request failures', async () => { const err = new Error('Hello World'); - catalogApi.getEntityByName.mockRejectedValue(err); + catalogApi.getEntityByRef.mockRejectedValue(err); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -101,7 +102,7 @@ describe('useEntityStore', () => { }); test('handles loading', async () => { - catalogApi.getEntityByName.mockReturnValue(new Promise(() => {})); + catalogApi.getEntityByRef.mockReturnValue(new Promise(() => {})); const { result } = renderHook(() => useEntityStore()); @@ -133,7 +134,7 @@ describe('useEntityStore', () => { }, }; - catalogApi.getEntityByName.mockResolvedValue(entity1); + catalogApi.getEntityByRef.mockResolvedValue(entity1); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -150,7 +151,7 @@ describe('useEntityStore', () => { }); }); - catalogApi.getEntityByName.mockResolvedValue(entity2); + catalogApi.getEntityByRef.mockResolvedValue(entity2); act(() => { result.current.requestEntities([ @@ -188,7 +189,7 @@ describe('useEntityStore', () => { }, }; - catalogApi.getEntityByName.mockResolvedValue(entity1); + catalogApi.getEntityByRef.mockResolvedValue(entity1); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -205,7 +206,7 @@ describe('useEntityStore', () => { }); }); - catalogApi.getEntityByName.mockResolvedValue(entity2); + catalogApi.getEntityByRef.mockResolvedValue(entity2); act(() => { result.current.requestEntities(['kind:namespace/name2']); @@ -233,6 +234,6 @@ describe('useEntityStore', () => { }); }); - expect(catalogApi.getEntityByName).toBeCalledTimes(2); + expect(catalogApi.getEntityByRef).toBeCalledTimes(2); }); }); diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts index b5ec3bef19..843b308281 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, parseEntityRef } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { useApi } from '@backstage/core-plugin-api'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import limiterFactory from 'p-limit'; @@ -73,9 +73,7 @@ export function useEntityStore(): { return; } - const promise = catalogClient.getEntityByName( - parseEntityRef(entityRef), - ); + const promise = catalogClient.getEntityByRef(entityRef); outstandingEntities.set(entityRef, promise); diff --git a/plugins/catalog-import/src/api/CatalogImportClient.test.ts b/plugins/catalog-import/src/api/CatalogImportClient.test.ts index f689c0a883..969ff25c00 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.test.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.test.ts @@ -93,6 +93,7 @@ describe('CatalogImportClient', () => { getEntities: jest.fn(), addLocation: jest.fn(), removeLocationById: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), diff --git a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx index fc9b35136a..dd8087c51e 100644 --- a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx +++ b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx @@ -38,6 +38,7 @@ describe('', () => { const catalogApi: jest.Mocked = { getEntities: jest.fn(), addLocation: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), diff --git a/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx b/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx index 316a0f2c06..310deabbb7 100644 --- a/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx +++ b/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx @@ -55,7 +55,7 @@ const mockConfigApi = { } as Partial; const mockCatalogApi = { - getEntityByName: () => Promise.resolve(mockUser), + getEntityByRef: () => Promise.resolve(mockUser), } as Partial; const mockIdentityApi = { diff --git a/plugins/catalog-react/src/hooks/useEntity.tsx b/plugins/catalog-react/src/hooks/useEntity.tsx index da3fb8f3a5..f80f858dec 100644 --- a/plugins/catalog-react/src/hooks/useEntity.tsx +++ b/plugins/catalog-react/src/hooks/useEntity.tsx @@ -115,7 +115,7 @@ export const useEntityFromUrl = (): EntityLoadingStatus => { loading, retry: refresh, } = useAsyncRetry( - () => catalogApi.getEntityByName({ kind, namespace, name }), + () => catalogApi.getEntityByRef({ kind, namespace, name }), [catalogApi, kind, namespace, name], ); diff --git a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx index 27fde81cc7..e3ab09f081 100644 --- a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx +++ b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx @@ -76,7 +76,7 @@ const mockIdentityApi: Partial = { }; const mockCatalogApi: Partial = { getEntities: jest.fn().mockImplementation(async () => ({ items: entities })), - getEntityByName: async () => undefined, + getEntityByRef: async () => undefined, }; const wrapper = ({ diff --git a/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx b/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx index 1bb9a649ad..4730f71a7d 100644 --- a/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx +++ b/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx @@ -30,13 +30,13 @@ import { loadCatalogOwnerRefs, useEntityOwnership } from './useEntityOwnership'; describe('useEntityOwnership', () => { type MockIdentityApi = jest.Mocked>; - type MockCatalogApi = jest.Mocked>; + type MockCatalogApi = jest.Mocked>; const mockIdentityApi: MockIdentityApi = { getBackstageIdentity: jest.fn(), }; const mockCatalogApi: MockCatalogApi = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), }; const identityApi = mockIdentityApi as unknown as IdentityApi; @@ -102,11 +102,11 @@ describe('useEntityOwnership', () => { describe('loadCatalogOwnerRefs', () => { it('loads the first user from the catalog', async () => { - mockCatalogApi.getEntityByName.mockResolvedValueOnce(user2Entity); + mockCatalogApi.getEntityByRef.mockResolvedValueOnce(user2Entity); await expect( loadCatalogOwnerRefs(catalogApi, ['user:default/user2']), ).resolves.toEqual(['group:default/group1']); - expect(mockCatalogApi.getEntityByName).toBeCalledWith({ + expect(mockCatalogApi.getEntityByRef).toBeCalledWith({ kind: 'user', namespace: 'default', name: 'user2', @@ -114,11 +114,11 @@ describe('useEntityOwnership', () => { }); it('gracefully handles missing user', async () => { - mockCatalogApi.getEntityByName.mockResolvedValueOnce(undefined); + mockCatalogApi.getEntityByRef.mockResolvedValueOnce(undefined); await expect( loadCatalogOwnerRefs(catalogApi, ['user:default/user2']), ).resolves.toEqual([]); - expect(mockCatalogApi.getEntityByName).toBeCalledWith({ + expect(mockCatalogApi.getEntityByRef).toBeCalledWith({ kind: 'user', namespace: 'default', name: 'user2', @@ -133,7 +133,7 @@ describe('useEntityOwnership', () => { userEntityRef: 'user:default/user1', ownershipEntityRefs: ['user:default/user1', 'group:default/group1'], }); - mockCatalogApi.getEntityByName.mockResolvedValue(undefined); + mockCatalogApi.getEntityByRef.mockResolvedValue(undefined); const { result, waitForValueToChange } = renderHook( () => useEntityOwnership(), diff --git a/plugins/catalog-react/src/hooks/useEntityOwnership.ts b/plugins/catalog-react/src/hooks/useEntityOwnership.ts index de3eca9dab..55b4e75ace 100644 --- a/plugins/catalog-react/src/hooks/useEntityOwnership.ts +++ b/plugins/catalog-react/src/hooks/useEntityOwnership.ts @@ -48,7 +48,7 @@ export async function loadCatalogOwnerRefs( const primaryUserRef = identityOwnerRefs.find(ref => ref.startsWith('user:')); if (primaryUserRef) { - const entity = await catalogApi.getEntityByName( + const entity = await catalogApi.getEntityByRef( parseEntityRef(primaryUserRef), ); if (entity) { diff --git a/plugins/catalog-react/src/hooks/useOwnUser.ts b/plugins/catalog-react/src/hooks/useOwnUser.ts index 74c44bc638..211321afdf 100644 --- a/plugins/catalog-react/src/hooks/useOwnUser.ts +++ b/plugins/catalog-react/src/hooks/useOwnUser.ts @@ -34,7 +34,12 @@ export function useOwnUser(): AsyncState { return useAsync(async () => { const identity = await identityApi.getBackstageIdentity(); - return catalogApi.getEntityByName( + // TODO(freben): Defensively parse with defaults even though getEntityByRef + // supports the string form, since some auth resolvers have been known to + // return incomplete refs (just the name part) historically. This can be + // simplified in the future to just pass the ref immediately to + // getEntityByRef. + return catalogApi.getEntityByRef( parseEntityRef(identity.userEntityRef, { defaultKind: 'User', defaultNamespace: DEFAULT_NAMESPACE, diff --git a/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts b/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts index 605305882e..f4d06b0bb2 100644 --- a/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts +++ b/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts @@ -40,7 +40,7 @@ export const useEntityFromUrl = (): EntityLoadingStatus => { loading, retry: refresh, } = useAsyncRetry( - () => catalogApi.getEntityByName({ kind, namespace, name }), + () => catalogApi.getEntityByRef({ kind, namespace, name }), [catalogApi, kind, namespace, name], ); diff --git a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx index cf2eedb14c..07bd81b287 100644 --- a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx +++ b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx @@ -17,6 +17,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Entity, + parseEntityRef, RELATION_MEMBER_OF, RELATION_OWNED_BY, } from '@backstage/catalog-model'; @@ -104,11 +105,11 @@ describe('DefaultCatalogPage', () => { }), getLocationByRef: () => Promise.resolve({ id: 'id', type: 'url', target: 'url' }), - getEntityByName: async entityName => { + getEntityByRef: async entityRef => { return { apiVersion: 'backstage.io/v1alpha1', kind: 'User', - metadata: { name: entityName.name }, + metadata: { name: parseEntityRef(entityRef).name }, relations: [ { type: RELATION_MEMBER_OF, diff --git a/plugins/code-coverage-backend/src/service/router.ts b/plugins/code-coverage-backend/src/service/router.ts index 4922f5c887..488103c835 100644 --- a/plugins/code-coverage-backend/src/service/router.ts +++ b/plugins/code-coverage-backend/src/service/router.ts @@ -18,7 +18,7 @@ import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; import xmlparser from 'express-xml-bodyparser'; -import { CatalogClient } from '@backstage/catalog-client'; +import { CatalogApi, CatalogClient } from '@backstage/catalog-client'; import { errorHandler, PluginDatabaseManager, @@ -33,10 +33,7 @@ import { aggregateCoverage, CoverageUtils } from './CoverageUtils'; import { Cobertura } from './converter/cobertura'; import { Jacoco } from './converter/jacoco'; import { Converter } from './converter'; -import { - getEntitySourceLocation, - parseEntityRef, -} from '@backstage/catalog-model'; +import { getEntitySourceLocation } from '@backstage/catalog-model'; export interface RouterOptions { config: Config; @@ -59,7 +56,7 @@ export const makeRouter = async ( await database.getClient(), ); const codecovUrl = await discovery.getExternalBaseUrl('code-coverage'); - const catalogApi = new CatalogClient({ discoveryApi: discovery }); + const catalogApi: CatalogApi = new CatalogClient({ discoveryApi: discovery }); const scm = ScmIntegrations.fromConfig(config); const router = Router(); @@ -77,8 +74,7 @@ export const makeRouter = async ( */ router.get('/report', async (req, res) => { const { entity } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } @@ -100,8 +96,7 @@ export const makeRouter = async ( */ router.get('/history', async (req, res) => { const { entity } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } @@ -119,8 +114,7 @@ export const makeRouter = async ( */ router.get('/file-content', async (req, res) => { const { entity, path } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } @@ -171,8 +165,7 @@ export const makeRouter = async ( */ router.post('/report', async (req, res) => { const { entity, coverageType } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } diff --git a/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx b/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx index 53b225daaf..c490dd00da 100644 --- a/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx +++ b/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx @@ -28,6 +28,7 @@ describe('', () => { getLocationById: jest.fn(), removeLocationById: jest.fn(), removeEntityByUid: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), diff --git a/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx b/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx index 1310d9f31e..d103541333 100644 --- a/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx +++ b/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx @@ -29,6 +29,7 @@ describe('', () => { getLocationById: jest.fn(), removeLocationById: jest.fn(), removeEntityByUid: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), diff --git a/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx b/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx index 255db061aa..99500336cf 100644 --- a/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx +++ b/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx @@ -29,6 +29,7 @@ describe('', () => { getLocationById: jest.fn(), removeLocationById: jest.fn(), removeEntityByUid: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), diff --git a/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx b/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx index 2759afd585..f178b7de7f 100644 --- a/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx +++ b/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx @@ -29,6 +29,7 @@ describe('', () => { const catalogApi: jest.Mocked = { addLocation: jest.fn(), getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), diff --git a/plugins/jenkins-backend/README.md b/plugins/jenkins-backend/README.md index 1413a45136..443cb33c05 100644 --- a/plugins/jenkins-backend/README.md +++ b/plugins/jenkins-backend/README.md @@ -166,7 +166,7 @@ class AcmeJenkinsInfoProvider implements JenkinsInfoProvider { const PAAS_ANNOTATION = 'acme.example.com/paas-project-name'; // lookup pass-project-name from entity annotation - const entity = await this.catalog.getEntityByName(opt.entityRef); + const entity = await this.catalog.getEntityByRef(opt.entityRef); if (!entity) { throw new Error( `Couldn't find entity with name: ${stringifyEntityRef(opt.entityRef)}`, diff --git a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts index 33079dfc8f..a3a5ba069d 100644 --- a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts +++ b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts @@ -160,7 +160,7 @@ describe('JenkinsConfig', () => { describe('DefaultJenkinsInfoProvider', () => { const mockCatalog: jest.Mocked = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), } as any as jest.Mocked; const entityRef: CompoundEntityRef = { @@ -171,7 +171,7 @@ describe('DefaultJenkinsInfoProvider', () => { function configureProvider(configData: any, entityData: any) { const config = new ConfigReader(configData); - mockCatalog.getEntityByName.mockReturnValueOnce( + mockCatalog.getEntityByRef.mockReturnValueOnce( Promise.resolve(entityData as Entity), ); @@ -185,7 +185,7 @@ describe('DefaultJenkinsInfoProvider', () => { const provider = configureProvider({ jenkins: {} }, undefined); await expect(provider.getInstance({ entityRef })).rejects.toThrowError(); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); }); it('Reads simple config and annotation', async () => { @@ -207,7 +207,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toStrictEqual({ baseUrl: 'https://jenkins.example.com', crumbIssuer: undefined, @@ -243,7 +243,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -280,7 +280,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -317,7 +317,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins-other.example.com', jobFullName: 'teamA/artistLookup-build', @@ -343,7 +343,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -369,7 +369,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -400,7 +400,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins-other.example.com', jobFullName: 'teamA/artistLookup-build', diff --git a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts index 528a1d038f..a88dbf965e 100644 --- a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts +++ b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts @@ -186,7 +186,7 @@ export class DefaultJenkinsInfoProvider implements JenkinsInfoProvider { jobFullName?: string; }): Promise { // load entity - const entity = await this.catalog.getEntityByName(opt.entityRef); + const entity = await this.catalog.getEntityByRef(opt.entityRef); if (!entity) { throw new Error( `Couldn't find entity with name: ${stringifyEntityRef(opt.entityRef)}`, diff --git a/plugins/rollbar/src/hooks/useCatalogEntity.ts b/plugins/rollbar/src/hooks/useCatalogEntity.ts index 468d0e8c50..21d22dd13c 100644 --- a/plugins/rollbar/src/hooks/useCatalogEntity.ts +++ b/plugins/rollbar/src/hooks/useCatalogEntity.ts @@ -30,7 +30,7 @@ export function useCatalogEntity() { error, loading, } = useAsync( - () => catalogApi.getEntityByName({ kind: 'Component', namespace, name }), + () => catalogApi.getEntityByRef({ kind: 'Component', namespace, name }), [catalogApi, namespace, name], ); diff --git a/plugins/scaffolder-backend/src/service/helpers.ts b/plugins/scaffolder-backend/src/service/helpers.ts index e3aae0951b..a7148dac40 100644 --- a/plugins/scaffolder-backend/src/service/helpers.ts +++ b/plugins/scaffolder-backend/src/service/helpers.ts @@ -22,6 +22,7 @@ import { ANNOTATION_SOURCE_LOCATION, CompoundEntityRef, DEFAULT_NAMESPACE, + stringifyEntityRef, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, InputError, NotFoundError } from '@backstage/errors'; @@ -106,9 +107,11 @@ export async function findTemplate(options: { throw new InputError(`Invalid kind, only 'Template' kind is supported`); } - const template = await catalogApi.getEntityByName(entityRef, { token }); + const template = await catalogApi.getEntityByRef(entityRef, { token }); if (!template) { - throw new NotFoundError(`Template ${entityRef} not found`); + throw new NotFoundError( + `Template ${stringifyEntityRef(entityRef)} not found`, + ); } return template as TemplateEntityV1beta3 | TemplateEntityV1beta2; diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index fa1b649231..4041e41cf8 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -53,7 +53,7 @@ import { stringifyEntityRef } from '@backstage/catalog-model'; const createCatalogClient = (template: any) => ({ - getEntityByName: async () => template, + getEntityByRef: async () => template, } as unknown as CatalogApi); function createDatabase(): PluginDatabaseManager { diff --git a/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts b/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts index a1542df351..19684486dc 100644 --- a/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts +++ b/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts @@ -20,7 +20,7 @@ import { CompoundEntityRef } from '@backstage/catalog-model'; describe('CachedEntityLoader', () => { const catalog: jest.Mocked = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), } as any; const cache: jest.Mocked = { @@ -53,7 +53,7 @@ describe('CachedEntityLoader', () => { it('writes entities to cache', async () => { cache.get.mockResolvedValue(undefined); - catalog.getEntityByName.mockResolvedValue(entity); + catalog.getEntityByRef.mockResolvedValue(entity); const result = await loader.load(entityName, token); @@ -71,12 +71,12 @@ describe('CachedEntityLoader', () => { const result = await loader.load(entityName, token); expect(result).toEqual(entity); - expect(catalog.getEntityByName).not.toBeCalled(); + expect(catalog.getEntityByRef).not.toBeCalled(); }); it('does not cache missing entites', async () => { cache.get.mockResolvedValue(undefined); - catalog.getEntityByName.mockResolvedValue(undefined); + catalog.getEntityByRef.mockResolvedValue(undefined); const result = await loader.load(entityName, token); @@ -86,7 +86,7 @@ describe('CachedEntityLoader', () => { it('uses entity ref as cache key for anonymous users', async () => { cache.get.mockResolvedValue(undefined); - catalog.getEntityByName.mockResolvedValue(entity); + catalog.getEntityByRef.mockResolvedValue(entity); const result = await loader.load(entityName, undefined); @@ -103,7 +103,7 @@ describe('CachedEntityLoader', () => { setTimeout(() => resolve(undefined), 10000); }), ); - catalog.getEntityByName.mockResolvedValue(entity); + catalog.getEntityByRef.mockResolvedValue(entity); const result = await loader.load(entityName, token); diff --git a/plugins/techdocs-backend/src/service/CachedEntityLoader.ts b/plugins/techdocs-backend/src/service/CachedEntityLoader.ts index cd771f7fff..424d9b9541 100644 --- a/plugins/techdocs-backend/src/service/CachedEntityLoader.ts +++ b/plugins/techdocs-backend/src/service/CachedEntityLoader.ts @@ -37,17 +37,17 @@ export class CachedEntityLoader { } async load( - entityName: CompoundEntityRef, + entityRef: CompoundEntityRef, token: string | undefined, ): Promise { - const cacheKey = this.getCacheKey(entityName, token); + const cacheKey = this.getCacheKey(entityRef, token); let result = await this.getFromCache(cacheKey); if (result) { return result; } - result = await this.catalog.getEntityByName(entityName, { token }); + result = await this.catalog.getEntityByRef(entityRef, { token }); if (result) { this.cache.set(cacheKey, result, { ttl: 5000 }); diff --git a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx index 33f5e0a08a..14fdb88395 100644 --- a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx +++ b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx @@ -45,7 +45,7 @@ jest.mock('@backstage/plugin-catalog-react', () => { }); const mockCatalogApi = { - getEntityByName: () => Promise.resolve(), + getEntityByRef: () => Promise.resolve(), getEntities: async () => ({ items: [ { diff --git a/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx b/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx index a8fd472f7c..16d8d48bc3 100644 --- a/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx +++ b/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx @@ -33,7 +33,7 @@ jest.mock('@backstage/plugin-catalog-react', () => { }); const mockCatalogApi = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), getEntities: async () => ({ items: [ { diff --git a/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx b/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx index e9c3a55b15..2088dbf4fc 100644 --- a/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx +++ b/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx @@ -31,7 +31,7 @@ jest.mock('@backstage/plugin-catalog-react', () => { }); const mockCatalogApi = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), getEntities: async () => ({ items: [ { diff --git a/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx b/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx index c8149f1ff2..d2f0266af6 100644 --- a/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx +++ b/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx @@ -232,7 +232,12 @@ function useOwnUser(): AsyncState { return useAsync(async () => { const identity = await identityApi.getBackstageIdentity(); - return catalogApi.getEntityByName( + // TODO(freben): Defensively parse with defaults even though getEntityByRef + // supports the string form, since some auth resolvers have been known to + // return incomplete refs (just the name part) historically. This can be + // simplified in the future to just pass the ref immediately to + // getEntityByRef. + return catalogApi.getEntityByRef( parseEntityRef(identity.userEntityRef, { defaultKind: 'User', defaultNamespace: DEFAULT_NAMESPACE, diff --git a/plugins/todo-backend/src/service/TodoReaderService.test.ts b/plugins/todo-backend/src/service/TodoReaderService.test.ts index 985687c6bb..943462bd63 100644 --- a/plugins/todo-backend/src/service/TodoReaderService.test.ts +++ b/plugins/todo-backend/src/service/TodoReaderService.test.ts @@ -44,6 +44,7 @@ function mockCatalogClient(entity?: Entity): jest.Mocked { const mock = { addLocation: jest.fn(), getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), @@ -54,7 +55,7 @@ function mockCatalogClient(entity?: Entity): jest.Mocked { getEntityFacets: jest.fn(), }; if (entity) { - mock.getEntityByName.mockReturnValue(entity); + mock.getEntityByRef.mockReturnValue(entity); } return mock; } @@ -93,7 +94,7 @@ describe('TodoReaderService', () => { offset: 0, limit: 10, }); - expect(catalogClient.getEntityByName).toHaveBeenCalledWith(entityName, { + expect(catalogClient.getEntityByRef).toHaveBeenCalledWith(entityName, { token: undefined, }); }); @@ -304,7 +305,7 @@ describe('TodoReaderService', () => { message: 'Entity not found, component:default/my-component', }), ); - expect(catalogClient.getEntityByName).toHaveBeenCalledWith(entityName, { + expect(catalogClient.getEntityByRef).toHaveBeenCalledWith(entityName, { token: undefined, }); }); diff --git a/plugins/todo-backend/src/service/TodoReaderService.ts b/plugins/todo-backend/src/service/TodoReaderService.ts index 86f53057da..4397f34045 100644 --- a/plugins/todo-backend/src/service/TodoReaderService.ts +++ b/plugins/todo-backend/src/service/TodoReaderService.ts @@ -66,7 +66,7 @@ export class TodoReaderService implements TodoService { throw new InputError('Entity filter is required to list TODOs'); } const token = options?.token; - const entity = await this.catalogClient.getEntityByName(req.entity, { + const entity = await this.catalogClient.getEntityByRef(req.entity, { token, }); if (!entity) { From 2255bfd1c40e155184646d62ebc56673ba8269f4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 3 Mar 2022 13:37:33 +0100 Subject: [PATCH 147/150] Apply suggestions from code review Signed-off-by: Patrik Oldsberg --- .changeset/small-hornets-dress.md | 6 +++++- plugins/catalog-common/src/search/CatalogEntityDocument.ts | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.changeset/small-hornets-dress.md b/.changeset/small-hornets-dress.md index 6d6817977d..2bb73b1ba3 100644 --- a/.changeset/small-hornets-dress.md +++ b/.changeset/small-hornets-dress.md @@ -3,4 +3,8 @@ '@backstage/plugin-catalog-common': patch --- -Moved the `CatalogEntityDocument` to `@backstage/plugin-catalog-common` and deprecated the export from `@backstage/plugin-catalog-backend` +Moved the `CatalogEntityDocument` to `@backstage/plugin-catalog-common` and deprecated the export from `@backstage/plugin-catalog-backend`. + +A new `type` field has also been added to `CatalogEntityDocument` as a replacement for `componentType`, which is now deprecated. Both fields are still present and should be set to the same value in order to avoid issues with indexing. + +Any search customizations need to be updated to use this new `type` field instead, including any custom frontend filters, custom frontend result components, custom search decorators, or non-default Catalog collator implementations. diff --git a/plugins/catalog-common/src/search/CatalogEntityDocument.ts b/plugins/catalog-common/src/search/CatalogEntityDocument.ts index 005bc79c92..834dde2568 100644 --- a/plugins/catalog-common/src/search/CatalogEntityDocument.ts +++ b/plugins/catalog-common/src/search/CatalogEntityDocument.ts @@ -21,7 +21,7 @@ import { IndexableDocument } from '@backstage/search-common'; * @public */ export interface CatalogEntityDocument extends IndexableDocument { - /** @deprecated use `type` as well, as `componentType` will be removed after a few releases but we dont want to break indexing */ + /** @deprecated `componentType` is being renamed to `type`. During the transition both of these fields should be set to the same value, in order to avoid issues with indexing. */ componentType: string; type: string; namespace: string; From 0874762ed0256ca2e7f6f003851005cc70622980 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 3 Mar 2022 13:43:09 +0100 Subject: [PATCH 148/150] add hack week notice Signed-off-by: Johan Haals --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1078631926..e78800ef84 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ # [Backstage](https://backstage.io) +_During March 7 to March 11 the maintainers will be taking part in Spotify's annual hack week. Development will continue as usual, but expect a slower pace for discussions and PR reviews. Why not take this opportunity to [build a plugin](https://backstage.io/docs/plugins/)?_ + [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![CNCF Status](https://img.shields.io/badge/cncf%20status-sandbox-blue.svg)](https://www.cncf.io/projects) [![Main CI Build](https://github.com/backstage/backstage/workflows/Main%20Master%20Build/badge.svg)](https://github.com/backstage/backstage/actions?query=workflow%3A%22Main+Master+Build%22) From 0513a720328553fb37b57d96bd489c62b3866dcc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 3 Mar 2022 13:50:02 +0100 Subject: [PATCH 149/150] Revert "scripts/prepare-release: update to detect patch versions from patch branches" Signed-off-by: Patrik Oldsberg --- docs/publishing.md | 9 ++ scripts/prepare-release.js | 174 ++++++++----------------------------- 2 files changed, 46 insertions(+), 137 deletions(-) diff --git a/docs/publishing.md b/docs/publishing.md index a0d67b7d24..fbbef93493 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -67,3 +67,12 @@ process is used to release an emergency fix as version `6.5.1` in the patch rele - [ ] The fix, which you can likely cherry-pick from your patch branch: `git cherry-pick origin/patch/v1.18.0^` - [ ] An updated `CHANGELOG.md` of all patched packages from the tip of the patch branch, `git checkout origin/patch/v1.18.0 -- {packages,plugins}/*/CHANGELOG.md`. - [ ] A changeset with the message "Applied the fix from version `6.5.1` of this package, which is part of the `v1.18.1` release of Backstage." + - [ ] An entry in `.changeset/patched.json` that sets the current release version to `6.5.1`: + + ```json + { + "currentReleaseVersion": { + "@backstage/plugin-foo": "6.5.1" + } + } + ``` diff --git a/scripts/prepare-release.js b/scripts/prepare-release.js index 5cc888fd66..be82e19344 100755 --- a/scripts/prepare-release.js +++ b/scripts/prepare-release.js @@ -28,10 +28,6 @@ const execFile = promisify(execFileCb); // All of these are considered to be main-line release branches const MAIN_BRANCHES = ['master', 'origin/master', 'changeset-release/master']; -// This prefix is used for patch branches, followed by the release version WITH a 'v' prefix -// For example, `patch/v1.2.0` -const PATCH_BRANCH_PREFIX = 'patch/'; - const DEPENDENCY_TYPES = [ 'dependencies', 'devDependencies', @@ -39,84 +35,25 @@ const DEPENDENCY_TYPES = [ 'peerDependencies', ]; -/** - * Returns the most recent release version on the main branch that is not a pre-release. - */ -async function getPreviousReleaseVersion(repo) { - // TODO(Rugvip): Figure out which field to sort by to avoid manual sort after - const { stdout: tagsStr } = await execFile( - 'git', - ['tag', '--list', 'v*', '--merged=HEAD'], - { shell: true, cwd: repo.root.dir }, - ); - const tags = tagsStr.trim().split(/\r\n|\n/); - const [latestTag] = semver.rsort(tags).filter(t => !semver.prerelease(t)); - return latestTag; -} - -/** - * Finds the tip of the patch branch of a given release version. - * Returns undefined if no patch branch exists. - */ -async function findTipOfPatchBranch(repo, release) { - try { - await execFile('git', ['fetch', 'origin', PATCH_BRANCH_PREFIX + release], { - shell: true, - cwd: repo.root.dir, - }); - } catch (error) { - if (error.stderr?.match(/fatal: couldn't find remote ref/i)) { - return undefined; - } - throw error; - } - const { stdout: refStr } = await execFile('git', ['rev-parse', 'FETCH_HEAD']); - return refStr.trim(); -} - -/** - * Returns a map of packages to their versions for any package version - * in that does not match the current version in the working directory. - */ -async function detectPatchVersionsForRef(repo, ref) { - const patchVersions = new Map(); - - for (const pkg of repo.packages) { - const pkgJsonPath = path.join( - path.relative(repo.root.dir, pkg.dir), - 'package.json', - ); - const { stdout: pkgJsonStr } = await execFile('git', [ - 'show', - `${ref}:${pkgJsonPath}`, - ]); - if (pkgJsonStr) { - const releasePkgJson = JSON.parse(pkgJsonStr); - const pkgJson = pkg.packageJson; - if (releasePkgJson.name !== pkgJson.name) { - throw new Error( - `Mismatched package name at ${pkg.dir}, ${releasePkgJson.name} !== ${pkgJson.name}`, - ); - } - if (releasePkgJson.version !== pkgJson.version) { - patchVersions.set(pkgJson.name, releasePkgJson.version); - } - } - } - - return patchVersions; -} - /** * Bumps up the versions of packages to account for * the base versions that are set in .changeset/patched.json. * This may be needed when we have made emergency releases. */ -async function applyPatchVersions(repo, patchVersions) { +async function updatePatchVersions() { + const patchedJsonPath = path.resolve('.changeset', 'patched.json'); + const { currentReleaseVersion } = await fs.readJson(patchedJsonPath); + if (Object.keys(currentReleaseVersion).length === 0) { + console.log('No currentReleaseVersion overrides found, skipping.'); + return; + } + + const { packages } = await getPackages(path.resolve('.')); + const pendingVersionBumps = new Map(); - for (const [name, version] of patchVersions) { - const pkg = repo.packages.find(p => p.packageJson.name === name); + for (const [name, version] of Object.entries(currentReleaseVersion)) { + const pkg = packages.find(p => p.packageJson.name === name); if (!pkg) { throw new Error(`Package ${name} not found`); } @@ -144,7 +81,7 @@ async function applyPatchVersions(repo, patchVersions) { }); } - for (const { dir, packageJson } of [repo.root, ...repo.packages]) { + for (const { dir, packageJson } of packages) { let hasChanges = false; if (pendingVersionBumps.has(packageJson.name)) { @@ -180,44 +117,20 @@ async function applyPatchVersions(repo, patchVersions) { }); } } -} -/** - * Detects any patched packages version since the most recent release on - * the main branch, and then bumps all packages in the repo accordingly. - */ -async function updatePackageVersions(repo) { - const previousRelease = await getPreviousReleaseVersion(repo); - console.log(`Found release version: ${previousRelease}`); - - const patchRef = await findTipOfPatchBranch(repo, previousRelease); - if (patchRef) { - console.log(`Tip of the patch branch: ${patchRef}`); - - const patchVersions = await detectPatchVersionsForRef(repo, patchRef); - if (patchVersions.size > 0) { - console.log( - `Found ${patchVersions.size} packages that were patched since the last release`, - ); - for (const [name, version] of patchVersions) { - console.log(` ${name}: ${version}`); - } - - await applyPatchVersions(repo, patchVersions); - } else { - console.log('No packages were patched since the last release'); - } - } else { - console.log('No patch branch found'); - } + await fs.writeJSON( + patchedJsonPath, + { currentReleaseVersion: {} }, + { spaces: 2, encoding: 'utf8' }, + ); } /** * Returns the mode and tag that is currently set * in the .changeset/pre.json file */ -async function getPreInfo(repo) { - const pre = path.join(repo.root.dir, '.changeset', 'pre.json'); +async function getPreInfo(rootPath) { + const pre = path.join(rootPath, '.changeset', 'pre.json'); if (!(await fs.pathExists(pre))) { return { mode: undefined, tag: undefined }; } @@ -226,30 +139,26 @@ async function getPreInfo(repo) { return { mode, tag }; } -/** - * Returns the name of the current git branch - */ -async function getCurrentBranch(repo) { - const { stdout } = await execFile( - 'git', - ['rev-parse', '--abbrev-ref', 'HEAD'], - { cwd: repo.root.dir, shell: true }, - ); - return stdout.trim(); -} - /** * Bumps the release version in the root package.json. * * This takes into account whether we're in pre-release mode or on a patch branch. */ -async function updateBackstageReleaseVersion(repo, type) { - const { mode: preMode, tag: preTag } = await getPreInfo(repo); +async function updateBackstageReleaseVersion() { + const rootPath = path.resolve(__dirname, '..'); + const branchName = await execFile( + 'git', + ['rev-parse', '--abbrev-ref', 'HEAD'], + { shell: true }, + ).then(({ stdout }) => stdout.trim()); + const { mode: preMode, tag: preTag } = await getPreInfo(rootPath); - const { version: currentVersion } = repo.root.packageJson; + const packagePath = path.join(rootPath, 'package.json'); + const package = await fs.readJson(packagePath); + const { version: currentVersion } = package; let nextVersion; - if (type === 'minor') { + if (MAIN_BRANCHES.includes(branchName)) { if (preMode === 'pre') { if (semver.prerelease(currentVersion)) { nextVersion = semver.inc(currentVersion, 'pre', preTag); @@ -261,7 +170,7 @@ async function updateBackstageReleaseVersion(repo, type) { } else { nextVersion = semver.inc(currentVersion, 'minor'); } - } else if (type === 'patch') { + } else { if (preMode) { throw new Error(`Unexpected pre mode ${preMode} on branch ${branchName}`); } @@ -269,9 +178,9 @@ async function updateBackstageReleaseVersion(repo, type) { } await fs.writeJson( - path.join(repo.root.dir, 'package.json'), + packagePath, { - ...repo.root.packageJson, + ...package, version: nextVersion, }, { spaces: 2, encoding: 'utf8' }, @@ -279,17 +188,8 @@ async function updateBackstageReleaseVersion(repo, type) { } async function main() { - const repo = await getPackages(__dirname); - const branchName = await getCurrentBranch(repo); - const isMainBranch = MAIN_BRANCHES.includes(branchName); - - console.log(`Current branch: ${branchName}`); - if (isMainBranch) { - console.log('Main release, updating package versions'); - await updatePackageVersions(repo); - } - - await updateBackstageReleaseVersion(repo, isMainBranch ? 'minor' : 'patch'); + await updatePatchVersions(); + await updateBackstageReleaseVersion(); } main().catch(error => { From 48997f9bcffb2c5c07172eb6b594997a446dec25 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 14:00:23 +0100 Subject: [PATCH 150/150] fix up the exports to keep the DEPRECATION Signed-off-by: blam --- .changeset/small-hornets-dress.md | 2 +- plugins/catalog-backend/api-report.md | 7 ++++--- .../src/search/DefaultCatalogCollatorFactory.ts | 5 ----- plugins/catalog-backend/src/search/index.ts | 9 +++++++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.changeset/small-hornets-dress.md b/.changeset/small-hornets-dress.md index 2bb73b1ba3..474c3011bd 100644 --- a/.changeset/small-hornets-dress.md +++ b/.changeset/small-hornets-dress.md @@ -3,7 +3,7 @@ '@backstage/plugin-catalog-common': patch --- -Moved the `CatalogEntityDocument` to `@backstage/plugin-catalog-common` and deprecated the export from `@backstage/plugin-catalog-backend`. +**DEPRECATION**: Moved the `CatalogEntityDocument` to `@backstage/plugin-catalog-common` and deprecated the export from `@backstage/plugin-catalog-backend`. A new `type` field has also been added to `CatalogEntityDocument` as a replacement for `componentType`, which is now deprecated. Both fields are still present and should be set to the same value in order to avoid issues with indexing. diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 9dcb2db752..dfe9a8943e 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -7,7 +7,7 @@ import { BitbucketIntegration } from '@backstage/integration'; import { CatalogApi } from '@backstage/catalog-client'; -import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; +import { CatalogEntityDocument as CatalogEntityDocument_2 } from '@backstage/plugin-catalog-common'; import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConditionalPolicyDecision } from '@backstage/plugin-permission-node'; import { Conditions } from '@backstage/plugin-permission-node'; @@ -248,7 +248,8 @@ export const catalogConditions: Conditions<{ >; }>; -export { CatalogEntityDocument }; +// @public @deprecated (undocumented) +export type CatalogEntityDocument = CatalogEntityDocument_2; // @public (undocumented) export type CatalogEnvironment = { @@ -423,7 +424,7 @@ export class DefaultCatalogCollator { // (undocumented) protected discovery: PluginEndpointDiscovery; // (undocumented) - execute(): Promise; + execute(): Promise; // (undocumented) protected filter?: GetEntitiesRequest['filter']; // (undocumented) diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts index ca6395b60a..440a8c8ebb 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -36,11 +36,6 @@ import { } from '@backstage/plugin-catalog-common'; import { Readable } from 'stream'; -/** - * @deprecated import from `@backstage/plugin-catalog-common` instead - */ -export type { CatalogEntityDocument }; - /** @public */ export type DefaultCatalogCollatorFactoryOptions = { discovery: PluginEndpointDiscovery; diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index bd54e33e59..eb5b1e7c22 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -17,8 +17,13 @@ export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; -/** @public @deprecated use the export from `plugin-catalog-common` instead */ -export type { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; +import { CatalogEntityDocument as CatalogEntityDocumentType } from '@backstage/plugin-catalog-common'; + +/** + * @deprecated import from `@backstage/plugin-catalog-common` instead + * @public + */ +export type CatalogEntityDocument = CatalogEntityDocumentType; /** * todo(backstage/techdocs-core): stop exporting this in a future release.