From 8ae4208c93f5f41ac8b3004d25dc7c551c9f9b15 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 12 Oct 2021 17:32:29 +0200 Subject: [PATCH 01/60] workflows/ci: run tests in a band as they are already parallelized by lerna Signed-off-by: Patrik Oldsberg --- .github/workflows/ci.yml | 4 ++-- .github/workflows/master-win.yml | 2 +- .github/workflows/master.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b68a96efbe..7222edef08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,7 @@ jobs: - name: test changed packages if: ${{ steps.yarn-lock.outcome == 'success' }} - run: yarn lerna -- run test --since origin/master -- --coverage + run: yarn lerna -- run test --since origin/master -- --coverage --runInBand env: BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }} BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }} @@ -149,7 +149,7 @@ jobs: - name: test all packages (and upload coverage) if: ${{ steps.yarn-lock.outcome == 'failure' }} run: | - yarn lerna -- run test -- --coverage + yarn lerna -- run test -- --coverage --runInBand bash <(curl -s https://codecov.io/bash) -N $(git rev-parse FETCH_HEAD) env: BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }} diff --git a/.github/workflows/master-win.yml b/.github/workflows/master-win.yml index 873f12086a..c6a9222483 100644 --- a/.github/workflows/master-win.yml +++ b/.github/workflows/master-win.yml @@ -58,7 +58,7 @@ jobs: run: lerna run --scope @backstage/core-* build - name: test - run: yarn lerna -- run test + run: yarn lerna -- run test --runInBand env: BACKSTAGE_TEST_DISABLE_DOCKER: 1 diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 7bd73e2c61..8106dd1198 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -116,7 +116,7 @@ jobs: - name: test (and upload coverage) run: | - yarn lerna -- run test -- --coverage + yarn lerna -- run test -- --coverage --runInBand bash <(curl -s https://codecov.io/bash) # Upload code coverage for some specific flags. Also see .codecov.yml bash <(curl -s https://codecov.io/bash) -f packages/core-app-api/coverage/* -F core-app-api From 42c618abf607abab92bcac80a657578524a34fc4 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Wed, 13 Oct 2021 16:00:51 +0200 Subject: [PATCH 02/60] Use `resolveSafeChildPath` in the `fetchContents` function Signed-off-by: Dominik Henneke --- .changeset/lucky-countries-smile.md | 5 +++++ .../actions/builtin/fetch/helpers.test.ts | 14 +++++++++++++- .../scaffolder/actions/builtin/fetch/helpers.ts | 15 +++++---------- 3 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 .changeset/lucky-countries-smile.md diff --git a/.changeset/lucky-countries-smile.md b/.changeset/lucky-countries-smile.md new file mode 100644 index 0000000000..c9066f109f --- /dev/null +++ b/.changeset/lucky-countries-smile.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Use `resolveSafeChildPath` in the `fetchContents` function to forbid reading files outside the base directory when a template is registered from a `file:` location. diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts index 7c7719cf68..a738afbe3f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts @@ -67,7 +67,19 @@ describe('fetchContent helper', () => { fetchUrl: '/etc/passwd', }), ).rejects.toThrow( - 'Fetch URL may not be absolute for file locations, /etc/passwd', + 'Relative path is not allowed to refer to a directory outside its parent', + ); + }); + + it('should reject relative file locations that exit the baseUrl', async () => { + await expect( + fetchContents({ + ...options, + baseUrl: 'file:///some/path', + fetchUrl: '../test', + }), + ).rejects.toThrow( + 'Relative path is not allowed to refer to a directory outside its parent', ); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts index 390ab7ec39..5625989417 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts @@ -14,12 +14,12 @@ * limitations under the License. */ -import fs from 'fs-extra'; -import { resolve as resolvePath, isAbsolute } from 'path'; -import { UrlReader } from '@backstage/backend-common'; +import { resolveSafeChildPath, UrlReader } from '@backstage/backend-common'; +import { JsonValue } from '@backstage/config'; import { InputError } from '@backstage/errors'; import { ScmIntegrations } from '@backstage/integration'; -import { JsonValue } from '@backstage/config'; +import fs from 'fs-extra'; +import * as path from 'path'; export async function fetchContents({ reader, @@ -52,12 +52,7 @@ export async function fetchContents({ // We handle both file locations and url ones if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) { const basePath = baseUrl.slice('file://'.length); - if (isAbsolute(fetchUrl)) { - throw new InputError( - `Fetch URL may not be absolute for file locations, ${fetchUrl}`, - ); - } - const srcDir = resolvePath(basePath, '..', fetchUrl); + const srcDir = resolveSafeChildPath(path.dirname(basePath), fetchUrl); await fs.copy(srcDir, outputPath); } else { let readUrl; From 76fef740fe6f67bf03b7bf26239528c24c53bb0c Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 13 Oct 2021 16:15:27 +0200 Subject: [PATCH 03/60] [TechDocs] Experimental useTechDocsReaderDom hook (#7476) * WIP! Co-authored-by: Camila Belo Signed-off-by: Eric Peterson * fix: use dom when updating sidebar Signed-off-by: Camila Belo * fix: extract small components and transformers Signed-off-by: Camila Belo * docs: describe the useTechDocsReaderDom Signed-off-by: Camila Belo * docs: update api report Signed-off-by: Camila Belo * docs: make the hook internal Signed-off-by: Camila Belo * fix: remove circular progress from the alert Signed-off-by: Camila Belo * Fix test Signed-off-by: Eric Peterson * Mark new exports as @internal, add a text string to make it easy to find in the future Signed-off-by: Eric Peterson * Changeset Signed-off-by: Eric Peterson * Do not show progress bar during checking; prevent jumpy site on navigation Signed-off-by: Eric Peterson * Combine progress/alert into a state indicator component and clarify vars. Signed-off-by: Eric Peterson * fix: documentation state updates Co-authored-by: Eric Peterson Signed-off-by: Camila Belo * fix: use circular progress again Co-authored-by: Eric Peterson Signed-off-by: Camila Belo * docs: update reader exports Co-authored-by: Eric Peterson Signed-off-by: Camila Belo * refactor: using provider instead of HOC Co-authored-by: Eric Peterson Signed-off-by: Camila Belo * refactor: update the changeset file Co-authored-by: Eric Peterson Signed-off-by: Camila Belo Co-authored-by: Eric Peterson --- .changeset/techdocs-dom-toretto.md | 5 + .../techdocs/src/reader/components/Reader.tsx | 494 +++++++++--------- .../src/reader/components/TechDocsSearch.tsx | 5 +- .../components/TechDocsStateIndicator.tsx | 142 +++++ .../techdocs/src/reader/components/index.ts | 15 + .../techdocs/src/reader/transformers/index.ts | 1 + .../transformers/scrollIntoAnchor.test.ts | 46 ++ .../reader/transformers/scrollIntoAnchor.ts | 30 ++ 8 files changed, 478 insertions(+), 260 deletions(-) create mode 100644 .changeset/techdocs-dom-toretto.md create mode 100644 plugins/techdocs/src/reader/components/TechDocsStateIndicator.tsx create mode 100644 plugins/techdocs/src/reader/transformers/scrollIntoAnchor.test.ts create mode 100644 plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts diff --git a/.changeset/techdocs-dom-toretto.md b/.changeset/techdocs-dom-toretto.md new file mode 100644 index 0000000000..3efde880f7 --- /dev/null +++ b/.changeset/techdocs-dom-toretto.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Refactored `` component internals to support future extensibility. diff --git a/plugins/techdocs/src/reader/components/Reader.tsx b/plugins/techdocs/src/reader/components/Reader.tsx index 826271a7c1..0cb6c039d0 100644 --- a/plugins/techdocs/src/reader/components/Reader.tsx +++ b/plugins/techdocs/src/reader/components/Reader.tsx @@ -14,22 +14,26 @@ * limitations under the License. */ +import React, { + PropsWithChildren, + ComponentType, + createContext, + useContext, + useCallback, + useEffect, + useRef, + useState, +} from 'react'; +import { useNavigate, useParams } from 'react-router-dom'; +import { Grid, makeStyles, useTheme } from '@material-ui/core'; + import { EntityName } from '@backstage/catalog-model'; -import { Progress } from '@backstage/core-components'; import { useApi } from '@backstage/core-plugin-api'; import { scmIntegrationsApiRef } from '@backstage/integration-react'; import { BackstageTheme } from '@backstage/theme'; -import { - Button, - CircularProgress, - Grid, - makeStyles, - useTheme, -} from '@material-ui/core'; -import { Alert } from '@material-ui/lab'; -import React, { useCallback, useEffect, useRef, useState } from 'react'; -import { useNavigate, useParams } from 'react-router-dom'; + import { techdocsStorageApiRef } from '../../api'; + import { addBaseUrl, addGitFeedbackLink, @@ -40,26 +44,21 @@ import { rewriteDocLinks, sanitizeDOM, simplifyMkdocsFooter, + scrollIntoAnchor, transform as transformer, } from '../transformers'; -import { TechDocsBuildLogs } from './TechDocsBuildLogs'; -import { TechDocsNotFound } from './TechDocsNotFound'; + import { TechDocsSearch } from './TechDocsSearch'; +import { TechDocsStateIndicator } from './TechDocsStateIndicator'; import { useReaderState } from './useReaderState'; type Props = { entityRef: EntityName; - onReady?: () => void; withSearch?: boolean; + onReady?: () => void; }; const useStyles = makeStyles(theme => ({ - message: { - // `word-break: break-word` is deprecated, but gives legacy support to browsers not supporting `overflow-wrap` yet - // https://developer.mozilla.org/en-US/docs/Web/CSS/word-break - wordBreak: 'break-word', - overflowWrap: 'anywhere', - }, searchBar: { marginLeft: '20rem', maxWidth: 'calc(100% - 20rem * 2 - 3rem)', @@ -71,44 +70,86 @@ const useStyles = makeStyles(theme => ({ }, })); -export const Reader = ({ entityRef, onReady, withSearch = true }: Props) => { - const { kind, namespace, name } = entityRef; - const theme = useTheme(); - const classes = useStyles(); +type TechDocsReaderValue = ReturnType; - const { - state, - path, - contentReload, - content: rawPage, - contentErrorMessage, - syncErrorMessage, - buildLog, - } = useReaderState(kind, namespace, name, useParams()['*']); +const TechDocsReaderContext = createContext( + {} as TechDocsReaderValue, +); - const techdocsStorageApi = useApi(techdocsStorageApiRef); - const [sidebars, setSidebars] = useState(); +const TechDocsReaderProvider = ({ children }: PropsWithChildren<{}>) => { + const { namespace = '', kind = '', name = '', '*': path } = useParams(); + const value = useReaderState(kind, namespace, name, path); + return ( + + {children} + + ); +}; + +/** + * Note: this HOC is currently being exported so that we can rapidly + * iterate on alternative implementations that extend core + * functionality. There is no guarantee that this HOC will continue to be + * exported by the package in the future! + * + * todo: Make public or stop exporting (ctrl+f "altReaderExperiments") + * @internal + */ +export const withTechDocsReaderProvider = + (Component: ComponentType) => + (props: T) => + ( + + + + ); + +/** + * Note: this hook is currently being exported so that we can rapidly + * iterate on alternative implementations that extend core + * functionality. There is no guarantee that this hook will continue to be + * exported by the package in the future! + * + * todo: Make public or stop exporting (ctrl+f "altReaderExperiments") + * @internal + */ +export const useTechDocsReader = () => useContext(TechDocsReaderContext); + +/** + * Hook that encapsulates the behavior of getting raw HTML and applying + * transforms to it in order to make it function at a basic level in the + * Backstage UI. + * + * Note: this hook is currently being exported so that we can rapidly iterate + * on alternative implementations that extend core functionality. + * There is no guarantee that this hook will continue to be exported by the + * package in the future! + * + * todo: Make public or stop exporting (see others: "altReaderExperiments") + * @internal + */ +export const useTechDocsReaderDom = (): Element | null => { const navigate = useNavigate(); - const shadowDomRef = useRef(null); + const theme = useTheme(); + const techdocsStorageApi = useApi(techdocsStorageApiRef); const scmIntegrationsApi = useApi(scmIntegrationsApiRef); + const { namespace = '', kind = '', name = '' } = useParams(); + const { state, path, content: rawPage } = useTechDocsReader(); + + const [sidebars, setSidebars] = useState(); + const [dom, setDom] = useState(null); const updateSidebarPosition = useCallback(() => { - if (!!shadowDomRef.current && !!sidebars) { - const shadowDiv: HTMLElement = shadowDomRef.current!; - const shadowRoot = - shadowDiv.shadowRoot || shadowDiv.attachShadow({ mode: 'open' }); - const mdTabs = shadowRoot.querySelector('.md-container > .md-tabs'); - sidebars!.forEach(sidebar => { - const newTop = Math.max( - shadowDomRef.current!.getBoundingClientRect().top, - 0, - ); - sidebar.style.top = mdTabs - ? `${newTop + mdTabs.getBoundingClientRect().height}px` - : `${newTop}px`; - }); - } - }, [shadowDomRef, sidebars]); + if (!dom || !sidebars) return; + // set sidebar height so they don't initially render in wrong position + const mdTabs = dom.querySelector('.md-container > .md-tabs'); + sidebars.forEach(sidebar => { + const newTop = Math.max(dom.getBoundingClientRect().top, 0); + sidebar.style.top = mdTabs + ? `${newTop + mdTabs.getBoundingClientRect().height}px` + : `${newTop}px`; + }); + }, [dom, sidebars]); useEffect(() => { updateSidebarPosition(); @@ -141,62 +182,62 @@ export const Reader = ({ entityRef, onReady, withSearch = true }: Props) => { addGitFeedbackLink(scmIntegrationsApi), injectCss({ css: ` - body { - font-family: ${theme.typography.fontFamily}; - --md-text-color: ${theme.palette.text.primary}; - --md-text-link-color: ${theme.palette.primary.main}; + body { + font-family: ${theme.typography.fontFamily}; + --md-text-color: ${theme.palette.text.primary}; + --md-text-link-color: ${theme.palette.primary.main}; - --md-code-fg-color: ${theme.palette.text.primary}; - --md-code-bg-color: ${theme.palette.background.paper}; - } - .md-main__inner { margin-top: 0; } - .md-sidebar { position: fixed; bottom: 100px; width: 20rem; } - .md-sidebar--secondary { right: 2rem; } - .md-content { margin-bottom: 50px } - .md-footer { position: fixed; bottom: 0px; width: 100vw; } - .md-footer-nav__link { width: 20rem;} - .md-content { margin-left: 20rem; max-width: calc(100% - 20rem * 2 - 3rem); } - .md-typeset { font-size: 1rem; } - .md-nav { font-size: 1rem; } - .md-grid { max-width: 90vw; margin: 0 } - .md-typeset table:not([class]) { - font-size: 1rem; - border: 1px solid ${theme.palette.text.primary}; - border-bottom: none; - border-collapse: collapse; - } - .md-typeset table:not([class]) td, .md-typeset table:not([class]) th { - border-bottom: 1px solid ${theme.palette.text.primary}; - } - .md-typeset table:not([class]) th { font-weight: bold; } - .md-typeset .admonition, .md-typeset details { - font-size: 1rem; - } - @media screen and (max-width: 76.1875em) { - .md-nav { - background-color: ${theme.palette.background.default}; - transition: none !important + --md-code-fg-color: ${theme.palette.text.primary}; + --md-code-bg-color: ${theme.palette.background.paper}; } - .md-sidebar--secondary { display: none; } - .md-sidebar--primary { left: 72px; width: 10rem } - .md-content { margin-left: 10rem; max-width: calc(100% - 10rem); } - .md-content__inner { font-size: 0.9rem } - .md-footer { - position: static; - margin-left: 10rem; - width: calc(100% - 10rem); + .md-main__inner { margin-top: 0; } + .md-sidebar { position: fixed; bottom: 100px; width: 20rem; } + .md-sidebar--secondary { right: 2rem; } + .md-content { margin-bottom: 50px } + .md-footer { position: fixed; bottom: 0px; width: 100vw; } + .md-footer-nav__link { width: 20rem;} + .md-content { margin-left: 20rem; max-width: calc(100% - 20rem * 2 - 3rem); } + .md-typeset { font-size: 1rem; } + .md-nav { font-size: 1rem; } + .md-grid { max-width: 90vw; margin: 0 } + .md-typeset table:not([class]) { + font-size: 1rem; + border: 1px solid ${theme.palette.text.primary}; + border-bottom: none; + border-collapse: collapse; } - .md-nav--primary .md-nav__title { - white-space: normal; - height: auto; - line-height: 1rem; - cursor: auto; + .md-typeset table:not([class]) td, .md-typeset table:not([class]) th { + border-bottom: 1px solid ${theme.palette.text.primary}; } - .md-nav--primary > .md-nav__title [for="none"] { - padding-top: 0; + .md-typeset table:not([class]) th { font-weight: bold; } + .md-typeset .admonition, .md-typeset details { + font-size: 1rem; } - } - `, + @media screen and (max-width: 76.1875em) { + .md-nav { + background-color: ${theme.palette.background.default}; + transition: none !important + } + .md-sidebar--secondary { display: none; } + .md-sidebar--primary { left: 72px; width: 10rem } + .md-content { margin-left: 10rem; max-width: calc(100% - 10rem); } + .md-content__inner { font-size: 0.9rem } + .md-footer { + position: static; + margin-left: 10rem; + width: calc(100% - 10rem); + } + .md-nav--primary .md-nav__title { + white-space: normal; + height: auto; + line-height: 1rem; + cursor: auto; + } + .md-nav--primary > .md-nav__title [for="none"] { + padding-top: 0; + } + } + `, }), injectCss({ // Disable CSS animations on link colors as they lead to issues in dark @@ -204,20 +245,20 @@ export const Reader = ({ entityRef, onReady, withSearch = true }: Props) => { // is always an animation from light to dark mode when navigation // between pages. css: ` - .md-nav__link, .md-typeset a, .md-typeset a::before, .md-typeset .headerlink { - transition: none; - } - `, + .md-nav__link, .md-typeset a, .md-typeset a::before, .md-typeset .headerlink { + transition: none; + } + `, }), injectCss({ // Properly style code blocks. css: ` - .md-typeset pre > code::-webkit-scrollbar-thumb { - background-color: hsla(0, 0%, 0%, 0.32); - } - .md-typeset pre > code::-webkit-scrollbar-thumb:hover { - background-color: hsla(0, 0%, 0%, 0.87); - } + .md-typeset pre > code::-webkit-scrollbar-thumb { + background-color: hsla(0, 0%, 0%, 0.32); + } + .md-typeset pre > code::-webkit-scrollbar-thumb:hover { + background-color: hsla(0, 0%, 0%, 0.87); + } `, }), injectCss({ @@ -230,30 +271,30 @@ export const Reader = ({ entityRef, onReady, withSearch = true }: Props) => { // example in the backend), we have to copy from main*.css and modify // them. css: ` - :host { - --md-admonition-icon--note: url('data:image/svg+xml;charset=utf-8,'); - --md-admonition-icon--abstract: url('data:image/svg+xml;charset=utf-8,'); - --md-admonition-icon--info: url('data:image/svg+xml;charset=utf-8,'); - --md-admonition-icon--tip: url('data:image/svg+xml;charset=utf-8,'); - --md-admonition-icon--success: url('data:image/svg+xml;charset=utf-8,'); - --md-admonition-icon--question: url('data:image/svg+xml;charset=utf-8,'); - --md-admonition-icon--warning: url('data:image/svg+xml;charset=utf-8,'); - --md-admonition-icon--failure: url('data:image/svg+xml;charset=utf-8,'); - --md-admonition-icon--danger: url('data:image/svg+xml;charset=utf-8,'); - --md-admonition-icon--bug: url('data:image/svg+xml;charset=utf-8,'); - --md-admonition-icon--example: url('data:image/svg+xml;charset=utf-8,'); - --md-admonition-icon--quote: url('data:image/svg+xml;charset=utf-8,'); - } - :host { - --md-footnotes-icon: url('data:image/svg+xml;charset=utf-8,'); - } - :host { - --md-details-icon: url('data:image/svg+xml;charset=utf-8,'); - } - :host { - --md-tasklist-icon: url('data:image/svg+xml;charset=utf-8,'); - --md-tasklist-icon--checked: url('data:image/svg+xml;charset=utf-8,'); - } + :host { + --md-admonition-icon--note: url('data:image/svg+xml;charset=utf-8,'); + --md-admonition-icon--abstract: url('data:image/svg+xml;charset=utf-8,'); + --md-admonition-icon--info: url('data:image/svg+xml;charset=utf-8,'); + --md-admonition-icon--tip: url('data:image/svg+xml;charset=utf-8,'); + --md-admonition-icon--success: url('data:image/svg+xml;charset=utf-8,'); + --md-admonition-icon--question: url('data:image/svg+xml;charset=utf-8,'); + --md-admonition-icon--warning: url('data:image/svg+xml;charset=utf-8,'); + --md-admonition-icon--failure: url('data:image/svg+xml;charset=utf-8,'); + --md-admonition-icon--danger: url('data:image/svg+xml;charset=utf-8,'); + --md-admonition-icon--bug: url('data:image/svg+xml;charset=utf-8,'); + --md-admonition-icon--example: url('data:image/svg+xml;charset=utf-8,'); + --md-admonition-icon--quote: url('data:image/svg+xml;charset=utf-8,'); + } + :host { + --md-footnotes-icon: url('data:image/svg+xml;charset=utf-8,'); + } + :host { + --md-details-icon: url('data:image/svg+xml;charset=utf-8,'); + } + :host { + --md-tasklist-icon: url('data:image/svg+xml;charset=utf-8,'); + --md-tasklist-icon--checked: url('data:image/svg+xml;charset=utf-8,'); + } `, }), ]), @@ -273,29 +314,18 @@ export const Reader = ({ entityRef, onReady, withSearch = true }: Props) => { // a function that performs transformations that are executed after adding it to the DOM const postRender = useCallback( - async (shadowRoot: ShadowRoot) => - transformer(shadowRoot.children[0], [ - dom => { - setTimeout(() => { - // Scoll to the desired anchor on initial navigation - if (window.location.hash) { - const hash = window.location.hash.slice(1); - shadowRoot?.getElementById(hash)?.scrollIntoView(); - } - }, 200); - return dom; - }, + async (transformedElement: Element) => + transformer(transformedElement, [ + scrollIntoAnchor(), addLinkClickListener({ baseUrl: window.location.origin, onClick: (_: MouseEvent, url: string) => { const parsedUrl = new URL(url); - if (parsedUrl.hash) { navigate(`${parsedUrl.pathname}${parsedUrl.hash}`); - // Scroll to hash if it's on the current page - shadowRoot - ?.getElementById(parsedUrl.hash.slice(1)) + transformedElement + ?.querySelector(`#${parsedUrl.hash.slice(1)}`) ?.scrollIntoView(); } else { navigate(parsedUrl.pathname); @@ -304,28 +334,18 @@ export const Reader = ({ entityRef, onReady, withSearch = true }: Props) => { }), onCssReady({ docStorageUrl: await techdocsStorageApi.getApiOrigin(), - onLoading: (dom: Element) => { - (dom as HTMLElement).style.setProperty('opacity', '0'); + onLoading: (renderedElement: Element) => { + (renderedElement as HTMLElement).style.setProperty('opacity', '0'); }, - onLoaded: (dom: Element) => { - (dom as HTMLElement).style.removeProperty('opacity'); + onLoaded: (renderedElement: Element) => { + (renderedElement as HTMLElement).style.removeProperty('opacity'); // disable MkDocs drawer toggling ('for' attribute => checkbox mechanism) - (dom as HTMLElement) + renderedElement .querySelector('.md-nav__title') ?.removeAttribute('for'); - const sideDivs: HTMLElement[] = Array.from( - shadowRoot!.querySelectorAll('.md-sidebar'), + setSidebars( + Array.from(renderedElement.querySelectorAll('.md-sidebar')), ); - setSidebars(sideDivs); - // set sidebar height so they don't initially render in wrong position - const docTopPosition = (dom as HTMLElement).getBoundingClientRect() - .top; - const mdTabs = dom.querySelector('.md-container > .md-tabs'); - sideDivs!.forEach(sidebar => { - sidebar.style.top = mdTabs - ? `${docTopPosition + mdTabs.getBoundingClientRect().height}px` - : `${docTopPosition}px`; - }); }, }), ]), @@ -333,23 +353,14 @@ export const Reader = ({ entityRef, onReady, withSearch = true }: Props) => { ); useEffect(() => { - if (!rawPage || !shadowDomRef.current) { - // clear the shadow dom if no content is available - if (shadowDomRef.current?.shadowRoot) { - shadowDomRef.current.shadowRoot.innerHTML = ''; - } - return () => {}; - } - if (onReady) { - onReady(); - } + if (!rawPage) return () => {}; // if false, there is already a newer execution of this effect let shouldReplaceContent = true; // Pre-render - preRender(rawPage, path).then(async transformedElement => { - if (!transformedElement?.innerHTML) { + preRender(rawPage, path).then(async preTransformedDomElement => { + if (!preTransformedDomElement?.innerHTML) { return; // An unexpected error occurred } @@ -358,94 +369,49 @@ export const Reader = ({ entityRef, onReady, withSearch = true }: Props) => { return; } - const shadowDiv: HTMLElement = shadowDomRef.current!; - const shadowRoot = - shadowDiv.shadowRoot || shadowDiv.attachShadow({ mode: 'open' }); - Array.from(shadowRoot.children).forEach(child => - shadowRoot.removeChild(child), - ); - shadowRoot.appendChild(transformedElement); - // Scroll to top after render window.scroll({ top: 0 }); // Post-render - await postRender(shadowRoot); + const postTransformedDomElement = await postRender( + preTransformedDomElement, + ); + setDom(postTransformedDomElement as HTMLElement); }); // cancel this execution return () => { shouldReplaceContent = false; }; - }, [onReady, path, postRender, preRender, rawPage]); + }, [rawPage, path, preRender, postRender]); + + return dom; +}; + +const TheReader = ({ + entityRef, + onReady = () => {}, + withSearch = true, +}: Props) => { + const classes = useStyles(); + const dom = useTechDocsReaderDom(); + const shadowDomRef = useRef(null); + + useEffect(() => { + if (!dom || !shadowDomRef.current) return; + const shadowDiv = shadowDomRef.current; + const shadowRoot = + shadowDiv.shadowRoot || shadowDiv.attachShadow({ mode: 'open' }); + Array.from(shadowRoot.children).forEach(child => + shadowRoot.removeChild(child), + ); + shadowRoot.appendChild(dom); + onReady(); + }, [dom, onReady]); return ( <> - {state === 'CHECKING' && } - {state === 'INITIAL_BUILD' && ( - } - action={} - > - Documentation is accessed for the first time and is being prepared. - The subsequent loads are much faster. - - )} - {state === 'CONTENT_STALE_REFRESHING' && ( - } - action={} - > - A newer version of this documentation is being prepared and will be - available shortly. - - )} - {state === 'CONTENT_STALE_READY' && ( - contentReload()}> - Refresh - - } - > - A newer version of this documentation is now available, please refresh - to view. - - )} - {state === 'CONTENT_STALE_ERROR' && ( - } - classes={{ message: classes.message }} - > - Building a newer version of this documentation failed.{' '} - {syncErrorMessage} - - )} - {state === 'CONTENT_NOT_FOUND' && ( - <> - {syncErrorMessage && ( - } - classes={{ message: classes.message }} - > - Building a newer version of this documentation failed.{' '} - {syncErrorMessage} - - )} - - - )} - + {withSearch && shadowDomRef?.current?.shadowRoot?.innerHTML && ( @@ -455,3 +421,17 @@ export const Reader = ({ entityRef, onReady, withSearch = true }: Props) => { ); }; + +export const Reader = ({ + entityRef, + onReady = () => {}, + withSearch = true, +}: Props) => ( + + + +); diff --git a/plugins/techdocs/src/reader/components/TechDocsSearch.tsx b/plugins/techdocs/src/reader/components/TechDocsSearch.tsx index 5d330100ce..64229e6620 100644 --- a/plugins/techdocs/src/reader/components/TechDocsSearch.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsSearch.tsx @@ -61,7 +61,7 @@ export const buildInitialFilters = ( return legacyPaths ? entityId : Object.entries(entityId).reduce((acc, [key, value]) => { - return { ...acc, [key]: value.toLocaleLowerCase('en-US') }; + return { ...acc, [key]: value?.toLocaleLowerCase('en-US') }; }, {}); }; @@ -175,7 +175,7 @@ const TechDocsSearchBar = ({ ); }; -const TechDocsSearch = (props: TechDocsSearchProps) => { +export const TechDocsSearch = (props: TechDocsSearchProps) => { const configApi = useApi(configApiRef); const legacyPaths = configApi.getOptionalBoolean( 'techdocs.legacyUseCaseSensitiveTripletPaths', @@ -192,4 +192,3 @@ const TechDocsSearch = (props: TechDocsSearchProps) => { ); }; -export { TechDocsSearch }; diff --git a/plugins/techdocs/src/reader/components/TechDocsStateIndicator.tsx b/plugins/techdocs/src/reader/components/TechDocsStateIndicator.tsx new file mode 100644 index 0000000000..e70c17d14a --- /dev/null +++ b/plugins/techdocs/src/reader/components/TechDocsStateIndicator.tsx @@ -0,0 +1,142 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { Progress } from '@backstage/core-components'; +import { CircularProgress, Button, makeStyles } from '@material-ui/core'; +import { Alert } from '@material-ui/lab'; + +import { TechDocsBuildLogs } from './TechDocsBuildLogs'; +import { TechDocsNotFound } from './TechDocsNotFound'; +import { useTechDocsReader } from './Reader'; + +const useStyles = makeStyles(() => ({ + message: { + // `word-break: break-word` is deprecated, but gives legacy support to browsers not supporting `overflow-wrap` yet + // https://developer.mozilla.org/en-US/docs/Web/CSS/word-break + wordBreak: 'break-word', + overflowWrap: 'anywhere', + }, +})); + +/** + * Note: this component is currently being exported so that we can rapidly + * iterate on alternative implementations that extend core + * functionality. There is no guarantee that this component will continue to be + * exported by the package in the future! + * + * todo: Make public or stop exporting (ctrl+f "altReaderExperiments") + * @internal + */ +export const TechDocsStateIndicator = () => { + let StateAlert: JSX.Element | null = null; + const classes = useStyles(); + + const { + state, + contentReload, + contentErrorMessage, + syncErrorMessage, + buildLog, + } = useTechDocsReader(); + + const ReaderProgress = state === 'CHECKING' ? : null; + + if (state === 'INITIAL_BUILD') { + StateAlert = ( + } + action={} + > + Documentation is accessed for the first time and is being prepared. The + subsequent loads are much faster. + + ); + } + + if (state === 'CONTENT_STALE_REFRESHING') { + StateAlert = ( + } + action={} + > + A newer version of this documentation is being prepared and will be + available shortly. + + ); + } + + if (state === 'CONTENT_STALE_READY') { + StateAlert = ( + contentReload()}> + Refresh + + } + > + A newer version of this documentation is now available, please refresh + to view. + + ); + } + + if (state === 'CONTENT_STALE_ERROR') { + StateAlert = ( + } + classes={{ message: classes.message }} + > + Building a newer version of this documentation failed.{' '} + {syncErrorMessage} + + ); + } + + if (state === 'CONTENT_NOT_FOUND') { + StateAlert = ( + <> + {syncErrorMessage && ( + } + classes={{ message: classes.message }} + > + Building a newer version of this documentation failed.{' '} + {syncErrorMessage} + + )} + + + ); + } + + return ( + <> + {ReaderProgress} + {StateAlert} + + ); +}; diff --git a/plugins/techdocs/src/reader/components/index.ts b/plugins/techdocs/src/reader/components/index.ts index 4f6d151e5d..99a0a7a3bc 100644 --- a/plugins/techdocs/src/reader/components/index.ts +++ b/plugins/techdocs/src/reader/components/index.ts @@ -17,3 +17,18 @@ export * from './Reader'; export * from './TechDocsPage'; export * from './TechDocsPageHeader'; +export * from './TechDocsStateIndicator'; + +/** + * Note: this component is currently being exported so that we can rapidly + * iterate on alternative implementations that extend core + * functionality. There is no guarantee that this component will continue to be + * exported by the package in the future! + * + * Why is this comment here instead of above the component itself? It's a + * workaround for some kind of bug in @microsoft/api-extractor. + * + * todo: Make public or stop exporting (ctrl+f "altReaderExperiments") + * @internal + */ +export { TechDocsSearch } from './TechDocsSearch'; diff --git a/plugins/techdocs/src/reader/transformers/index.ts b/plugins/techdocs/src/reader/transformers/index.ts index cd1ad6511c..530bea2092 100644 --- a/plugins/techdocs/src/reader/transformers/index.ts +++ b/plugins/techdocs/src/reader/transformers/index.ts @@ -23,4 +23,5 @@ export * from './simplifyMkdocsFooter'; export * from './onCssReady'; export * from './sanitizeDOM'; export * from './injectCss'; +export * from './scrollIntoAnchor'; export * from './transformer'; diff --git a/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.test.ts b/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.test.ts new file mode 100644 index 0000000000..b06762399b --- /dev/null +++ b/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { scrollIntoAnchor } from '../transformers'; + +jest.useFakeTimers(); + +describe('scrollIntoAnchor', () => { + const transformer = scrollIntoAnchor(); + const dom = { querySelector: jest.fn() }; + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('does nothing if there is no anchor element', async () => { + transformer(dom as unknown as Element); + jest.advanceTimersByTime(200); + expect(dom.querySelector).not.toHaveBeenCalled(); + }); + + it('scroll to the hash anchor element', async () => { + const scrollIntoView = jest.fn(); + dom.querySelector.mockReturnValue({ scrollIntoView }); + const hash = '#hash'; + window.location.hash = hash; + transformer(dom as unknown as Element); + jest.advanceTimersByTime(200); + expect(dom.querySelector).toHaveBeenCalledWith(`#${hash.slice(1)}`); + expect(scrollIntoView).toHaveBeenCalledWith(); + window.location.hash = ''; + }); +}); diff --git a/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts b/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts new file mode 100644 index 0000000000..0fa977865e --- /dev/null +++ b/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts @@ -0,0 +1,30 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Transformer } from './transformer'; + +export const scrollIntoAnchor = (): Transformer => { + return dom => { + setTimeout(() => { + // Scroll to the desired anchor on initial navigation + if (window.location.hash) { + const hash = window.location.hash.slice(1); + dom?.querySelector(`#${hash}`)?.scrollIntoView(); + } + }, 200); + return dom; + }; +}; From 9f6c4233ac7aa37e019dd1b61f6b80d538d84c7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 13 Oct 2021 16:18:59 +0200 Subject: [PATCH 04/60] made more tests async MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../PreviewCatalogInfoComponent.test.tsx | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/PreviewCatalogInfoComponent.test.tsx b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/PreviewCatalogInfoComponent.test.tsx index 2618fab57a..ea3ea3e96f 100644 --- a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/PreviewCatalogInfoComponent.test.tsx +++ b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/PreviewCatalogInfoComponent.test.tsx @@ -16,7 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { makeStyles } from '@material-ui/core'; -import { render } from '@testing-library/react'; +import { render, screen } from '@testing-library/react'; import { renderHook } from '@testing-library/react-hooks'; import React from 'react'; import { PreviewCatalogInfoComponent } from './PreviewCatalogInfoComponent'; @@ -46,15 +46,17 @@ const entities: Entity[] = [ describe('', () => { it('renders without exploding', async () => { - const { getByText, findByText } = render( + render( , ); - const repositoryUrl = getByText('http://my-repository/a/catalog-info.yaml'); - const kindText = await findByText('Kind_2'); + const repositoryUrl = screen.getByText( + 'http://my-repository/a/catalog-info.yaml', + ); + const kindText = await screen.findByText('Kind_2'); expect(repositoryUrl).toBeInTheDocument(); expect(repositoryUrl).toBeVisible(); expect(kindText).toBeInTheDocument(); @@ -64,7 +66,7 @@ describe('', () => { it('renders card with custom styles', async () => { const { result } = renderHook(() => useStyles()); - const { getByText } = render( + render( ', () => { />, ); - const repositoryUrl = getByText('http://my-repository/a/catalog-info.yaml'); - const kindText = getByText('Kind_2'); + const repositoryUrl = screen.getByText( + 'http://my-repository/a/catalog-info.yaml', + ); + const kindText = await screen.findByText('Kind_2'); expect(repositoryUrl).toBeInTheDocument(); expect(repositoryUrl).not.toBeVisible(); expect(kindText).toBeInTheDocument(); @@ -83,7 +87,7 @@ describe('', () => { it('renders with custom styles', async () => { const { result } = renderHook(() => useStyles()); - const { getByText } = render( + render( ', () => { />, ); - const repositoryUrl = getByText('http://my-repository/a/catalog-info.yaml'); - const kindText = getByText('Kind_2'); + const repositoryUrl = screen.getByText( + 'http://my-repository/a/catalog-info.yaml', + ); + const kindText = await screen.findByText('Kind_2'); expect(repositoryUrl).toBeInTheDocument(); expect(repositoryUrl).toBeVisible(); expect(kindText).toBeInTheDocument(); From b486adb8c6707e010044178ba3f19557ac4ee6a7 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 9 Oct 2021 14:24:01 +0200 Subject: [PATCH 05/60] cli: rework jest ESM handling by transforming by default Signed-off-by: Patrik Oldsberg --- .changeset/brave-waves-explain.md | 15 +++++ .changeset/wise-camels-run.md | 13 ++++ package.json | 5 -- packages/cli/config/jest.js | 56 ++++++++++++---- packages/cli/config/jestSucraseTransform.js | 66 +++++++++++++++++++ .../templates/default-app/package.json.hbs | 5 -- 6 files changed, 136 insertions(+), 24 deletions(-) create mode 100644 .changeset/brave-waves-explain.md create mode 100644 .changeset/wise-camels-run.md create mode 100644 packages/cli/config/jestSucraseTransform.js diff --git a/.changeset/brave-waves-explain.md b/.changeset/brave-waves-explain.md new file mode 100644 index 0000000000..5e76f192e0 --- /dev/null +++ b/.changeset/brave-waves-explain.md @@ -0,0 +1,15 @@ +--- +'@backstage/create-app': patch +--- + +Removed the included `jest` configuration from the root `package.json` as the `transformModules` option no longer exists. + +To apply this change to an existing app, make the follow change to the root `package.json`: + +```diff +- "jest": { +- "transformModules": [ +- "@asyncapi/react-component" +- ] +- } +``` diff --git a/.changeset/wise-camels-run.md b/.changeset/wise-camels-run.md new file mode 100644 index 0000000000..1e81e2f9ac --- /dev/null +++ b/.changeset/wise-camels-run.md @@ -0,0 +1,13 @@ +--- +'@backstage/cli': minor +--- + +The Jest configuration that's included with the Backstage CLI has received several changes. + +As a part of migrating to more widespread usage of ESM modules, the default configuration now transforms all source files everywhere, including those within `node_modules`. Due to this change the existing `transformModules` option has been removed and will be ignored. There is also a list of known packages that do not require transforms in the CLI, which will evolve over time. If needed there will also be an option to add packages to this list in the future, but it is not included yet to avoid clutter. + +To counteract the slowdown of the additional transforms that have been introduced, the default configuration has also been reworked to enable caching across different packages. Previously each package in a Backstage monorepo would have its own isolated Jest cache, but it is now shared between packages that have a similar enough Jest configuration. + +Another change that will speed up test execution is that the transformer for `.esm.js` files has been switched. It used to be an ESM transformer based on Babel, but it is also done by sucrase now since it is significantly faster. + +The changes above are not strictly breaking as all tests should still work. It may however cause excessive slowdowns in projects that have configured custom transforms in the `jest` field within `package.json` files. In this case it is either best to consider removing the custom transforms, or overriding the `transformIgnorePatterns` to instead use Jest's default `'/node_modules/'` pattern. diff --git a/package.json b/package.json index 9c0f41ca53..4aa6905085 100644 --- a/package.json +++ b/package.json @@ -85,10 +85,5 @@ "*.md": [ "node ./scripts/check-docs-quality" ] - }, - "jest": { - "transformModules": [ - "@asyncapi/react-component" - ] } } diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index c687616a7d..e889d23c3a 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -16,7 +16,25 @@ const fs = require('fs-extra'); const path = require('path'); +const crypto = require('crypto'); const glob = require('util').promisify(require('glob')); +const { version } = require('../package.json'); + +const transformIgnorePattern = [ + '@material-ui', + '@rjsf', + 'ajv', + 'core-js', + 'jest-.*', + 'jsdom', + 'knex', + 'react', + 'react-dom', + 'highlight\\.js', + 'prismjs', + 'react-use', + 'typescript', +].join('|'); async function getProjectConfig(targetPath, displayName) { const configJsPath = path.resolve(targetPath, 'jest.config.js'); @@ -58,10 +76,7 @@ async function getProjectConfig(targetPath, displayName) { currentPath = newPath; } - // We add an additional Jest config parameter only known by the Backstage CLI - // called `transformModules`. It's a list of modules that we want to apply - // our configured jest transformations for. - // This is useful when packages are published in untranspiled ESM or TS form. + // This is an old deprecated option that is no longer used. const transformModules = pkgJsonConfigs .flatMap(conf => { const modules = conf.transformModules || []; @@ -70,10 +85,14 @@ async function getProjectConfig(targetPath, displayName) { }) .map(name => `${name}/`) .join('|'); - const transformModulePattern = transformModules && `(?!${transformModules})`; + if (transformModules.length > 0) { + console.warn( + 'The Backstage CLI jest transformModules option is no longer used and will be ignored. All modules are now always transformed.', + ); + } const options = { - displayName, + ...(displayName && { displayName }), rootDir: path.resolve(targetPath, 'src'), coverageDirectory: path.resolve(targetPath, 'coverage'), collectCoverageFrom: ['**/*.{js,jsx,ts,tsx}', '!**/*.d.ts'], @@ -82,8 +101,7 @@ async function getProjectConfig(targetPath, displayName) { }, transform: { - '\\.esm\\.js$': require.resolve('./jestEsmTransform.js'), // See jestEsmTransform.js - '\\.(js|jsx|ts|tsx)$': require.resolve('@sucrase/jest-plugin'), + '\\.(js|jsx|ts|tsx)$': require.resolve('./jestSucraseTransform.js'), '\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg|eot|woff|woff2|ttf)$': require.resolve('./jestFileTransform.js'), '\\.(yaml)$': require.resolve('jest-transform-yaml'), @@ -92,11 +110,7 @@ async function getProjectConfig(targetPath, displayName) { // A bit more opinionated testMatch: ['**/?(*.)test.{js,jsx,mjs,ts,tsx}'], - // Default behaviour is to not apply transforms for node_modules, but we still want - // to apply the esm-transformer to .esm.js files, since that's what we use in backstage packages. - transformIgnorePatterns: [ - `/node_modules/${transformModulePattern}.*\\.(?:(?/setupTests.ts']; } - return Object.assign(options, ...pkgJsonConfigs); + const config = Object.assign(options, ...pkgJsonConfigs); + + // The config name is a cache key that lets us share the jest cache across projects. + // If no explicit name was configured, generated one based on the configuration. + if (!config.name) { + const configHash = crypto + .createHash('md5') + .update(version) + .update(Buffer.alloc(1)) + .update(JSON.stringify(config.transform)) + .digest('hex'); + config.name = `backstage_cli_${configHash}`; + } + + return config; } // This loads the root jest config, which in turn will either refer to a single diff --git a/packages/cli/config/jestSucraseTransform.js b/packages/cli/config/jestSucraseTransform.js new file mode 100644 index 0000000000..01acfee38d --- /dev/null +++ b/packages/cli/config/jestSucraseTransform.js @@ -0,0 +1,66 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { createHash } = require('crypto'); +const { transform } = require('sucrase'); +const sucrasePkg = require('sucrase/package.json'); +const sucrasePluginPkg = require('@sucrase/jest-plugin/package.json'); + +const ESM_REGEX = /\b(?:import|export)\b/; + +function process(source, filePath) { + let transforms; + + if (filePath.endsWith('.esm.js')) { + transforms = ['imports']; + } else if (filePath.endsWith('.js')) { + // This is a very rough filter to avoid transforming things that we quickly + // can be sure are definitely not ESM modules. + if (ESM_REGEX.test(source)) { + transforms = ['imports', 'jsx']; // JSX within .js is currently allowed + } + } else if (filePath.endsWith('.jsx')) { + transforms = ['jsx', 'imports']; + } else if (filePath.endsWith('.ts')) { + transforms = ['typescript', 'imports']; + } else if (filePath.endsWith('.tsx')) { + transforms = ['typescript', 'jsx', 'imports']; + } + + // Only apply the jest transform to the test files themselves + if (transforms && filePath.includes('.test.')) { + transforms.push('jest'); + } + + if (transforms) { + return transform(source, { transforms, filePath }).code; + } + + return source; +} + +// TODO: contribute something like this to @sucrase/jest-plugin +function getCacheKey(sourceText) { + return createHash('md5') + .update(sourceText) + .update(Buffer.alloc(1)) + .update(sucrasePkg.version) + .update(Buffer.alloc(1)) + .update(sucrasePluginPkg.version) + .digest('hex'); +} + +module.exports = { process, getCacheKey }; diff --git a/packages/create-app/templates/default-app/package.json.hbs b/packages/create-app/templates/default-app/package.json.hbs index 7e2702e213..5ed066a30b 100644 --- a/packages/create-app/templates/default-app/package.json.hbs +++ b/packages/create-app/templates/default-app/package.json.hbs @@ -49,10 +49,5 @@ "*.{json,md}": [ "prettier --write" ] - }, - "jest": { - "transformModules": [ - "@asyncapi/react-component" - ] } } From 4050e33f039c984be6d0a6edccb8f5d2fbcbadbd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 12 Oct 2021 18:01:56 +0200 Subject: [PATCH 06/60] cli: remove old jest ESM transform Signed-off-by: Patrik Oldsberg --- .changeset/wise-camels-run.md | 2 ++ packages/cli/config/jestEsmTransform.js | 36 ------------------------- packages/cli/package.json | 3 --- yarn.lock | 4 +-- 4 files changed, 4 insertions(+), 41 deletions(-) delete mode 100644 packages/cli/config/jestEsmTransform.js diff --git a/.changeset/wise-camels-run.md b/.changeset/wise-camels-run.md index 1e81e2f9ac..4438032b77 100644 --- a/.changeset/wise-camels-run.md +++ b/.changeset/wise-camels-run.md @@ -11,3 +11,5 @@ To counteract the slowdown of the additional transforms that have been introduce Another change that will speed up test execution is that the transformer for `.esm.js` files has been switched. It used to be an ESM transformer based on Babel, but it is also done by sucrase now since it is significantly faster. The changes above are not strictly breaking as all tests should still work. It may however cause excessive slowdowns in projects that have configured custom transforms in the `jest` field within `package.json` files. In this case it is either best to consider removing the custom transforms, or overriding the `transformIgnorePatterns` to instead use Jest's default `'/node_modules/'` pattern. + +This change also removes the `@backstage/cli/config/jestEsmTransform.js` transform, which can be replaced by using the `@backstage/cli/config/sucraseEsmTransform.js` transform instead. diff --git a/packages/cli/config/jestEsmTransform.js b/packages/cli/config/jestEsmTransform.js deleted file mode 100644 index 742822274d..0000000000 --- a/packages/cli/config/jestEsmTransform.js +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const babel = require('@babel/core'); - -// We build .esm.js files with plugin:build, so to be able to load these in tests they need to be transformed -// TODO: jest is working on module support, it's possible that we can remove this in the future -module.exports = { - process(src) { - const result = babel.transform(src, { - babelrc: false, - compact: false, - plugins: [ - // This transforms the regular ESM syntax, import and export statements - require.resolve('@babel/plugin-transform-modules-commonjs'), - // This transforms dynamic `import()`, which is not supported yet in the Node.js VM API - require.resolve('babel-plugin-dynamic-import-node'), - ], - }); - - return result.code; - }, -}; diff --git a/packages/cli/package.json b/packages/cli/package.json index 53c1b06e76..426c4fd1ac 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -28,8 +28,6 @@ "backstage-cli": "bin/backstage-cli" }, "dependencies": { - "@babel/core": "^7.4.4", - "@babel/plugin-transform-modules-commonjs": "^7.4.4", "@backstage/cli-common": "^0.1.4", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.6.10", @@ -54,7 +52,6 @@ "@typescript-eslint/eslint-plugin": "^v4.30.0", "@typescript-eslint/parser": "^v4.28.3", "@yarnpkg/lockfile": "^1.1.0", - "babel-plugin-dynamic-import-node": "^2.3.3", "bfj": "^7.0.2", "buffer": "^6.0.3", "chalk": "^4.0.0", diff --git a/yarn.lock b/yarn.lock index 1a0caad7e9..521628b019 100644 --- a/yarn.lock +++ b/yarn.lock @@ -375,7 +375,7 @@ semver "^5.4.1" source-map "^0.5.0" -"@babel/core@^7.0.0", "@babel/core@^7.1.0", "@babel/core@^7.12.10", "@babel/core@^7.12.3", "@babel/core@^7.13.16", "@babel/core@^7.4.4", "@babel/core@^7.7.5": +"@babel/core@^7.0.0", "@babel/core@^7.1.0", "@babel/core@^7.12.10", "@babel/core@^7.12.3", "@babel/core@^7.13.16", "@babel/core@^7.7.5": version "7.14.8" resolved "https://registry.npmjs.org/@babel/core/-/core-7.14.8.tgz#20cdf7c84b5d86d83fac8710a8bc605a7ba3f010" integrity sha512-/AtaeEhT6ErpDhInbXmjHcUQXH0L0TEgscfcxk1qbOvLuKCa5aZT0SOOtDKFY96/CLROwbLSKyFor6idgNaU4Q== @@ -1536,7 +1536,7 @@ "@babel/helper-plugin-utils" "^7.14.5" babel-plugin-dynamic-import-node "^2.3.3" -"@babel/plugin-transform-modules-commonjs@^7.0.0", "@babel/plugin-transform-modules-commonjs@^7.13.8", "@babel/plugin-transform-modules-commonjs@^7.14.0", "@babel/plugin-transform-modules-commonjs@^7.14.5", "@babel/plugin-transform-modules-commonjs@^7.4.4": +"@babel/plugin-transform-modules-commonjs@^7.0.0", "@babel/plugin-transform-modules-commonjs@^7.13.8", "@babel/plugin-transform-modules-commonjs@^7.14.0", "@babel/plugin-transform-modules-commonjs@^7.14.5": version "7.14.5" resolved "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.14.5.tgz#7aaee0ea98283de94da98b28f8c35701429dad97" integrity sha512-en8GfBtgnydoao2PS+87mKyw62k02k7kJ9ltbKe0fXTHrQmG6QZZflYuGI1VVG7sVpx4E1n7KBpNlPb8m78J+A== From dd9cf64dd5e28264a1f1484f2fe17e69268c1cad Mon Sep 17 00:00:00 2001 From: Connor Younglund Date: Wed, 13 Oct 2021 13:10:15 -0400 Subject: [PATCH 07/60] replace `\n` with newline for GitHub App private key Signed-off-by: Connor Younglund --- packages/integration/src/github/GithubCredentialsProvider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/integration/src/github/GithubCredentialsProvider.ts b/packages/integration/src/github/GithubCredentialsProvider.ts index 7e6057fc8d..278a1ed028 100644 --- a/packages/integration/src/github/GithubCredentialsProvider.ts +++ b/packages/integration/src/github/GithubCredentialsProvider.ts @@ -74,7 +74,7 @@ class GithubAppManager { this.baseUrl = baseUrl; this.baseAuthConfig = { appId: config.appId, - privateKey: config.privateKey, + privateKey: config.privateKey.replace(/\\n/gm, '\n'), }; this.appClient = new Octokit({ baseUrl, From eab072161e36cd0a8457212f8d1f21e3671894e6 Mon Sep 17 00:00:00 2001 From: Connor Younglund Date: Wed, 13 Oct 2021 13:30:09 -0400 Subject: [PATCH 08/60] generate changeset Signed-off-by: Connor Younglund --- .changeset/flat-camels-scream.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/flat-camels-scream.md diff --git a/.changeset/flat-camels-scream.md b/.changeset/flat-camels-scream.md new file mode 100644 index 0000000000..2c49af5010 --- /dev/null +++ b/.changeset/flat-camels-scream.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': patch +--- + +By replacing `\n` with a newline for GitHub Apps private keys, this allows users to store the private key as an environment variable and reference it in the YAML. From e63a74563a760f6c9388f2cf0ddd73289052c894 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Oct 2021 04:09:59 +0000 Subject: [PATCH 09/60] Bump @types/body-parser from 1.19.0 to 1.19.1 Bumps [@types/body-parser](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/body-parser) from 1.19.0 to 1.19.1. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/body-parser) --- updated-dependencies: - dependency-name: "@types/body-parser" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 59 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/yarn.lock b/yarn.lock index c17580a415..f2a5002d77 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6421,7 +6421,15 @@ dependencies: "@babel/types" "^7.3.0" -"@types/body-parser@*", "@types/body-parser@1.19.0", "@types/body-parser@^1.19.0": +"@types/body-parser@*", "@types/body-parser@^1.19.0": + version "1.19.1" + resolved "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.1.tgz#0c0174c42a7d017b818303d4b5d969cb0b75929c" + integrity sha512-a6bTJ21vFOGIkwM0kzh9Yr89ziVxq4vYH2fQ6N8AeipEzai/cFK6aGMArIkUeIdRIgpwQa+2bXiLuUJCpSf2Cg== + dependencies: + "@types/connect" "*" + "@types/node" "*" + +"@types/body-parser@1.19.0": version "1.19.0" resolved "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz#0685b3c47eb3006ffed117cdd55164b61f80538f" integrity sha512-W98JrE0j2K78swW4ukqMleo8R7h/pFETjM2DQ90MF6XK2i4LO4W3gQ71Lt4w3bfm2EvVSyWHplECvB5sK22yFQ== @@ -7123,10 +7131,10 @@ "@types/node" "*" form-data "^3.0.0" -"@types/node@*", "@types/node@>= 8", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@^14.0.10", "@types/node@^14.14.31", "@types/node@^14.14.32", "@types/node@^14.14.33": - version "14.17.8" - resolved "https://registry.npmjs.org/@types/node/-/node-14.17.8.tgz#813b73ab7d82ac06ddfd2458b13c88459a3b319f" - integrity sha512-0CHLt50GbUmH/6MrlBIKNdWCglvlyQKkorRf08/0DIi0ryuTPP+ijWLSI19SbDTHSKaagGDELiImY4BSikt61w== +"@types/node@*", "@types/node@>= 8", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@^15.6.1": + version "15.14.9" + resolved "https://registry.npmjs.org/@types/node/-/node-15.14.9.tgz#bc43c990c3c9be7281868bbc7b8fdd6e2b57adfa" + integrity sha512-qjd88DrCxupx/kJD5yQgZdcYKZKSIGBVDIBE1/LTGcNm3d2Np/jxojkdePDdfnBHJc5W7vSMpbJ1aB7p/Py69A== "@types/node@10.17.13", "@types/node@^10.1.0", "@types/node@^10.12.0": version "10.17.13" @@ -7138,10 +7146,10 @@ resolved "https://registry.npmjs.org/@types/node/-/node-12.12.58.tgz#46dae9b2b9ee5992818c8f7cee01ff4ce03ab44c" integrity sha512-Be46CNIHWAagEfINOjmriSxuv7IVcqbGe+sDSg2SYCEz/0CRBy7LRASGfRbD8KZkqoePU73Wsx3UvOSFcq/9hA== -"@types/node@^15.6.1": - version "15.14.9" - resolved "https://registry.npmjs.org/@types/node/-/node-15.14.9.tgz#bc43c990c3c9be7281868bbc7b8fdd6e2b57adfa" - integrity sha512-qjd88DrCxupx/kJD5yQgZdcYKZKSIGBVDIBE1/LTGcNm3d2Np/jxojkdePDdfnBHJc5W7vSMpbJ1aB7p/Py69A== +"@types/node@^14.0.10", "@types/node@^14.14.31", "@types/node@^14.14.32", "@types/node@^14.14.33": + version "14.17.8" + resolved "https://registry.npmjs.org/@types/node/-/node-14.17.8.tgz#813b73ab7d82ac06ddfd2458b13c88459a3b319f" + integrity sha512-0CHLt50GbUmH/6MrlBIKNdWCglvlyQKkorRf08/0DIi0ryuTPP+ijWLSI19SbDTHSKaagGDELiImY4BSikt61w== "@types/normalize-package-data@^2.4.0": version "2.4.0" @@ -7365,7 +7373,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@>=16.9.0", "@types/react@^16.9": +"@types/react@*", "@types/react@>=16.9.0": version "16.14.15" resolved "https://registry.npmjs.org/@types/react/-/react-16.14.15.tgz#95d8fa3148050e94bcdc5751447921adbe19f9e6" integrity sha512-jOxlBV9RGZhphdeqJTCv35VZOkjY+XIEY2owwSk84BNDdDv2xS6Csj6fhi+B/q30SR9Tz8lDNt/F2Z5RF3TrRg== @@ -7374,6 +7382,15 @@ "@types/scheduler" "*" csstype "^3.0.2" +"@types/react@^16.9": + version "16.14.17" + resolved "https://registry.npmjs.org/@types/react/-/react-16.14.17.tgz#c57fcfb05efa6423f5b65fcd4a75f63f05b162bf" + integrity sha512-pMLc/7+7SEdQa9A+hN9ujI8blkjFqYAZVqh3iNXqdZ0cQ8TIR502HMkNJniaOGv9SAgc47jxVKoiBJ7c0AakvQ== + dependencies: + "@types/prop-types" "*" + "@types/scheduler" "*" + csstype "^3.0.2" + "@types/recharts@^1.8.14", "@types/recharts@^1.8.15": version "1.8.19" resolved "https://registry.npmjs.org/@types/recharts/-/recharts-1.8.19.tgz#047f72cf4c25df545aa1085fe3a085e58a2483c1" @@ -10237,9 +10254,9 @@ canvas@^2.6.1: simple-get "^3.0.3" canvg@^3.0.6: - version "3.0.8" - resolved "https://registry.npmjs.org/canvg/-/canvg-3.0.8.tgz#9125a4b7033d2f237402241b192587385f4589e6" - integrity sha512-9De5heHfVRgCkln3CGEeSJMviN5U2RyxL4uutYoe8HxI60BjH2XnT2ZUHIp+ZaAZNTUd5Asqfut8WEEdANqfAg== + version "3.0.9" + resolved "https://registry.npmjs.org/canvg/-/canvg-3.0.9.tgz#9ba095f158b94b97ca2c9c1c40785b11dc08df6d" + integrity sha512-rDXcnRPuz4QHoCilMeoTxql+fvGqNAxp+qV/KHD8rOiJSAfVjFclbdUNHD2Uqfthr+VMg17bD2bVuk6F07oLGw== dependencies: "@babel/runtime" "^7.12.5" "@types/raf" "^3.4.0" @@ -10248,7 +10265,7 @@ canvg@^3.0.6: regenerator-runtime "^0.13.7" rgbcolor "^1.0.1" stackblur-canvas "^2.0.0" - svg-pathdata "^5.0.5" + svg-pathdata "^6.0.3" capital-case@^1.0.4: version "1.0.4" @@ -11297,9 +11314,9 @@ core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: integrity sha512-GUbtPllXMYRzIgHNZ4dTYTcUemls2cni83Q4Q/TrFONHfhcg9oEGOtaGHfb0cpzec60P96UKPvMkjX1jET8rUw== core-js@^3.6.0, core-js@^3.8.3: - version "3.18.1" - resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.1.tgz#289d4be2ce0085d40fc1244c0b1a54c00454622f" - integrity sha512-vJlUi/7YdlCZeL6fXvWNaLUPh/id12WXj3MbkMw5uOyF0PfWPBNOCNbs53YqgrvtujLNlt9JQpruyIKkUZ+PKA== + version "3.18.3" + resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.3.tgz#86a0bba2d8ec3df860fefcc07a8d119779f01509" + integrity sha512-tReEhtMReZaPFVw7dajMx0vlsz3oOb8ajgPoHVYGxr8ErnZ6PcYEvvmjGmXlfpnxpkYSdOQttjB+MvVbCGfvLw== core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" @@ -25785,10 +25802,10 @@ svg-parser@^2.0.2: resolved "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz#fdc2e29e13951736140b76cb122c8ee6630eb6b5" integrity sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ== -svg-pathdata@^5.0.5: - version "5.0.5" - resolved "https://registry.npmjs.org/svg-pathdata/-/svg-pathdata-5.0.5.tgz#65e8d765642ba15fe15434444087d082bc526b29" - integrity sha512-TAAvLNSE3fEhyl/Da19JWfMAdhSXTYeviXsLSoDT1UM76ADj5ndwAPX1FKQEgB/gFMPavOy6tOqfalXKUiXrow== +svg-pathdata@^6.0.3: + version "6.0.3" + resolved "https://registry.npmjs.org/svg-pathdata/-/svg-pathdata-6.0.3.tgz#80b0e0283b652ccbafb69ad4f8f73e8d3fbf2cac" + integrity sha512-qsjeeq5YjBZ5eMdFuUa4ZosMLxgr5RZ+F+Y1OrDhuOCEInRMA3x74XdBtggJcj9kOeInz0WE+LgCPDkZFlBYJw== svgo@^1.0.0, svgo@^1.2.2: version "1.3.2" From 06c894f7d9691e6f9aa2b4bc4272bdf2993a9612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 14 Oct 2021 09:25:58 +0200 Subject: [PATCH 10/60] fixup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- yarn.lock | 49 ++++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/yarn.lock b/yarn.lock index f2a5002d77..7ba8bdb9ef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7131,10 +7131,10 @@ "@types/node" "*" form-data "^3.0.0" -"@types/node@*", "@types/node@>= 8", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@^15.6.1": - version "15.14.9" - resolved "https://registry.npmjs.org/@types/node/-/node-15.14.9.tgz#bc43c990c3c9be7281868bbc7b8fdd6e2b57adfa" - integrity sha512-qjd88DrCxupx/kJD5yQgZdcYKZKSIGBVDIBE1/LTGcNm3d2Np/jxojkdePDdfnBHJc5W7vSMpbJ1aB7p/Py69A== +"@types/node@*", "@types/node@>= 8", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@^14.0.10", "@types/node@^14.14.31", "@types/node@^14.14.32", "@types/node@^14.14.33": + version "14.17.8" + resolved "https://registry.npmjs.org/@types/node/-/node-14.17.8.tgz#813b73ab7d82ac06ddfd2458b13c88459a3b319f" + integrity sha512-0CHLt50GbUmH/6MrlBIKNdWCglvlyQKkorRf08/0DIi0ryuTPP+ijWLSI19SbDTHSKaagGDELiImY4BSikt61w== "@types/node@10.17.13", "@types/node@^10.1.0", "@types/node@^10.12.0": version "10.17.13" @@ -7146,10 +7146,10 @@ resolved "https://registry.npmjs.org/@types/node/-/node-12.12.58.tgz#46dae9b2b9ee5992818c8f7cee01ff4ce03ab44c" integrity sha512-Be46CNIHWAagEfINOjmriSxuv7IVcqbGe+sDSg2SYCEz/0CRBy7LRASGfRbD8KZkqoePU73Wsx3UvOSFcq/9hA== -"@types/node@^14.0.10", "@types/node@^14.14.31", "@types/node@^14.14.32", "@types/node@^14.14.33": - version "14.17.8" - resolved "https://registry.npmjs.org/@types/node/-/node-14.17.8.tgz#813b73ab7d82ac06ddfd2458b13c88459a3b319f" - integrity sha512-0CHLt50GbUmH/6MrlBIKNdWCglvlyQKkorRf08/0DIi0ryuTPP+ijWLSI19SbDTHSKaagGDELiImY4BSikt61w== +"@types/node@^15.6.1": + version "15.14.9" + resolved "https://registry.npmjs.org/@types/node/-/node-15.14.9.tgz#bc43c990c3c9be7281868bbc7b8fdd6e2b57adfa" + integrity sha512-qjd88DrCxupx/kJD5yQgZdcYKZKSIGBVDIBE1/LTGcNm3d2Np/jxojkdePDdfnBHJc5W7vSMpbJ1aB7p/Py69A== "@types/normalize-package-data@^2.4.0": version "2.4.0" @@ -7373,7 +7373,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@>=16.9.0": +"@types/react@*", "@types/react@>=16.9.0", "@types/react@^16.9": version "16.14.15" resolved "https://registry.npmjs.org/@types/react/-/react-16.14.15.tgz#95d8fa3148050e94bcdc5751447921adbe19f9e6" integrity sha512-jOxlBV9RGZhphdeqJTCv35VZOkjY+XIEY2owwSk84BNDdDv2xS6Csj6fhi+B/q30SR9Tz8lDNt/F2Z5RF3TrRg== @@ -7382,15 +7382,6 @@ "@types/scheduler" "*" csstype "^3.0.2" -"@types/react@^16.9": - version "16.14.17" - resolved "https://registry.npmjs.org/@types/react/-/react-16.14.17.tgz#c57fcfb05efa6423f5b65fcd4a75f63f05b162bf" - integrity sha512-pMLc/7+7SEdQa9A+hN9ujI8blkjFqYAZVqh3iNXqdZ0cQ8TIR502HMkNJniaOGv9SAgc47jxVKoiBJ7c0AakvQ== - dependencies: - "@types/prop-types" "*" - "@types/scheduler" "*" - csstype "^3.0.2" - "@types/recharts@^1.8.14", "@types/recharts@^1.8.15": version "1.8.19" resolved "https://registry.npmjs.org/@types/recharts/-/recharts-1.8.19.tgz#047f72cf4c25df545aa1085fe3a085e58a2483c1" @@ -10254,9 +10245,9 @@ canvas@^2.6.1: simple-get "^3.0.3" canvg@^3.0.6: - version "3.0.9" - resolved "https://registry.npmjs.org/canvg/-/canvg-3.0.9.tgz#9ba095f158b94b97ca2c9c1c40785b11dc08df6d" - integrity sha512-rDXcnRPuz4QHoCilMeoTxql+fvGqNAxp+qV/KHD8rOiJSAfVjFclbdUNHD2Uqfthr+VMg17bD2bVuk6F07oLGw== + version "3.0.8" + resolved "https://registry.npmjs.org/canvg/-/canvg-3.0.8.tgz#9125a4b7033d2f237402241b192587385f4589e6" + integrity sha512-9De5heHfVRgCkln3CGEeSJMviN5U2RyxL4uutYoe8HxI60BjH2XnT2ZUHIp+ZaAZNTUd5Asqfut8WEEdANqfAg== dependencies: "@babel/runtime" "^7.12.5" "@types/raf" "^3.4.0" @@ -10265,7 +10256,7 @@ canvg@^3.0.6: regenerator-runtime "^0.13.7" rgbcolor "^1.0.1" stackblur-canvas "^2.0.0" - svg-pathdata "^6.0.3" + svg-pathdata "^5.0.5" capital-case@^1.0.4: version "1.0.4" @@ -11314,9 +11305,9 @@ core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: integrity sha512-GUbtPllXMYRzIgHNZ4dTYTcUemls2cni83Q4Q/TrFONHfhcg9oEGOtaGHfb0cpzec60P96UKPvMkjX1jET8rUw== core-js@^3.6.0, core-js@^3.8.3: - version "3.18.3" - resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.3.tgz#86a0bba2d8ec3df860fefcc07a8d119779f01509" - integrity sha512-tReEhtMReZaPFVw7dajMx0vlsz3oOb8ajgPoHVYGxr8ErnZ6PcYEvvmjGmXlfpnxpkYSdOQttjB+MvVbCGfvLw== + version "3.18.1" + resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.1.tgz#289d4be2ce0085d40fc1244c0b1a54c00454622f" + integrity sha512-vJlUi/7YdlCZeL6fXvWNaLUPh/id12WXj3MbkMw5uOyF0PfWPBNOCNbs53YqgrvtujLNlt9JQpruyIKkUZ+PKA== core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" @@ -25802,10 +25793,10 @@ svg-parser@^2.0.2: resolved "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz#fdc2e29e13951736140b76cb122c8ee6630eb6b5" integrity sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ== -svg-pathdata@^6.0.3: - version "6.0.3" - resolved "https://registry.npmjs.org/svg-pathdata/-/svg-pathdata-6.0.3.tgz#80b0e0283b652ccbafb69ad4f8f73e8d3fbf2cac" - integrity sha512-qsjeeq5YjBZ5eMdFuUa4ZosMLxgr5RZ+F+Y1OrDhuOCEInRMA3x74XdBtggJcj9kOeInz0WE+LgCPDkZFlBYJw== +svg-pathdata@^5.0.5: + version "5.0.5" + resolved "https://registry.npmjs.org/svg-pathdata/-/svg-pathdata-5.0.5.tgz#65e8d765642ba15fe15434444087d082bc526b29" + integrity sha512-TAAvLNSE3fEhyl/Da19JWfMAdhSXTYeviXsLSoDT1UM76ADj5ndwAPX1FKQEgB/gFMPavOy6tOqfalXKUiXrow== svgo@^1.0.0, svgo@^1.2.2: version "1.3.2" From 88faa29c76ed024c5ff49ff309795a31b43cf3cd Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 14 Oct 2021 11:44:19 +0200 Subject: [PATCH 11/60] chore: force `swagger-client` to `3.16.1` because of removal of dependency Signed-off-by: blam --- plugins/api-docs/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/api-docs/package.json b/plugins/api-docs/package.json index 9a89837102..377df7d13d 100644 --- a/plugins/api-docs/package.json +++ b/plugins/api-docs/package.json @@ -49,7 +49,8 @@ "react-router": "6.0.0-beta.0", "react-router-dom": "6.0.0-beta.0", "react-use": "^17.2.4", - "swagger-ui-react": "^4.0.0-rc.3" + "swagger-ui-react": "^4.0.0-rc.3", + "swagger-client": "3.16.1" }, "devDependencies": { "@backstage/cli": "^0.7.15", From 6ed423c139ae54c7e48d6e1a517173b8ce4a911e Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 14 Oct 2021 11:46:53 +0200 Subject: [PATCH 12/60] chore: added changeset Signed-off-by: blam --- .changeset/modern-carrots-occur.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/modern-carrots-occur.md diff --git a/.changeset/modern-carrots-occur.md b/.changeset/modern-carrots-occur.md new file mode 100644 index 0000000000..b9076a7231 --- /dev/null +++ b/.changeset/modern-carrots-occur.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-api-docs': patch +--- + +lock down the `swagger-client` From b2113723d0192f24014c01c6cc72db3a8e3d8d2b Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 22 Sep 2021 17:47:09 +0200 Subject: [PATCH 13/60] feat(catalog-model): adding `TemplateEntityV1Beta3` as a kind in `catalog-model` Signed-off-by: blam --- .../src/kinds/TemplateEntityV1beta3.test.ts | 160 +++++++++++++++ .../src/kinds/TemplateEntityV1beta3.ts | 43 +++++ .../schema/kinds/Template.v1beta3.schema.json | 182 ++++++++++++++++++ 3 files changed, 385 insertions(+) create mode 100644 packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts create mode 100644 packages/catalog-model/src/kinds/TemplateEntityV1beta3.ts create mode 100644 packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts b/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts new file mode 100644 index 0000000000..092dfd7c7b --- /dev/null +++ b/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts @@ -0,0 +1,160 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + TemplateEntityV1beta3, + templateEntityV1beta3Validator as validator, +} from './TemplateEntityV1beta3'; + +describe('templateEntityV1beta3Validator', () => { + let entity: TemplateEntityV1beta3; + + beforeEach(() => { + entity = { + apiVersion: 'backstage.io/v1beta3', + kind: 'Template', + metadata: { + name: 'test', + }, + spec: { + parameters: { + required: ['storePath', 'owner'], + properties: { + owner: { + type: 'string', + title: 'Owner', + description: 'Who is going to own this component', + }, + storePath: { + type: 'string', + title: 'Store path', + description: 'GitHub store path in org/repo format', + }, + }, + }, + steps: [ + { + id: 'fetch', + name: 'Fetch', + action: 'fetch:plan', + input: { + url: './template', + }, + if: '${{ parameters.owner }}', + }, + ], + output: { + fetchUrl: '${{ steps.fetch.output.targetUrl }}', + }, + owner: 'team-b@example.com', + }, + }; + }); + + it('happy path: accepts valid data', async () => { + await expect(validator.check(entity)).resolves.toBe(true); + }); + + it('ignores unknown apiVersion', async () => { + (entity as any).apiVersion = 'backstage.io/v1beta0'; + await expect(validator.check(entity)).resolves.toBe(false); + }); + + it('ignores unknown kind', async () => { + (entity as any).kind = 'Wizard'; + await expect(validator.check(entity)).resolves.toBe(false); + }); + + it('rejects missing type', async () => { + delete (entity as any).spec.type; + await expect(validator.check(entity)).rejects.toThrow(/type/); + }); + + it('accepts any other type', async () => { + (entity as any).spec.type = 'hallo'; + await expect(validator.check(entity)).resolves.toBe(true); + }); + + it('accepts missing parameters', async () => { + delete (entity as any).spec.parameters; + await expect(validator.check(entity)).resolves.toBe(true); + }); + + it('accepts missing outputs', async () => { + delete (entity as any).spec.outputs; + await expect(validator.check(entity)).resolves.toBe(true); + }); + + it('rejects empty type', async () => { + (entity as any).spec.type = ''; + await expect(validator.check(entity)).rejects.toThrow(/type/); + }); + + it('rejects missing steps', async () => { + delete (entity as any).spec.steps; + await expect(validator.check(entity)).rejects.toThrow(/steps/); + }); + + it('accepts step with missing id', async () => { + delete (entity as any).spec.steps[0].id; + await expect(validator.check(entity)).resolves.toBe(true); + }); + + it('accepts step with missing name', async () => { + delete (entity as any).spec.steps[0].name; + await expect(validator.check(entity)).resolves.toBe(true); + }); + + it('rejects step with missing action', async () => { + delete (entity as any).spec.steps[0].action; + await expect(validator.check(entity)).rejects.toThrow(/action/); + }); + + it('accepts missing owner', async () => { + delete (entity as any).spec.owner; + await expect(validator.check(entity)).resolves.toBe(true); + }); + + it('rejects empty owner', async () => { + (entity as any).spec.owner = ''; + await expect(validator.check(entity)).rejects.toThrow(/owner/); + }); + + it('rejects wrong type owner', async () => { + (entity as any).spec.owner = 5; + await expect(validator.check(entity)).rejects.toThrow(/owner/); + }); + + it('accepts missing if', async () => { + delete (entity as any).spec.steps[0].if; + await expect(validator.check(entity)).resolves.toBe(true); + }); + + it('accepts boolean in if', async () => { + (entity as any).spec.steps[0].if = true; + await expect(validator.check(entity)).resolves.toBe(true); + }); + + it('accepts empty if', async () => { + (entity as any).spec.steps[0].if = ''; + await expect(validator.check(entity)).resolves.toBe(true); + }); + + it('rejects wrong type if', async () => { + (entity as any).spec.steps[0].if = 5; + await expect(validator.check(entity)).rejects.toThrow(/if/); + }); +}); diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1beta3.ts b/packages/catalog-model/src/kinds/TemplateEntityV1beta3.ts new file mode 100644 index 0000000000..2da9a0b6b8 --- /dev/null +++ b/packages/catalog-model/src/kinds/TemplateEntityV1beta3.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject } from '@backstage/config'; +import type { Entity } from '../entity/Entity'; +import schema from '../schema/kinds/Template.v1beta3.schema.json'; +import { ajvCompiledJsonSchemaValidator } from './util'; + +/** @public */ +export interface TemplateEntityV1beta3 extends Entity { + apiVersion: 'backstage.io/v1beta3'; + kind: 'Template'; + spec: { + type: string; + parameters?: JsonObject | JsonObject[]; + steps: Array<{ + id?: string; + name?: string; + action: string; + input?: JsonObject; + if?: string | boolean; + }>; + output?: { [name: string]: string }; + owner?: string; + }; +} + +/** @public */ +export const templateEntityV1beta3Validator = + ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json b/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json new file mode 100644 index 0000000000..236944c9a1 --- /dev/null +++ b/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json @@ -0,0 +1,182 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "TemplateV1beta3", + "description": "A Template describes a scaffolding task for use with the Scaffolder. It describes the required parameters as well as a series of steps that will be taken to execute the scaffolding task.", + "examples": [ + { + "apiVersion": "backstage.io/v1beta3", + "kind": "Template", + "metadata": { + "name": "react-ssr-template", + "title": "React SSR Template", + "description": "Next.js application skeleton for creating isomorphic web applications.", + "tags": ["recommended", "react"] + }, + "spec": { + "owner": "artist-relations-team", + "type": "website", + "parameters": { + "required": ["name", "description"], + "properties": { + "name": { + "title": "Name", + "type": "string", + "description": "Unique name of the component" + }, + "description": { + "title": "Description", + "type": "string", + "description": "Description of the component" + } + } + }, + "steps": [ + { + "id": "fetch", + "name": "Fetch", + "action": "fetch:plain", + "parameters": { + "url": "./template" + } + }, + { + "id": "publish", + "name": "Publish to GitHub", + "action": "publish:github", + "parameters": { + "repoUrl": "{{ parameters.repoUrl }}" + }, + "if": "{{ parameters.repoUrl }}" + } + ], + "output": { + "catalogInfoUrl": "{{ steps.publish.output.catalogInfoUrl }}" + } + } + } + ], + "allOf": [ + { + "$ref": "Entity" + }, + { + "type": "object", + "required": ["spec"], + "properties": { + "apiVersion": { + "enum": ["backstage.io/v1beta3"] + }, + "kind": { + "enum": ["Template"] + }, + "spec": { + "type": "object", + "required": ["type", "steps"], + "properties": { + "type": { + "type": "string", + "description": "The type of component created by the template. The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, a website type component may present tooling in the Backstage interface that is specific to just websites.", + "examples": ["service", "website", "library"], + "minLength": 1 + }, + "parameters": { + "oneOf": [ + { + "type": "object", + "description": "The JSONSchema describing the inputs for the template." + }, + { + "type": "array", + "description": "A list of separate forms to collect parameters.", + "items": { + "type": "object", + "description": "The JSONSchema describing the inputs for the template." + } + } + ] + }, + "steps": { + "type": "array", + "description": "A list of steps to execute.", + "items": { + "type": "object", + "description": "A description of the step to execute.", + "required": ["action"], + "properties": { + "id": { + "type": "string", + "description": "The ID of the step, which can be used to refer to its outputs." + }, + "name": { + "type": "string", + "description": "The name of the step, which will be displayed in the UI during the scaffolding process." + }, + "action": { + "type": "string", + "description": "The name of the action to execute." + }, + "input": { + "type": "object", + "description": "A templated object describing the inputs to the action." + }, + "if": { + "type": ["string", "boolean"], + "description": "A templated condition that skips the step when evaluated to false. If the condition is true or not defined, the step is executed. The condition is true, if the input is not `false`, `undefined`, `null`, `\"\"`, `0`, or `[]`." + } + } + } + }, + "output": { + "type": "object", + "description": "A templated object describing the outputs of the scaffolding task.", + "properties": { + "links": { + "type": "array", + "description": "A list of external hyperlinks, typically pointing to resources created or updated by the template", + "items": { + "type": "object", + "required": [], + "properties": { + "url": { + "type": "string", + "description": "A url in a standard uri format.", + "examples": ["https://github.com/my-org/my-new-repo"], + "minLength": 1 + }, + "entityRef": { + "type": "string", + "description": "An entity reference to an entity in the catalog.", + "examples": ["Component:default/my-app"], + "minLength": 1 + }, + "title": { + "type": "string", + "description": "A user friendly display name for the link.", + "examples": ["View new repo"], + "minLength": 1 + }, + "icon": { + "type": "string", + "description": "A key representing a visual icon to be displayed in the UI.", + "examples": ["dashboard"], + "minLength": 1 + } + } + } + } + }, + "additionalProperties": { + "type": "string" + } + }, + "owner": { + "type": "string", + "description": "The user (or group) owner of the template", + "minLength": 1 + } + } + } + } + } + ] +} From ccea0b8271b3cd054297383b15b11e85ddc4bfcb Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 22 Sep 2021 18:08:21 +0200 Subject: [PATCH 14/60] feat(catalog-backend): add the new kind to the validation workflow Signed-off-by: blam --- packages/catalog-model/src/kinds/index.ts | 2 ++ .../processors/BuiltinKindsEntityProcessor.ts | 7 ++++- .../fixtures/test-v1beta3/template.yaml | 30 +++++++++++++++++++ .../scaffolder/actions/builtin/debug/log.ts | 5 ++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml diff --git a/packages/catalog-model/src/kinds/index.ts b/packages/catalog-model/src/kinds/index.ts index be9f7a0d6a..edac9c505a 100644 --- a/packages/catalog-model/src/kinds/index.ts +++ b/packages/catalog-model/src/kinds/index.ts @@ -52,6 +52,8 @@ export type { } from './SystemEntityV1alpha1'; export { templateEntityV1beta2Validator } from './TemplateEntityV1beta2'; export type { TemplateEntityV1beta2 } from './TemplateEntityV1beta2'; +export { templateEntityV1beta3Validator } from './TemplateEntityV1beta3'; +export type { TemplateEntityV1beta3 } from './TemplateEntityV1beta3'; export type { KindValidator } from './types'; export { userEntityV1alpha1Validator } from './UserEntityV1alpha1'; export type { diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts index bcabc00f32..18a8ad2a27 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts @@ -48,6 +48,8 @@ import { systemEntityV1alpha1Validator, TemplateEntityV1beta2, templateEntityV1beta2Validator, + TemplateEntityV1beta3, + templateEntityV1beta3Validator, UserEntity, userEntityV1alpha1Validator, } from '@backstage/catalog-model'; @@ -61,6 +63,9 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { resourceEntityV1alpha1Validator, groupEntityV1alpha1Validator, locationEntityV1alpha1Validator, + templateEntityV1beta3Validator, + + // TODO: remove once beta3 is stable templateEntityV1beta2Validator, userEntityV1alpha1Validator, systemEntityV1alpha1Validator, @@ -134,7 +139,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { * Emit relations for the Template kind */ if (entity.kind === 'Template') { - const template = entity as TemplateEntityV1beta2; + const template = entity as TemplateEntityV1beta2 | TemplateEntityV1beta3; doEmit( template.spec.owner, { defaultKind: 'Group', defaultNamespace: selfRef.namespace }, diff --git a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml new file mode 100644 index 0000000000..b885357ff9 --- /dev/null +++ b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml @@ -0,0 +1,30 @@ +apiVersion: backstage.io/v1beta3 +kind: Template +metadata: + name: test-v1beta3 + title: Test v1beta3 + description: Test V1 Beta 3 Demo Templates +spec: + type: website + parameters: + - name: Enter some stuff + description: Enter some stuff + properties: + inputString: + type: string + inputObject: + type: object + properties: + first: + type: string + second: + type: number + steps: + - id: debug + name: Debug + action: debug:log + input: + message: ${{ parameters.inputString }} + extra: ${{ parameters.inputObject }} + + diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/log.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/log.ts index 557190dbbc..9bf9b7fb90 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/log.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/log.ts @@ -39,10 +39,15 @@ export function createDebugLogAction() { title: 'List all files in the workspace, if true.', type: 'boolean', }, + extra: { + title: 'Extra info', + }, }, }, }, async handler(ctx) { + ctx.logger.info(JSON.stringify(ctx.input, null, 2)); + if (ctx.input?.message) { ctx.logStream.write(ctx.input.message); } From e3dbdf66797fb332897cd93950e51a610782d84f Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 23 Sep 2021 15:20:43 +0200 Subject: [PATCH 15/60] feat(TaskWorker): split the parsing of the templates with the new syntax Signed-off-by: blam --- .../src/scaffolder/tasks/TaskWorker.ts | 9 ++++++++ .../scaffolder-backend/src/service/router.ts | 22 +++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 819df56109..7577a62d1f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -92,6 +92,15 @@ export class TaskWorker { }; } = { parameters: task.spec.values, steps: {} }; + const { output } = + task.spec.apiVersion === 'backstage.io/v1beta3' + ? await TemplateWorkflowRunner.execute(task) + : await LegacyWorkflowRunner.execute(task); + if (task.spec.apiVersion === 'backstage.io/v1beta3') { + const { output } = await TemplateWorkflowRunnger.execute(task); + } else { + } + for (const step of task.spec.steps) { const metadata = { stepId: step.id }; try { diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index f2a9c5b708..077d605083 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -34,7 +34,11 @@ import { } from '@backstage/backend-common'; import { InputError, NotFoundError } from '@backstage/errors'; import { CatalogApi } from '@backstage/catalog-client'; -import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; +import { + TemplateEntityV1beta2, + Entity, + TemplateEntityV1beta3, +} from '@backstage/catalog-model'; import { ScmIntegrations } from '@backstage/integration'; import { TemplateAction } from '../scaffolder/actions'; import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; @@ -50,10 +54,13 @@ export interface RouterOptions { containerRunner: ContainerRunner; } -function isBeta2Template( - entity: TemplateEntityV1beta2, -): entity is TemplateEntityV1beta2 { - return entity.apiVersion === 'backstage.io/v1beta2'; +function isSupportedTemplate( + entity: TemplateEntityV1beta2 | TemplateEntityV1beta3, +) { + return ( + entity.apiVersion === 'backstage.io/v1beta2' || + entity.apiVersion === 'backstage.io/v1beta3' + ); } export async function createRouter( @@ -128,7 +135,7 @@ export async function createRouter( const template = await entityClient.findTemplate(name, { token: getBearerToken(req.headers.authorization), }); - if (isBeta2Template(template)) { + if (isSupportedTemplate(template)) { const parameters = [template.spec.parameters ?? []].flat(); res.json({ title: template.metadata.title ?? template.metadata.name, @@ -166,7 +173,7 @@ export async function createRouter( let taskSpec; - if (isBeta2Template(template)) { + if (isSupportedTemplate(template)) { for (const parameters of [template.spec.parameters ?? []].flat()) { const result = validate(values, parameters); @@ -179,6 +186,7 @@ export async function createRouter( const baseUrl = getEntityBaseUrl(template); taskSpec = { + apiVersion: template.apiVersion, baseUrl, values, steps: template.spec.steps.map((step, index) => ({ From edb04593ff85eb77c335809ff993a813728321e8 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 24 Sep 2021 14:01:23 +0200 Subject: [PATCH 16/60] feat(TaskWorker): move out the logic into the `WorkflowRunner` implementations Signed-off-by: blam --- .../src/kinds/TemplateEntityV1beta3.test.ts | 6 +- .../schema/kinds/Template.v1beta3.schema.json | 14 +- .../scaffolder/tasks/DefaultWorkflowRunner.ts | 31 ++ .../scaffolder/tasks/LegacyWorkflowRunner.ts | 297 ++++++++++++++++++ .../src/scaffolder/tasks/TaskWorker.ts | 268 +--------------- .../src/scaffolder/tasks/types.ts | 6 + 6 files changed, 355 insertions(+), 267 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts b/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts index 092dfd7c7b..cc275435e8 100644 --- a/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts +++ b/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts @@ -30,6 +30,7 @@ describe('templateEntityV1beta3Validator', () => { name: 'test', }, spec: { + type: 'website', parameters: { required: ['storePath', 'owner'], properties: { @@ -38,11 +39,6 @@ describe('templateEntityV1beta3Validator', () => { title: 'Owner', description: 'Who is going to own this component', }, - storePath: { - type: 'string', - title: 'Store path', - description: 'GitHub store path in org/repo format', - }, }, }, steps: [ diff --git a/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json b/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json index 236944c9a1..e87a436d34 100644 --- a/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json +++ b/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json @@ -14,9 +14,8 @@ }, "spec": { "owner": "artist-relations-team", - "type": "website", "parameters": { - "required": ["name", "description"], + "required": ["name", "description", "repoUrl"], "properties": { "name": { "title": "Name", @@ -27,6 +26,11 @@ "title": "Description", "type": "string", "description": "Description of the component" + }, + "repoUrl": { + "title": "Pick a repository", + "type": "string", + "ui:field": "RepoUrlPicker" } } }, @@ -44,13 +48,13 @@ "name": "Publish to GitHub", "action": "publish:github", "parameters": { - "repoUrl": "{{ parameters.repoUrl }}" + "repoUrl": "${{ parameters.repoUrl }}" }, - "if": "{{ parameters.repoUrl }}" + "if": "${{ parameters.repoUrl }}" } ], "output": { - "catalogInfoUrl": "{{ steps.publish.output.catalogInfoUrl }}" + "catalogInfoUrl": "${{ steps.publish.output.catalogInfoUrl }}" } } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts new file mode 100644 index 0000000000..813fa1b363 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ScmIntegrations } from '@backstage/integration'; +import { TemplateActionRegistry } from '..'; +import { Task, WorkflowResponse, WorkflowRunner } from './types'; + +type Options = { + workingDirectory: string; + actionRegistry: TemplateActionRegistry; + integrations: ScmIntegrations; +}; + +export class DefaultWorkflowRunner implements WorkflowRunner { + constructor(private readonly options: Options) {} + async execute(task: Task): Promise { + throw new Error('Method not implemented.'); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts new file mode 100644 index 0000000000..a5bf740c07 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts @@ -0,0 +1,297 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Task, WorkflowRunner, WorkflowResponse } from './types'; +import * as Handlebars from 'handlebars'; +import { TemplateActionRegistry } from '..'; +import { ScmIntegrations } from '@backstage/integration'; +import { parseRepoUrl } from '../actions/builtin/publish/util'; +import { isTruthy } from './helper'; +import { PassThrough } from 'stream'; +import * as winston from 'winston'; +import { Logger } from 'winston'; +import path from 'path'; +import fs from 'fs-extra'; +import { validate as validateJsonSchema } from 'jsonschema'; +import { JsonObject, JsonValue } from '@backstage/config'; +import { InputError } from '@backstage/errors'; + +type Options = { + workingDirectory: string; + actionRegistry: TemplateActionRegistry; + integrations: ScmIntegrations; + logger: Logger; +}; + +/** + * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced + * with the default workflow runner interface in the future so this entire thing can go bye bye. + */ +export class LegacyWorkflowRunner implements WorkflowRunner { + private readonly handlebars: typeof Handlebars; + + constructor(private readonly options: Options) { + this.handlebars = Handlebars.create(); + + // TODO(blam): this should be a public facing API but it's a little + // scary right now, so we're going to lock it off like the component API is + // in the frontend until we can work out a nice way to do it. + this.handlebars.registerHelper('parseRepoUrl', repoUrl => { + return JSON.stringify(parseRepoUrl(repoUrl, this.options.integrations)); + }); + + this.handlebars.registerHelper('projectSlug', repoUrl => { + const { owner, repo } = parseRepoUrl(repoUrl, this.options.integrations); + return `${owner}/${repo}`; + }); + + this.handlebars.registerHelper('json', obj => JSON.stringify(obj)); + + this.handlebars.registerHelper('not', value => !isTruthy(value)); + + this.handlebars.registerHelper('eq', (a, b) => a === b); + } + + async execute(task: Task): Promise { + const { actionRegistry } = this.options; + + const workspacePath = path.join( + this.options.workingDirectory, + await task.getWorkspaceName(), + ); + try { + await fs.ensureDir(workspacePath); + await task.emitLog( + `Starting up task with ${task.spec.steps.length} steps`, + ); + + const templateCtx: { + parameters: JsonObject; + steps: { + [stepName: string]: { output: { [outputName: string]: JsonValue } }; + }; + } = { parameters: task.spec.values, steps: {} }; + + for (const step of task.spec.steps) { + const metadata = { stepId: step.id }; + try { + const taskLogger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: {}, + }); + + const stream = new PassThrough(); + stream.on('data', async data => { + const message = data.toString().trim(); + if (message?.length > 1) { + await task.emitLog(message, metadata); + } + }); + + taskLogger.add(new winston.transports.Stream({ stream })); + + if (step.if !== undefined) { + // Support passing values like false to disable steps + let skip = !step.if; + + // Evaluate strings as handlebar templates + if (typeof step.if === 'string') { + const condition = JSON.parse( + JSON.stringify(step.if), + (_key, value) => { + if (typeof value === 'string') { + const templated = this.handlebars.compile(value, { + noEscape: true, + data: false, + preventIndent: true, + })(templateCtx); + + // If it's just an empty string, treat it as undefined + if (templated === '') { + return undefined; + } + + try { + return JSON.parse(templated); + } catch { + return templated; + } + } + + return value; + }, + ); + + skip = !isTruthy(condition); + } + + if (skip) { + await task.emitLog(`Skipped step ${step.name}`, { + ...metadata, + status: 'skipped', + }); + continue; + } + } + + await task.emitLog(`Beginning step ${step.name}`, { + ...metadata, + status: 'processing', + }); + + const action = actionRegistry.get(step.action); + if (!action) { + throw new Error(`Action '${step.action}' does not exist`); + } + + const input = + step.input && + JSON.parse(JSON.stringify(step.input), (_key, value) => { + if (typeof value === 'string') { + const templated = this.handlebars.compile(value, { + noEscape: true, + data: false, + preventIndent: true, + })(templateCtx); + + // If it smells like a JSON object then give it a parse as an object and if it fails return the string + if ( + (templated.startsWith('"') && templated.endsWith('"')) || + (templated.startsWith('{') && templated.endsWith('}')) || + (templated.startsWith('[') && templated.endsWith(']')) + ) { + try { + // Don't recursively JSON parse the values of this string. + // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else + return JSON.parse(templated); + } catch { + return templated; + } + } + return templated; + } + + return value; + }); + + if (action.schema?.input) { + const validateResult = validateJsonSchema( + input, + action.schema.input, + ); + if (!validateResult.valid) { + const errors = validateResult.errors.join(', '); + throw new InputError( + `Invalid input passed to action ${action.id}, ${errors}`, + ); + } + } + + const stepOutputs: { [name: string]: JsonValue } = {}; + + // Keep track of all tmp dirs that are created by the action so we can remove them after + const tmpDirs = new Array(); + + this.options.logger.debug(`Running ${action.id} with input`, { + input: JSON.stringify(input, null, 2), + }); + + await action.handler({ + baseUrl: task.spec.baseUrl, + logger: taskLogger, + logStream: stream, + input, + token: task.secrets?.token, + workspacePath, + async createTemporaryDirectory() { + const tmpDir = await fs.mkdtemp( + `${workspacePath}_step-${step.id}-`, + ); + tmpDirs.push(tmpDir); + return tmpDir; + }, + output(name: string, value: JsonValue) { + stepOutputs[name] = value; + }, + }); + + // Remove all temporary directories that were created when executing the action + for (const tmpDir of tmpDirs) { + await fs.remove(tmpDir); + } + + templateCtx.steps[step.id] = { output: stepOutputs }; + + await task.emitLog(`Finished step ${step.name}`, { + ...metadata, + status: 'completed', + }); + } catch (error) { + await task.emitLog(String(error.stack), { + ...metadata, + status: 'failed', + }); + throw error; + } + } + + const output = JSON.parse( + JSON.stringify(task.spec.output), + (_key, value) => { + if (typeof value === 'string') { + const templated = this.handlebars.compile(value, { + noEscape: true, + data: false, + preventIndent: true, + })(templateCtx); + + // If it's just an empty string, treat it as undefined + if (templated === '') { + return undefined; + } + + // If it smells like a JSON object then give it a parse as an object and if it fails return the string + if ( + (templated.startsWith('"') && templated.endsWith('"')) || + (templated.startsWith('{') && templated.endsWith('}')) || + (templated.startsWith('[') && templated.endsWith(']')) + ) { + try { + // Don't recursively JSON parse the values of this string. + // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else + return JSON.parse(templated); + } catch { + return templated; + } + } + return templated; + } + return value; + }, + ); + + return { output }; + } finally { + if (workspacePath) { + await fs.remove(workspacePath); + } + } + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 7577a62d1f..a1af3df2b0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -17,17 +17,17 @@ import { JsonObject, JsonValue } from '@backstage/config'; import { InputError } from '@backstage/errors'; import fs from 'fs-extra'; -import * as Handlebars from 'handlebars'; -import { validate as validateJsonSchema } from 'jsonschema'; + import path from 'path'; -import { PassThrough } from 'stream'; -import * as winston from 'winston'; + import { Logger } from 'winston'; import { parseRepoUrl } from '../actions/builtin/publish/util'; import { TemplateActionRegistry } from '../actions/TemplateActionRegistry'; import { isTruthy } from './helper'; -import { Task, TaskBroker } from './types'; +import { Task, TaskBroker, WorkflowRunner } from './types'; import { ScmIntegrations } from '@backstage/integration'; +import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; type Options = { logger: Logger; @@ -38,28 +38,12 @@ type Options = { }; export class TaskWorker { - private readonly handlebars: typeof Handlebars; + private readonly legacyWorkflowRunner: LegacyWorkflowRunner; + private readonly workflowRunner: WorkflowRunner; constructor(private readonly options: Options) { - this.handlebars = Handlebars.create(); - - // TODO(blam): this should be a public facing API but it's a little - // scary right now, so we're going to lock it off like the component API is - // in the frontend until we can work out a nice way to do it. - this.handlebars.registerHelper('parseRepoUrl', repoUrl => { - return JSON.stringify(parseRepoUrl(repoUrl, options.integrations)); - }); - - this.handlebars.registerHelper('projectSlug', repoUrl => { - const { owner, repo } = parseRepoUrl(repoUrl, options.integrations); - return `${owner}/${repo}`; - }); - - this.handlebars.registerHelper('json', obj => JSON.stringify(obj)); - - this.handlebars.registerHelper('not', value => !isTruthy(value)); - - this.handlebars.registerHelper('eq', (a, b) => a === b); + this.legacyWorkflowRunner = new LegacyWorkflowRunner(options); + this.workflowRunner = new DefaultWorkflowRunner(options); } start() { @@ -72,247 +56,17 @@ export class TaskWorker { } async runOneTask(task: Task) { - let workspacePath: string | undefined = undefined; try { - const { actionRegistry } = this.options; - - workspacePath = path.join( - this.options.workingDirectory, - await task.getWorkspaceName(), - ); - await fs.ensureDir(workspacePath); - await task.emitLog( - `Starting up task with ${task.spec.steps.length} steps`, - ); - - const templateCtx: { - parameters: JsonObject; - steps: { - [stepName: string]: { output: { [outputName: string]: JsonValue } }; - }; - } = { parameters: task.spec.values, steps: {} }; - const { output } = task.spec.apiVersion === 'backstage.io/v1beta3' - ? await TemplateWorkflowRunner.execute(task) - : await LegacyWorkflowRunner.execute(task); - if (task.spec.apiVersion === 'backstage.io/v1beta3') { - const { output } = await TemplateWorkflowRunnger.execute(task); - } else { - } - - for (const step of task.spec.steps) { - const metadata = { stepId: step.id }; - try { - const taskLogger = winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', - format: winston.format.combine( - winston.format.colorize(), - winston.format.timestamp(), - winston.format.simple(), - ), - defaultMeta: {}, - }); - - const stream = new PassThrough(); - stream.on('data', async data => { - const message = data.toString().trim(); - if (message?.length > 1) { - await task.emitLog(message, metadata); - } - }); - - taskLogger.add(new winston.transports.Stream({ stream })); - - if (step.if !== undefined) { - // Support passing values like false to disable steps - let skip = !step.if; - - // Evaluate strings as handlebar templates - if (typeof step.if === 'string') { - const condition = JSON.parse( - JSON.stringify(step.if), - (_key, value) => { - if (typeof value === 'string') { - const templated = this.handlebars.compile(value, { - noEscape: true, - data: false, - preventIndent: true, - })(templateCtx); - - // If it's just an empty string, treat it as undefined - if (templated === '') { - return undefined; - } - - try { - return JSON.parse(templated); - } catch { - return templated; - } - } - - return value; - }, - ); - - skip = !isTruthy(condition); - } - - if (skip) { - await task.emitLog(`Skipped step ${step.name}`, { - ...metadata, - status: 'skipped', - }); - continue; - } - } - - await task.emitLog(`Beginning step ${step.name}`, { - ...metadata, - status: 'processing', - }); - - const action = actionRegistry.get(step.action); - if (!action) { - throw new Error(`Action '${step.action}' does not exist`); - } - - const input = - step.input && - JSON.parse(JSON.stringify(step.input), (_key, value) => { - if (typeof value === 'string') { - const templated = this.handlebars.compile(value, { - noEscape: true, - data: false, - preventIndent: true, - })(templateCtx); - - // If it smells like a JSON object then give it a parse as an object and if it fails return the string - if ( - (templated.startsWith('"') && templated.endsWith('"')) || - (templated.startsWith('{') && templated.endsWith('}')) || - (templated.startsWith('[') && templated.endsWith(']')) - ) { - try { - // Don't recursively JSON parse the values of this string. - // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else - return JSON.parse(templated); - } catch { - return templated; - } - } - return templated; - } - - return value; - }); - - if (action.schema?.input) { - const validateResult = validateJsonSchema( - input, - action.schema.input, - ); - if (!validateResult.valid) { - const errors = validateResult.errors.join(', '); - throw new InputError( - `Invalid input passed to action ${action.id}, ${errors}`, - ); - } - } - - const stepOutputs: { [name: string]: JsonValue } = {}; - - // Keep track of all tmp dirs that are created by the action so we can remove them after - const tmpDirs = new Array(); - - this.options.logger.debug(`Running ${action.id} with input`, { - input: JSON.stringify(input, null, 2), - }); - - await action.handler({ - baseUrl: task.spec.baseUrl, - logger: taskLogger, - logStream: stream, - input, - token: task.secrets?.token, - workspacePath, - async createTemporaryDirectory() { - const tmpDir = await fs.mkdtemp( - `${workspacePath}_step-${step.id}-`, - ); - tmpDirs.push(tmpDir); - return tmpDir; - }, - output(name: string, value: JsonValue) { - stepOutputs[name] = value; - }, - }); - - // Remove all temporary directories that were created when executing the action - for (const tmpDir of tmpDirs) { - await fs.remove(tmpDir); - } - - templateCtx.steps[step.id] = { output: stepOutputs }; - - await task.emitLog(`Finished step ${step.name}`, { - ...metadata, - status: 'completed', - }); - } catch (error) { - await task.emitLog(String(error.stack), { - ...metadata, - status: 'failed', - }); - throw error; - } - } - - const output = JSON.parse( - JSON.stringify(task.spec.output), - (_key, value) => { - if (typeof value === 'string') { - const templated = this.handlebars.compile(value, { - noEscape: true, - data: false, - preventIndent: true, - })(templateCtx); - - // If it's just an empty string, treat it as undefined - if (templated === '') { - return undefined; - } - - // If it smells like a JSON object then give it a parse as an object and if it fails return the string - if ( - (templated.startsWith('"') && templated.endsWith('"')) || - (templated.startsWith('{') && templated.endsWith('}')) || - (templated.startsWith('[') && templated.endsWith(']')) - ) { - try { - // Don't recursively JSON parse the values of this string. - // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else - return JSON.parse(templated); - } catch { - return templated; - } - } - return templated; - } - return value; - }, - ); + ? await this.workflowRunner.execute(task) + : await this.legacyWorkflowRunner.execute(task); await task.complete('completed', { output }); } catch (error) { await task.complete('failed', { error: { name: error.name, message: error.message }, }); - } finally { - if (workspacePath) { - await fs.remove(workspacePath); - } } } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 3b1805f117..5ff40b9459 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -44,6 +44,7 @@ export type DbTaskEventRow = { }; export type TaskSpec = { + apiVersion: 'backstage.io/v1beta2' | 'backstage.io/v1beta3'; baseUrl?: string; values: JsonObject; steps: Array<{ @@ -122,3 +123,8 @@ export interface TaskStore { after, }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; } + +export type WorkflowResponse = { output: { [name: string]: JsonValue } }; +export interface WorkflowRunner { + execute(task: Task): Promise; +} From 4e005b13cfc09a0ca5c227367794ab1664beb500 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 27 Sep 2021 14:26:23 +0200 Subject: [PATCH 17/60] feat: moving all the legacy tests over to the new LegacyWorkflowRunner tests Signed-off-by: blam --- .../tasks/LegacyWorkflowRunner.test.ts | 400 ++++++++++++++++++ 1 file changed, 400 insertions(+) create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts new file mode 100644 index 0000000000..a82f6b25fa --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts @@ -0,0 +1,400 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createTemplateAction, TemplateActionRegistry } from '../actions'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +import os from 'os'; +import { Task, TaskSpec } from './types'; +import { RepoSpec } from '../actions/builtin/publish/util'; + +describe('LegacyWorkflowRunner', () => { + let runner: LegacyWorkflowRunner; + const workingDirectory = os.tmpdir(); + const logger = getVoidLogger(); + let actionRegistry = new TemplateActionRegistry(); + + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'token' }], + }, + }), + ); + + const createMockTaskWithSpec = (spec: TaskSpec): Task => ({ + spec, + complete: async () => {}, + done: false, + emitLog: async () => {}, + getWorkspaceName: () => Promise.resolve('test-workspace'), + }); + + beforeEach(() => { + actionRegistry = new TemplateActionRegistry(); + actionRegistry.register({ + id: 'test-action', + handler: async ctx => { + ctx.output('testOutput', 'mockOutputData'); + ctx.output('badOutput', false); + }, + }); + + runner = new LegacyWorkflowRunner({ + actionRegistry, + integrations, + workingDirectory, + logger, + }); + }); + + it('should fail when the action does not exist', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta2', + steps: [{ id: 'test', name: 'test', action: 'not-found-action' }], + output: { + result: '{{ steps.test.output.testOutput }}', + }, + values: {}, + }); + + await expect(() => runner.execute(task)).rejects.toThrow( + /Template action with ID 'not-found-action' is not registered/, + ); + }); + + describe('templating', () => { + it('should template the output', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta2', + steps: [{ id: 'test', name: 'test', action: 'test-action' }], + output: { + result: '{{ steps.test.output.testOutput }}', + }, + values: {}, + }); + + const { output } = await runner.execute(task); + + expect(output.result).toBe('mockOutputData'); + }); + + it('should template the input', async () => { + const inputAction = createTemplateAction<{ + name: string; + }>({ + id: 'test-input', + schema: { + input: { + type: 'object', + required: ['name'], + properties: { + name: { + title: 'name', + description: 'Enter name', + type: 'string', + }, + }, + }, + }, + async handler(ctx) { + if (ctx.input.name !== 'mockOutputData') { + throw new Error( + `expected name to be "mockOutputData" got ${ctx.input.name}`, + ); + } + }, + }); + actionRegistry.register(inputAction); + + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta2', + steps: [ + { id: 'test', name: 'test', action: 'test-action' }, + { + id: 'test-input', + name: 'test-input', + action: 'test-input', + input: { + name: '{{ steps.test.output.testOutput }}', + }, + }, + ], + output: { + result: '{{ steps.test.output.testOutput }}', + }, + values: {}, + }); + + const { output } = await runner.execute(task); + + expect(output.result).toBe('mockOutputData'); + }); + }); + + describe('conditionals', () => { + it('should execute steps conditionally', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta2', + steps: [ + { id: 'test', name: 'test', action: 'test-action' }, + { + id: 'conditional', + name: 'conditional', + action: 'test-action', + if: '{{ steps.test.output.testOutput }}', + }, + ], + output: { + result: '{{ steps.conditional.output.testOutput }}', + }, + values: {}, + }); + + const { output } = await runner.execute(task); + + expect(output.result).toBe('mockOutputData'); + }); + + it('should execute steps conditionally with eq helper', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta2', + steps: [ + { id: 'test', name: 'test', action: 'test-action' }, + { + id: 'conditional', + name: 'conditional', + action: 'test-action', + if: '{{ eq steps.test.output.testOutput "mockOutputData" }}', + }, + ], + output: { + result: '{{ steps.conditional.output.testOutput }}', + }, + values: {}, + }); + + const { output } = await runner.execute(task); + + expect(output.result).toBe('mockOutputData'); + }); + + it('should skip test conditionally', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta2', + steps: [ + { id: 'test', name: 'test', action: 'test-action' }, + { + id: 'conditional', + name: 'conditional', + action: 'test-action', + if: '{{ steps.test.output.badOutput }}', + }, + ], + output: { + result: '{{ steps.conditional.output.testOutput }}', + }, + values: {}, + }); + + const { output } = await runner.execute(task); + + expect(output.result).toBeUndefined(); + }); + }); + + describe('parsing', () => { + it('should parse strings as objects if possible', async () => { + const inputAction = createTemplateAction<{ + address: { line1: string }; + list: string[]; + address2: string; + }>({ + id: 'test-input', + schema: { + input: { + type: 'object', + required: ['address'], + properties: { + address: { + title: 'address', + description: 'Enter name', + type: 'object', + properties: { + line1: { + type: 'string', + }, + }, + }, + address2: { + type: 'string', + }, + list: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + }, + async handler(ctx) { + if (ctx.input.list.length !== 1) { + throw new Error( + `expected list to have length "1" got ${ctx.input.list.length}`, + ); + } + if (ctx.input.address.line1 !== 'line 1') { + throw new Error( + `expected address.line1 to be "line 1" got ${ctx.input.address.line1}`, + ); + } + + if (ctx.input.address2 !== '{"not valid"}') { + throw new Error( + `expected address2 to be "{"not valid"}" got ${ctx.input.address2}`, + ); + } + ctx.output('address', ctx.input.address.line1); + }, + }); + actionRegistry.register(inputAction); + + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta2', + steps: [ + { + id: 'test-input', + name: 'test-input', + action: 'test-input', + input: { + address: JSON.stringify({ line1: 'line 1' }), + list: JSON.stringify(['hey!']), + address2: '{"not valid"}', + }, + }, + ], + output: { + result: '{{ steps.test-input.output.address }}', + }, + values: {}, + }); + + const { output } = await runner.execute(task); + + expect(output.result).toBe('line 1'); + }); + + it('should provide a parseRepoUrl helper', async () => { + const inputAction = createTemplateAction<{ + destination: RepoSpec; + }>({ + id: 'test-input', + schema: { + input: { + type: 'object', + required: ['destination'], + properties: { + destination: { + title: 'destination', + type: 'object', + properties: { + repo: { + type: 'string', + }, + host: { + type: 'string', + }, + owner: { + type: 'string', + }, + organization: { + type: 'string', + }, + workspace: { + type: 'string', + }, + project: { + type: 'string', + }, + }, + }, + }, + }, + }, + async handler(ctx) { + ctx.output('host', ctx.input.destination.host); + ctx.output('repo', ctx.input.destination.repo); + + if (ctx.input.destination.owner) { + ctx.output('owner', ctx.input.destination.owner); + } + + if (ctx.input.destination.host !== 'github.com') { + throw new Error( + `expected host to be "github.com" got ${ctx.input.destination.host}`, + ); + } + + if (ctx.input.destination.repo !== 'repo') { + throw new Error( + `expected repo to be "repo" got ${ctx.input.destination.repo}`, + ); + } + + if ( + ctx.input.destination.owner && + ctx.input.destination.owner !== 'owner' + ) { + throw new Error( + `expected repo to be "owner" got ${ctx.input.destination.owner}`, + ); + } + }, + }); + actionRegistry.register(inputAction); + + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta2', + steps: [ + { + id: 'test-input', + name: 'test-input', + action: 'test-input', + input: { + destination: '{{ parseRepoUrl parameters.repoUrl }}', + }, + }, + ], + output: { + host: '{{ steps.test-input.output.host }}', + repo: '{{ steps.test-input.output.repo }}', + owner: '{{ steps.test-input.output.owner }}', + }, + values: { + repoUrl: 'github.com?repo=repo&owner=owner', + }, + }); + + const { output } = await runner.execute(task); + + expect(output.host).toBe('github.com'); + expect(output.repo).toBe('repo'); + expect(output.owner).toBe('owner'); + }); + }); +}); From 6ab1202fc8ec684b058b2f3cac548b4986460f37 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 27 Sep 2021 17:09:34 +0200 Subject: [PATCH 18/60] feat: rework the TaskWorker to just be concerned with the task delegation Signed-off-by: blam --- .../src/scaffolder/tasks/TaskWorker.test.ts | 459 +++--------------- .../src/scaffolder/tasks/TaskWorker.ts | 34 +- 2 files changed, 63 insertions(+), 430 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index 2c274df7fc..66c435862a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -23,6 +23,8 @@ import { DatabaseTaskStore } from './DatabaseTaskStore'; import { StorageTaskBroker } from './StorageTaskBroker'; import { TaskWorker } from './TaskWorker'; import { ScmIntegrations } from '@backstage/integration'; +import { WorkflowRunner } from './types'; +import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; async function createStore(): Promise { const manager = DatabaseManager.fromConfig( @@ -40,71 +42,91 @@ async function createStore(): Promise { describe('TaskWorker', () => { let storage: DatabaseTaskStore; - let actionRegistry = new TemplateActionRegistry(); + const workflowRunner: WorkflowRunner = { + execute: jest.fn(), + } as unknown as WorkflowRunner; - const integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [{ host: 'github.com', token: 'token' }], - }, - }), - ); + const legacyWorkflowRunner: LegacyWorkflowRunner = { + execute: jest.fn(), + } as unknown as LegacyWorkflowRunner; beforeAll(async () => { storage = await createStore(); }); beforeEach(() => { - actionRegistry = new TemplateActionRegistry(); - actionRegistry.register({ - id: 'test-action', - handler: async ctx => { - ctx.output('testOutput', 'winning'); - ctx.output('badOutput', false); - }, - }); + jest.resetAllMocks(); }); const logger = getVoidLogger(); - it('should fail when action does not exist', async () => { + it('should call the legacy workflow runner when the apiVersion is not beta3', async () => { const broker = new StorageTaskBroker(storage, logger); const taskWorker = new TaskWorker({ - logger, - workingDirectory: os.tmpdir(), - actionRegistry, taskBroker: broker, - integrations, + runners: { + legacyWorkflowRunner, + workflowRunner, + }, }); - const { taskId } = await broker.dispatch({ + + await broker.dispatch({ + apiVersion: 'backstage.io/v1beta2', steps: [{ id: 'test', name: 'test', action: 'not-found-action' }], output: { result: '{{ steps.test.output.testOutput }}', }, values: {}, }); + const task = await broker.claim(); await taskWorker.runOneTask(task); - const { events } = await storage.listEvents({ taskId }); - const event = events.find(e => e.type === 'completion'); - expect((event?.body?.error as JsonObject)?.message).toBe( - "Template action with ID 'not-found-action' is not registered.", - ); + expect(legacyWorkflowRunner.execute).toHaveBeenCalled(); }); - it('should template output', async () => { + it('should call the default workflow runner when the apiVersion is beta3', async () => { const broker = new StorageTaskBroker(storage, logger); const taskWorker = new TaskWorker({ - logger, - workingDirectory: os.tmpdir(), - actionRegistry, taskBroker: broker, - integrations, + runners: { + legacyWorkflowRunner, + workflowRunner, + }, + }); + + await broker.dispatch({ + apiVersion: 'backstage.io/v1beta3', + steps: [{ id: 'test', name: 'test', action: 'not-found-action' }], + output: { + result: '{{ steps.test.output.testOutput }}', + }, + values: {}, + }); + + const task = await broker.claim(); + await taskWorker.runOneTask(task); + + expect(workflowRunner.execute).toHaveBeenCalled(); + }); + + it('should save the output to the task', async () => { + (workflowRunner.execute as jest.Mock).mockResolvedValue({ + output: { testOutput: 'testmockoutput' }, + }); + + const broker = new StorageTaskBroker(storage, logger); + const taskWorker = new TaskWorker({ + taskBroker: broker, + runners: { + legacyWorkflowRunner, + workflowRunner, + }, }); const { taskId } = await broker.dispatch({ - steps: [{ id: 'test', name: 'test', action: 'test-action' }], + apiVersion: 'backstage.io/v1beta3', + steps: [{ id: 'test', name: 'test', action: 'not-found-action' }], output: { result: '{{ steps.test.output.testOutput }}', }, @@ -116,375 +138,6 @@ describe('TaskWorker', () => { const { events } = await storage.listEvents({ taskId }); const event = events.find(e => e.type === 'completion'); - expect((event?.body?.output as JsonObject).result).toBe('winning'); - }); - - it('should template input', async () => { - const inputAction = createTemplateAction<{ - name: string; - }>({ - id: 'test-input', - schema: { - input: { - type: 'object', - required: ['name'], - properties: { - name: { - title: 'name', - description: 'Enter name', - type: 'string', - }, - }, - }, - }, - async handler(ctx) { - if (ctx.input.name !== 'winning') { - throw new Error( - `expected name to be "winning" got ${ctx.input.name}`, - ); - } - }, - }); - actionRegistry.register(inputAction); - - const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ - logger, - workingDirectory: os.tmpdir(), - actionRegistry, - taskBroker: broker, - integrations, - }); - - const { taskId } = await broker.dispatch({ - steps: [ - { id: 'test', name: 'test', action: 'test-action' }, - { - id: 'test-input', - name: 'test-input', - action: 'test-input', - input: { - name: '{{ steps.test.output.testOutput }}', - }, - }, - ], - output: { - result: '{{ steps.test.output.testOutput }}', - }, - values: {}, - }); - - const task = await broker.claim(); - await taskWorker.runOneTask(task); - - const { events } = await storage.listEvents({ taskId }); - const event = events.find(e => e.type === 'completion'); - expect((event?.body?.output as JsonObject).result).toBe('winning'); - }); - - it('should execute steps conditionally', async () => { - const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ - logger, - workingDirectory: os.tmpdir(), - actionRegistry, - taskBroker: broker, - integrations, - }); - - const { taskId } = await broker.dispatch({ - steps: [ - { id: 'test', name: 'test', action: 'test-action' }, - { - id: 'conditional', - name: 'conditional', - action: 'test-action', - if: '{{ steps.test.output.testOutput }}', - }, - ], - output: { - result: '{{ steps.conditional.output.testOutput }}', - }, - values: {}, - }); - - const task = await broker.claim(); - await taskWorker.runOneTask(task); - - const { events } = await storage.listEvents({ taskId }); - const event = events.find(e => e.type === 'completion'); - expect((event?.body?.output as JsonObject).result).toBe('winning'); - }); - - it('should execute steps conditionally with eq helper', async () => { - const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ - logger, - workingDirectory: os.tmpdir(), - actionRegistry, - taskBroker: broker, - integrations, - }); - - const { taskId } = await broker.dispatch({ - steps: [ - { id: 'test', name: 'test', action: 'test-action' }, - { - id: 'conditional', - name: 'conditional', - action: 'test-action', - if: '{{ eq steps.test.output.testOutput "winning" }}', - }, - ], - output: { - result: '{{ steps.conditional.output.testOutput }}', - }, - values: {}, - }); - - const task = await broker.claim(); - await taskWorker.runOneTask(task); - - const { events } = await storage.listEvents({ taskId }); - const event = events.find(e => e.type === 'completion'); - expect((event?.body?.output as JsonObject).result).toBe('winning'); - }); - - it('should skip steps conditionally', async () => { - const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ - logger, - workingDirectory: os.tmpdir(), - actionRegistry, - taskBroker: broker, - integrations, - }); - - const { taskId } = await broker.dispatch({ - steps: [ - { id: 'test', name: 'test', action: 'test-action' }, - { - id: 'conditional', - name: 'conditional', - action: 'test-action', - if: '{{ steps.test.output.badOutput }}', - }, - ], - output: { - result: '{{ steps.conditional.output.testOutput }}', - }, - values: {}, - }); - - const task = await broker.claim(); - await taskWorker.runOneTask(task); - - const { events } = await storage.listEvents({ taskId }); - const event = events.find(e => e.type === 'completion'); - expect((event?.body?.output as JsonObject).result).toBeUndefined(); - }); - - it('should parse strings as objects if possible', async () => { - const inputAction = createTemplateAction<{ - address: { line1: string }; - list: string[]; - address2: string; - }>({ - id: 'test-input', - schema: { - input: { - type: 'object', - required: ['address'], - properties: { - address: { - title: 'address', - description: 'Enter name', - type: 'object', - properties: { - line1: { - type: 'string', - }, - }, - }, - address2: { - type: 'string', - }, - list: { - type: 'array', - items: { - type: 'string', - }, - }, - }, - }, - }, - async handler(ctx) { - if (ctx.input.list.length !== 1) { - throw new Error( - `expected list to have length "1" got ${ctx.input.list.length}`, - ); - } - if (ctx.input.address.line1 !== 'line 1') { - throw new Error( - `expected address.line1 to be "line 1" got ${ctx.input.address.line1}`, - ); - } - - if (ctx.input.address2 !== '{"not valid"}') { - throw new Error( - `expected address2 to be "{"not valid"}" got ${ctx.input.address2}`, - ); - } - ctx.output('address', ctx.input.address.line1); - }, - }); - actionRegistry.register(inputAction); - - const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ - logger, - workingDirectory: os.tmpdir(), - actionRegistry, - taskBroker: broker, - integrations, - }); - - const { taskId } = await broker.dispatch({ - steps: [ - { - id: 'test-input', - name: 'test-input', - action: 'test-input', - input: { - address: JSON.stringify({ line1: 'line 1' }), - list: JSON.stringify(['hey!']), - address2: '{"not valid"}', - }, - }, - ], - output: { - result: '{{ steps.test-input.output.address }}', - }, - values: {}, - }); - - const task = await broker.claim(); - await taskWorker.runOneTask(task); - - const { events } = await storage.listEvents({ taskId }); - const event = events.find(e => e.type === 'completion'); - - expect((event?.body?.output as JsonObject).result).toBe('line 1'); - }); - - // TODO(blam): Can delete this test when we make the helpers a public API - it('should provide a repoUrlParse helper for the templates', async () => { - const inputAction = createTemplateAction<{ - destination: RepoSpec; - }>({ - id: 'test-input', - schema: { - input: { - type: 'object', - required: ['destination'], - properties: { - destination: { - title: 'destination', - type: 'object', - properties: { - repo: { - type: 'string', - }, - host: { - type: 'string', - }, - owner: { - type: 'string', - }, - organization: { - type: 'string', - }, - workspace: { - type: 'string', - }, - project: { - type: 'string', - }, - }, - }, - }, - }, - }, - async handler(ctx) { - ctx.output('host', ctx.input.destination.host); - ctx.output('repo', ctx.input.destination.repo); - - if (ctx.input.destination.owner) { - ctx.output('owner', ctx.input.destination.owner); - } - - if (ctx.input.destination.host !== 'github.com') { - throw new Error( - `expected host to be "github.com" got ${ctx.input.destination.host}`, - ); - } - - if (ctx.input.destination.repo !== 'repo') { - throw new Error( - `expected repo to be "repo" got ${ctx.input.destination.repo}`, - ); - } - - if ( - ctx.input.destination.owner && - ctx.input.destination.owner !== 'owner' - ) { - throw new Error( - `expected repo to be "owner" got ${ctx.input.destination.owner}`, - ); - } - }, - }); - actionRegistry.register(inputAction); - - const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ - logger, - workingDirectory: os.tmpdir(), - actionRegistry, - taskBroker: broker, - integrations, - }); - - const { taskId } = await broker.dispatch({ - steps: [ - { - id: 'test-input', - name: 'test-input', - action: 'test-input', - input: { - destination: '{{ parseRepoUrl parameters.repoUrl }}', - }, - }, - ], - output: { - host: '{{ steps.test-input.output.host }}', - repo: '{{ steps.test-input.output.repo }}', - owner: '{{ steps.test-input.output.owner }}', - }, - values: { - repoUrl: 'github.com?repo=repo&owner=owner', - }, - }); - - const task = await broker.claim(); - await taskWorker.runOneTask(task); - - const { events } = await storage.listEvents({ taskId }); - const event = events.find(e => e.type === 'completion'); - - expect((event?.body?.output as JsonObject).host).toBe('github.com'); - expect((event?.body?.output as JsonObject).repo).toBe('repo'); - expect((event?.body?.output as JsonObject).owner).toBe('owner'); + expect(event?.body.output).toEqual({ testOutput: 'testmockoutput' }); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index a1af3df2b0..5b69ecc6df 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -13,39 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import { JsonObject, JsonValue } from '@backstage/config'; -import { InputError } from '@backstage/errors'; -import fs from 'fs-extra'; - -import path from 'path'; - -import { Logger } from 'winston'; -import { parseRepoUrl } from '../actions/builtin/publish/util'; -import { TemplateActionRegistry } from '../actions/TemplateActionRegistry'; -import { isTruthy } from './helper'; import { Task, TaskBroker, WorkflowRunner } from './types'; -import { ScmIntegrations } from '@backstage/integration'; import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; -import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; type Options = { - logger: Logger; taskBroker: TaskBroker; - workingDirectory: string; - actionRegistry: TemplateActionRegistry; - integrations: ScmIntegrations; + runners: { + legacyWorkflowRunner: LegacyWorkflowRunner; + workflowRunner: WorkflowRunner; + }; }; export class TaskWorker { - private readonly legacyWorkflowRunner: LegacyWorkflowRunner; - private readonly workflowRunner: WorkflowRunner; - - constructor(private readonly options: Options) { - this.legacyWorkflowRunner = new LegacyWorkflowRunner(options); - this.workflowRunner = new DefaultWorkflowRunner(options); - } - + constructor(private readonly options: Options) {} start() { (async () => { for (;;) { @@ -59,8 +39,8 @@ export class TaskWorker { try { const { output } = task.spec.apiVersion === 'backstage.io/v1beta3' - ? await this.workflowRunner.execute(task) - : await this.legacyWorkflowRunner.execute(task); + ? await this.options.runners.workflowRunner.execute(task) + : await this.options.runners.legacyWorkflowRunner.execute(task); await task.complete('completed', { output }); } catch (error) { From 34b1278929753161cfae704843016141c9523d02 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 02:56:37 +0200 Subject: [PATCH 19/60] chore: worked out a niceish way to template objects and stuff as first class citizens Signed-off-by: blam --- .../tasks/DefaultWorkflowRunner.test.ts | 193 ++++++++++++++++++ .../scaffolder/tasks/DefaultWorkflowRunner.ts | 151 +++++++++++++- .../scaffolder/tasks/LegacyWorkflowRunner.ts | 12 +- .../src/scaffolder/tasks/types.ts | 23 ++- 4 files changed, 372 insertions(+), 7 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts new file mode 100644 index 0000000000..e508d619c4 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -0,0 +1,193 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import os from 'os'; +import { getVoidLogger } from '@backstage/backend-common'; +import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; +import { TemplateActionRegistry } from '../actions'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { Task, TaskSpec } from './types'; + +describe('DefaultWorkflowRunner', () => { + const workingDirectory = os.tmpdir(); + const logger = getVoidLogger(); + let actionRegistry = new TemplateActionRegistry(); + let runner: DefaultWorkflowRunner; + let fakeActionHandler: jest.Mock; + + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'token' }], + }, + }), + ); + + const createMockTaskWithSpec = (spec: TaskSpec): Task => ({ + spec, + complete: async () => {}, + done: false, + emitLog: async () => {}, + getWorkspaceName: () => Promise.resolve('test-workspace'), + }); + + beforeEach(() => { + jest.resetAllMocks(); + actionRegistry = new TemplateActionRegistry(); + fakeActionHandler = jest.fn(); + + actionRegistry.register({ + id: 'jest-mock-action', + description: 'Mock action for testing', + handler: fakeActionHandler, + }); + + runner = new DefaultWorkflowRunner({ + actionRegistry, + integrations, + workingDirectory, + logger, + }); + }); + + it('should throw an error if the action does not exist', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + parameters: {}, + output: {}, + steps: [{ id: 'test', name: 'name', action: 'does-not-exist' }], + }); + + await expect(runner.execute(task)).rejects.toThrowError( + "Template action with ID 'does-not-exist' is not registered.", + ); + }); + + describe('validation', () => {}); + describe('running', () => {}); + + describe('templating', () => { + it('should template the input to an action', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'jest-mock-action', + input: { + foo: '${{parameters.input | lower }}', + }, + }, + ], + output: {}, + parameters: { + input: 'BACKSTAGE', + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { foo: 'backstage' } }), + ); + }); + + it('should template complex values into the action', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'jest-mock-action', + input: { + foo: '${{parameters.complex}}', + }, + }, + ], + output: {}, + parameters: { + complex: { bar: 'BACKSTAGE' }, + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { foo: { bar: 'BACKSTAGE' } } }), + ); + }); + + it('supports really complex structures', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'jest-mock-action', + input: { + foo: '${{parameters.complex.baz.something}}', + }, + }, + ], + output: {}, + parameters: { + complex: { + bar: 'BACKSTAGE', + baz: { something: 'nested', here: 'yas' }, + }, + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { foo: 'nested' } }), + ); + }); + + it('supports numbers as first class too', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'jest-mock-action', + input: { + foo: '${{parameters.complex.baz.number}}', + }, + }, + ], + output: {}, + parameters: { + complex: { + bar: 'BACKSTAGE', + baz: { number: 1 }, + }, + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { foo: 1 } }), + ); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 813fa1b363..e84d7a71e6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -15,17 +15,162 @@ */ import { ScmIntegrations } from '@backstage/integration'; import { TemplateActionRegistry } from '..'; -import { Task, WorkflowResponse, WorkflowRunner } from './types'; +import { + Task, + TaskSpec, + TaskSpecV1beta3, + TaskStep, + WorkflowResponse, + WorkflowRunner, +} from './types'; +import * as winston from 'winston'; +import nunjucks from 'nunjucks'; +import fs from 'fs-extra'; +import path from 'path'; +import { JsonObject, JsonValue } from '@backstage/config'; +import { InputError } from '@backstage/errors'; +import { PassThrough } from 'stream'; type Options = { workingDirectory: string; actionRegistry: TemplateActionRegistry; integrations: ScmIntegrations; + logger: winston.Logger; +}; + +type TemplateContext = { + parameters: JsonObject; + steps: { + [stepName: string]: { output: { [outputName: string]: JsonValue } }; + }; +}; + +const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta3 => { + return taskSpec.apiVersion === 'backstage.io/v1beta3'; +}; + +const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { + const metadata = { stepId: step.id }; + const taskLogger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: {}, + }); + + const streamLogger = new PassThrough(); + streamLogger.on('data', async data => { + const message = data.toString().trim(); + if (message?.length > 1) { + await task.emitLog(message, metadata); + } + }); + + taskLogger.add(new winston.transports.Stream({ stream: streamLogger })); + + return { taskLogger, streamLogger }; }; export class DefaultWorkflowRunner implements WorkflowRunner { - constructor(private readonly options: Options) {} + private readonly nunjucks: nunjucks.Environment; + + constructor(private readonly options: Options) { + this.nunjucks = nunjucks.configure({ + autoescape: false, + tags: { + variableStart: '${{', + variableEnd: '}}', + }, + }); + } + async execute(task: Task): Promise { - throw new Error('Method not implemented.'); + if (!isValidTaskSpec(task.spec)) { + throw new InputError( + 'Wrong template version executed with the workflow engine', + ); + } + const workspacePath = path.join( + this.options.workingDirectory, + await task.getWorkspaceName(), + ); + try { + await fs.ensureDir(workspacePath); + await task.emitLog( + `Starting up task with ${task.spec.steps.length} steps`, + ); + + /** + * This is a little bit of a hack / magic so that when we use nunjucks and we try to + * pass through an object from the `parameters` section of the task spec, it will + * actually work as the toString method is called from the nunjucks template. + */ + const parsedParams = JSON.parse( + JSON.stringify(task.spec.parameters), + (key: string, value: JsonObject) => { + if (typeof value === 'object' && key) { + value.toString = () => JSON.stringify(value); + } + + return value; + }, + ); + + const context: TemplateContext = { + parameters: parsedParams, + steps: {}, + }; + + for (const step of task.spec.steps) { + const action = this.options.actionRegistry.get(step.action); + const { taskLogger, streamLogger } = createStepLogger({ task, step }); + + const input = + step.input && + JSON.parse(JSON.stringify(step.input), (_key, value) => { + try { + if (typeof value === 'string') { + const templated = this.nunjucks.renderString(value, context); + try { + return JSON.parse(templated); + } catch { + return templated; + } + } + } catch { + return value; + } + return value; + }); + + const tmpDirs = new Array(); + const stepOutputs: { [outputName: string]: JsonValue } = {}; + + await action.handler({ + baseUrl: task.spec.baseUrl, + input, + logger: taskLogger, + logStream: streamLogger, + workspacePath, + createTemporaryDirectory: async () => { + const tmpDir = await fs.mkdtemp( + `${workspacePath}_step-${step.id}-`, + ); + tmpDirs.push(tmpDir); + return tmpDir; + }, + output(name: string, value: JsonValue) { + stepOutputs[name] = value; + }, + }); + } + } finally { + if (workspacePath) { + await fs.remove(workspacePath); + } + } } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts index a5bf740c07..a3c98f6cd4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts @@ -13,7 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Task, WorkflowRunner, WorkflowResponse } from './types'; +import { + Task, + WorkflowRunner, + WorkflowResponse, + TaskSpecV1beta2, +} from './types'; import * as Handlebars from 'handlebars'; import { TemplateActionRegistry } from '..'; import { ScmIntegrations } from '@backstage/integration'; @@ -35,6 +40,8 @@ type Options = { logger: Logger; }; +const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 => + taskSpec.apiVersion === 'backstage.io/v1beta2'; /** * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced * with the default workflow runner interface in the future so this entire thing can go bye bye. @@ -65,6 +72,9 @@ export class LegacyWorkflowRunner implements WorkflowRunner { } async execute(task: Task): Promise { + if (!isValidTaskSpec(task.spec)) { + throw new InputError(`Task spec is not a valid v1beta2 task spec`); + } const { actionRegistry } = this.options; const workspacePath = path.join( diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 5ff40b9459..2ab38edec9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -43,8 +43,8 @@ export type DbTaskEventRow = { createdAt: string; }; -export type TaskSpec = { - apiVersion: 'backstage.io/v1beta2' | 'backstage.io/v1beta3'; +export interface TaskSpecV1beta2 { + apiVersion: 'backstage.io/v1beta2'; baseUrl?: string; values: JsonObject; steps: Array<{ @@ -55,7 +55,24 @@ export type TaskSpec = { if?: string | boolean; }>; output: { [name: string]: string }; -}; +} + +export interface TaskStep { + id: string; + name: string; + action: string; + input?: JsonObject; + if?: string | boolean; +} +export interface TaskSpecV1beta3 { + apiVersion: 'backstage.io/v1beta3'; + baseUrl?: string; + parameters: JsonObject; + steps: TaskStep[]; + output: { [name: string]: string }; +} + +export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; export type TaskSecrets = { token: string | undefined; From 1ec5ef3699ec198f4089abb836cc34d9bbd254ba Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 03:43:26 +0200 Subject: [PATCH 20/60] chore: acutally make the scaffolder backend run the v3 template Signed-off-by: blam --- .../fixtures/test-v1beta3/template.yaml | 4 + .../tasks/DefaultWorkflowRunner.test.ts | 30 ++++ .../scaffolder/tasks/DefaultWorkflowRunner.ts | 136 +++++++++++------- .../scaffolder/tasks/LegacyWorkflowRunner.ts | 2 + .../src/scaffolder/tasks/TaskWorker.test.ts | 4 +- .../scaffolder-backend/src/service/router.ts | 62 +++++--- 6 files changed, 166 insertions(+), 72 deletions(-) diff --git a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml index b885357ff9..007e23a3d6 100644 --- a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml +++ b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml @@ -12,13 +12,17 @@ spec: properties: inputString: type: string + title: string input test inputObject: type: object + title: object input test properties: first: type: string + title: first second: type: number + title: second steps: - id: debug name: Debug diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index e508d619c4..dea94e9b52 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -55,6 +55,14 @@ describe('DefaultWorkflowRunner', () => { handler: fakeActionHandler, }); + actionRegistry.register({ + id: 'output-action', + description: 'Mock action for testing', + handler: async ctx => { + ctx.output('mock', 'backstage'); + }, + }); + runner = new DefaultWorkflowRunner({ actionRegistry, integrations, @@ -189,5 +197,27 @@ describe('DefaultWorkflowRunner', () => { expect.objectContaining({ input: { foo: 1 } }), ); }); + + it('should template the output from simple actions', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'output-action', + input: {}, + }, + ], + output: { + foo: '${{steps.test.output.mock | upper}}', + }, + parameters: {}, + }); + + const { output } = await runner.execute(task); + + expect(output.foo).toEqual('BACKSTAGE'); + }); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index e84d7a71e6..c8f77f066a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -87,6 +87,43 @@ export class DefaultWorkflowRunner implements WorkflowRunner { }); } + private render(input: T, context: TemplateContext): T { + return JSON.parse(JSON.stringify(input), (_key, value) => { + try { + if (typeof value === 'string') { + const templated = this.nunjucks.renderString(value, context); + try { + return JSON.parse(templated); + } catch { + return templated; + } + } + } catch { + return value; + } + return value; + }); + } + + private makeStringifyableParams(input: T): T { + /** + * This is a little bit of a hack / magic so that when we use nunjucks and we try to + * pass through something other than a string from the parameters section. + * When an accessor is used that is an object, it's toString is the JSON.stringify'd version of it's children + * Which makes it work really well in string templating as we can parse the result again after.yarn + */ + return JSON.parse( + JSON.stringify(input), + (key: string, value: JsonObject) => { + if (typeof value === 'object' && key) { + value.toString = () => JSON.stringify(value); + } + + return value; + }, + ); + } + async execute(task: Task): Promise { if (!isValidTaskSpec(task.spec)) { throw new InputError( @@ -103,70 +140,61 @@ export class DefaultWorkflowRunner implements WorkflowRunner { `Starting up task with ${task.spec.steps.length} steps`, ); - /** - * This is a little bit of a hack / magic so that when we use nunjucks and we try to - * pass through an object from the `parameters` section of the task spec, it will - * actually work as the toString method is called from the nunjucks template. - */ - const parsedParams = JSON.parse( - JSON.stringify(task.spec.parameters), - (key: string, value: JsonObject) => { - if (typeof value === 'object' && key) { - value.toString = () => JSON.stringify(value); - } - - return value; - }, - ); - const context: TemplateContext = { - parameters: parsedParams, + parameters: this.makeStringifyableParams(task.spec.parameters), steps: {}, }; for (const step of task.spec.steps) { - const action = this.options.actionRegistry.get(step.action); - const { taskLogger, streamLogger } = createStepLogger({ task, step }); + try { + const action = this.options.actionRegistry.get(step.action); + const { taskLogger, streamLogger } = createStepLogger({ task, step }); - const input = - step.input && - JSON.parse(JSON.stringify(step.input), (_key, value) => { - try { - if (typeof value === 'string') { - const templated = this.nunjucks.renderString(value, context); - try { - return JSON.parse(templated); - } catch { - return templated; - } - } - } catch { - return value; - } - return value; + const input = step.input && this.render(step.input, context); + + const tmpDirs = new Array(); + const stepOutput: { [outputName: string]: JsonValue } = {}; + + await task.emitLog(`Beginning step ${step.name}`, { + stepId: step.id, + status: 'processing', }); - const tmpDirs = new Array(); - const stepOutputs: { [outputName: string]: JsonValue } = {}; + await action.handler({ + baseUrl: task.spec.baseUrl, + input, + logger: taskLogger, + logStream: streamLogger, + workspacePath, + createTemporaryDirectory: async () => { + const tmpDir = await fs.mkdtemp( + `${workspacePath}_step-${step.id}-`, + ); + tmpDirs.push(tmpDir); + return tmpDir; + }, + output(name: string, value: JsonValue) { + stepOutput[name] = value; + }, + }); - await action.handler({ - baseUrl: task.spec.baseUrl, - input, - logger: taskLogger, - logStream: streamLogger, - workspacePath, - createTemporaryDirectory: async () => { - const tmpDir = await fs.mkdtemp( - `${workspacePath}_step-${step.id}-`, - ); - tmpDirs.push(tmpDir); - return tmpDir; - }, - output(name: string, value: JsonValue) { - stepOutputs[name] = value; - }, - }); + context.steps[step.id] = { output: stepOutput }; + + await task.emitLog(`Finished step ${step.name}`, { + stepId: step.id, + status: 'completed', + }); + } catch (err) { + await task.emitLog(String(err.stack), { + stepId: step.id, + status: 'failed', + }); + } } + + const output = this.render(task.spec.output, context); + + return { output }; } finally { if (workspacePath) { await fs.remove(workspacePath); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts index a3c98f6cd4..97e3a4dec4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts @@ -18,6 +18,7 @@ import { WorkflowRunner, WorkflowResponse, TaskSpecV1beta2, + TaskSpec, } from './types'; import * as Handlebars from 'handlebars'; import { TemplateActionRegistry } from '..'; @@ -75,6 +76,7 @@ export class LegacyWorkflowRunner implements WorkflowRunner { if (!isValidTaskSpec(task.spec)) { throw new InputError(`Task spec is not a valid v1beta2 task spec`); } + const { actionRegistry } = this.options; const workspacePath = path.join( diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index 66c435862a..456836046c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -101,7 +101,7 @@ describe('TaskWorker', () => { output: { result: '{{ steps.test.output.testOutput }}', }, - values: {}, + parameters: {}, }); const task = await broker.claim(); @@ -130,7 +130,7 @@ describe('TaskWorker', () => { output: { result: '{{ steps.test.output.testOutput }}', }, - values: {}, + parameters: {}, }); const task = await broker.claim(); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 077d605083..b75a5d0368 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -42,6 +42,8 @@ import { import { ScmIntegrations } from '@backstage/integration'; import { TemplateAction } from '../scaffolder/actions'; import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; +import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; +import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; export interface RouterOptions { logger: Logger; @@ -90,14 +92,28 @@ export async function createRouter( ); const taskBroker = new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); + const legacyWorkflowRunner = new LegacyWorkflowRunner({ + logger, + actionRegistry, + integrations, + workingDirectory, + }); + + const workflowRunner = new DefaultWorkflowRunner({ + actionRegistry, + integrations, + logger, + workingDirectory, + }); + const workers = []; for (let i = 0; i < (taskWorkers || 1); i++) { const worker = new TaskWorker({ - logger, taskBroker, - actionRegistry, - workingDirectory, - integrations, + runners: { + legacyWorkflowRunner, + workflowRunner, + }, }); workers.push(worker); } @@ -184,18 +200,32 @@ export async function createRouter( } const baseUrl = getEntityBaseUrl(template); - - taskSpec = { - apiVersion: template.apiVersion, - baseUrl, - values, - steps: template.spec.steps.map((step, index) => ({ - ...step, - id: step.id ?? `step-${index + 1}`, - name: step.name ?? step.action, - })), - output: template.spec.output ?? {}, - }; + // TODO: need to make sure that the TaskSpec is the right format here. + // If it's beta2 use values, beta3 uses parameters to clear that up. + taskSpec = + template.apiVersion === 'backstage.io/v1beta2' + ? { + apiVersion: template.apiVersion, + baseUrl, + values, + steps: template.spec.steps.map((step, index) => ({ + ...step, + id: step.id ?? `step-${index + 1}`, + name: step.name ?? step.action, + })), + output: template.spec.output ?? {}, + } + : { + apiVersion: template.apiVersion, + baseUrl, + parameters: values, + steps: template.spec.steps.map((step, index) => ({ + ...step, + id: step.id ?? `step-${index + 1}`, + name: step.name ?? step.action, + })), + output: template.spec.output ?? {}, + }; } else { throw new InputError( `Unsupported apiVersion field in schema entity, ${ From c7b089a5aeb2817e91543dee1db571fe7e66d7d3 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 04:10:33 +0200 Subject: [PATCH 21/60] chore: adding the new if syntax and using the power of nunjucks Signed-off-by: blam --- .../tasks/DefaultWorkflowRunner.test.ts | 72 ++++++++++++++++++- .../scaffolder/tasks/DefaultWorkflowRunner.ts | 15 ++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index dea94e9b52..c249606658 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -60,6 +60,7 @@ describe('DefaultWorkflowRunner', () => { description: 'Mock action for testing', handler: async ctx => { ctx.output('mock', 'backstage'); + ctx.output('shouldRun', true); }, }); @@ -85,7 +86,76 @@ describe('DefaultWorkflowRunner', () => { }); describe('validation', () => {}); - describe('running', () => {}); + describe('conditionals', () => { + it('should execute steps conditionally', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { id: 'test', name: 'test', action: 'output-action' }, + { + id: 'conditional', + name: 'conditional', + action: 'output-action', + if: '${{ steps.test.output.shouldRun }}', + }, + ], + output: { + result: '${{ steps.conditional.output.mock }}', + }, + parameters: {}, + }); + + const { output } = await runner.execute(task); + + expect(output.result).toBe('backstage'); + }); + + it('should skips steps conditionally', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { id: 'test', name: 'test', action: 'output-action' }, + { + id: 'conditional', + name: 'conditional', + action: 'output-action', + if: '${{ not steps.test.output.shouldRun}}', + }, + ], + output: { + result: '${{ steps.conditional.output.mock }}', + }, + parameters: {}, + }); + + const { output } = await runner.execute(task); + + expect(output.result).toBeUndefined(); + }); + + it('should skips steps using the negating equals operator', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { id: 'test', name: 'test', action: 'output-action' }, + { + id: 'conditional', + name: 'conditional', + action: 'output-action', + if: '${{ steps.test.output.mock !== "backstage"}}', + }, + ], + output: { + result: '${{ steps.conditional.output.mock }}', + }, + parameters: {}, + }); + + const { output } = await runner.execute(task); + + expect(output.result).toBeUndefined(); + }); + }); describe('templating', () => { it('should template the input to an action', async () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index c8f77f066a..889048d04a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -30,6 +30,7 @@ import path from 'path'; import { JsonObject, JsonValue } from '@backstage/config'; import { InputError } from '@backstage/errors'; import { PassThrough } from 'stream'; +import { isTruthy } from './helper'; type Options = { workingDirectory: string; @@ -92,6 +93,9 @@ export class DefaultWorkflowRunner implements WorkflowRunner { try { if (typeof value === 'string') { const templated = this.nunjucks.renderString(value, context); + if (templated === '') { + return undefined; + } try { return JSON.parse(templated); } catch { @@ -147,6 +151,16 @@ export class DefaultWorkflowRunner implements WorkflowRunner { for (const step of task.spec.steps) { try { + if (step.if) { + const ifResult = await this.render(step.if, context); + if (!isTruthy(ifResult)) { + await task.emitLog( + `Skipping step ${step.id} because it's if condition was false`, + ); + continue; + } + } + const action = this.options.actionRegistry.get(step.action); const { taskLogger, streamLogger } = createStepLogger({ task, step }); @@ -189,6 +203,7 @@ export class DefaultWorkflowRunner implements WorkflowRunner { stepId: step.id, status: 'failed', }); + throw err; } } From fe2324e63007b50e5be1c885c9de406c6df8415a Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 04:29:19 +0200 Subject: [PATCH 22/60] chore: added validation for schemas for actions Signed-off-by: blam Signed-off-by: blam --- .../tasks/DefaultWorkflowRunner.test.ts | 52 ++++++++++++++++++- .../scaffolder/tasks/DefaultWorkflowRunner.ts | 18 ++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index c249606658..6447782654 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -55,6 +55,23 @@ describe('DefaultWorkflowRunner', () => { handler: fakeActionHandler, }); + actionRegistry.register({ + id: 'jest-validated-action', + description: 'Mock action for testing', + handler: fakeActionHandler, + schema: { + input: { + type: 'object', + required: ['foo'], + properties: { + foo: { + type: 'number', + }, + }, + }, + }, + }); + actionRegistry.register({ id: 'output-action', description: 'Mock action for testing', @@ -85,7 +102,40 @@ describe('DefaultWorkflowRunner', () => { ); }); - describe('validation', () => {}); + describe('validation', () => { + it('should throw an error if the action has a schema and the input does not match', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + parameters: {}, + output: {}, + steps: [{ id: 'test', name: 'name', action: 'jest-validated-action' }], + }); + + await expect(runner.execute(task)).rejects.toThrowError( + /Invalid input passed to action jest-validated-action, instance requires property \"foo\"/, + ); + }); + + it('should run the action when the validation passes', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + parameters: {}, + output: {}, + steps: [ + { + id: 'test', + name: 'name', + action: 'jest-validated-action', + input: { foo: 1 }, + }, + ], + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledTimes(1); + }); + }); describe('conditionals', () => { it('should execute steps conditionally', async () => { const task = createMockTaskWithSpec({ diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 889048d04a..c08363857d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -31,6 +31,7 @@ import { JsonObject, JsonValue } from '@backstage/config'; import { InputError } from '@backstage/errors'; import { PassThrough } from 'stream'; import { isTruthy } from './helper'; +import { validate as validateJsonSchema } from 'jsonschema'; type Options = { workingDirectory: string; @@ -79,6 +80,8 @@ export class DefaultWorkflowRunner implements WorkflowRunner { private readonly nunjucks: nunjucks.Environment; constructor(private readonly options: Options) { + // TODO(blam): Probably need the repo helper here. + // Or we move to returning Objects in the RepoUrlPickerV2 or something? this.nunjucks = nunjucks.configure({ autoescape: false, tags: { @@ -164,7 +167,20 @@ export class DefaultWorkflowRunner implements WorkflowRunner { const action = this.options.actionRegistry.get(step.action); const { taskLogger, streamLogger } = createStepLogger({ task, step }); - const input = step.input && this.render(step.input, context); + const input = (step.input && this.render(step.input, context)) ?? {}; + + if (action.schema?.input) { + const validateResult = validateJsonSchema( + input, + action.schema.input, + ); + if (!validateResult.valid) { + const errors = validateResult.errors.join(', '); + throw new InputError( + `Invalid input passed to action ${action.id}, ${errors}`, + ); + } + } const tmpDirs = new Array(); const stepOutput: { [outputName: string]: JsonValue } = {}; From 154b86e1da0bb1eb2f4773f32c4c4eec9c430f97 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 04:45:09 +0200 Subject: [PATCH 23/60] feat: think we're at feature parity Signed-off-by: blam --- .../fixtures/test-v1beta3/template.yaml | 47 +++++++++---------- .../tasks/DefaultWorkflowRunner.test.ts | 29 ++++++++++++ .../scaffolder/tasks/DefaultWorkflowRunner.ts | 15 +++++- .../src/scaffolder/tasks/types.ts | 4 +- 4 files changed, 67 insertions(+), 28 deletions(-) diff --git a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml index 007e23a3d6..01bd5c2e4f 100644 --- a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml +++ b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml @@ -7,28 +7,27 @@ metadata: spec: type: website parameters: - - name: Enter some stuff - description: Enter some stuff - properties: - inputString: - type: string - title: string input test - inputObject: - type: object - title: object input test - properties: - first: - type: string - title: first - second: - type: number - title: second + - name: Enter some stuff + description: Enter some stuff + properties: + inputString: + type: string + title: string input test + inputObject: + type: object + title: object input test + properties: + first: + type: string + title: first + second: + type: number + title: second steps: - - id: debug - name: Debug - action: debug:log - input: - message: ${{ parameters.inputString }} - extra: ${{ parameters.inputObject }} - - + - id: debug + if: ${{ true === true }} + name: Debug + action: debug:log + input: + message: ${{ parameters.inputString }} + extra: ${{ parameters.inputObject }} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index 6447782654..d1e2c7e098 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -136,6 +136,7 @@ describe('DefaultWorkflowRunner', () => { expect(fakeActionHandler).toHaveBeenCalledTimes(1); }); }); + describe('conditionals', () => { it('should execute steps conditionally', async () => { const task = createMockTaskWithSpec({ @@ -340,4 +341,32 @@ describe('DefaultWorkflowRunner', () => { expect(output.foo).toEqual('BACKSTAGE'); }); }); + + describe('filters', () => { + it('provides the parseRepoUrl filter', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'output-action', + input: {}, + }, + ], + output: { + foo: '${{parameters.repoUrl | parseRepoUrl}}', + }, + parameters: { + repoUrl: 'github.com?repo=repo&owner=owner', + }, + }); + + const { output } = await runner.execute(task); + + expect(output.foo.host).toEqual('github.com'); + expect(output.foo.owner).toEqual('owner'); + expect(output.foo.repo).toEqual('repo'); + }); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index c08363857d..30b855ac30 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -32,6 +32,7 @@ import { InputError } from '@backstage/errors'; import { PassThrough } from 'stream'; import { isTruthy } from './helper'; import { validate as validateJsonSchema } from 'jsonschema'; +import { parseRepoUrl } from '../actions/builtin/publish/util'; type Options = { workingDirectory: string; @@ -80,8 +81,6 @@ export class DefaultWorkflowRunner implements WorkflowRunner { private readonly nunjucks: nunjucks.Environment; constructor(private readonly options: Options) { - // TODO(blam): Probably need the repo helper here. - // Or we move to returning Objects in the RepoUrlPickerV2 or something? this.nunjucks = nunjucks.configure({ autoescape: false, tags: { @@ -89,6 +88,18 @@ export class DefaultWorkflowRunner implements WorkflowRunner { variableEnd: '}}', }, }); + + // TODO(blam): let's work out how we can deprecate these. + // We shouln't really need to be exposing these now we can deal with + // objects in the params block + this.nunjucks.addFilter('parseRepoUrl', repoUrl => { + return JSON.stringify(parseRepoUrl(repoUrl, this.options.integrations)); + }); + + this.nunjucks.addFilter('projectSlug', repoUrl => { + const { owner, repo } = parseRepoUrl(repoUrl, this.options.integrations); + return `${owner}/${repo}`; + }); } private render(input: T, context: TemplateContext): T { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 2ab38edec9..08baaca7f0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -69,7 +69,7 @@ export interface TaskSpecV1beta3 { baseUrl?: string; parameters: JsonObject; steps: TaskStep[]; - output: { [name: string]: string }; + output: { [name: string]: JsonValue }; } export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; @@ -141,7 +141,7 @@ export interface TaskStore { }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; } -export type WorkflowResponse = { output: { [name: string]: JsonValue } }; +export type WorkflowResponse = { output: { [key: string]: JsonObject } }; export interface WorkflowRunner { execute(task: Task): Promise; } From 5bdbb6caec21c6ce2e09ecb7d57f15346b2aab44 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 04:56:34 +0200 Subject: [PATCH 24/60] chore: fix up some types Signed-off-by: blam --- .../src/scaffolder/tasks/DefaultWorkflowRunner.test.ts | 8 +++++--- .../src/scaffolder/tasks/DefaultWorkflowRunner.ts | 3 ++- .../src/scaffolder/tasks/TaskWorker.test.ts | 6 +----- plugins/scaffolder-backend/src/scaffolder/tasks/types.ts | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index d1e2c7e098..0217236e92 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -364,9 +364,11 @@ describe('DefaultWorkflowRunner', () => { const { output } = await runner.execute(task); - expect(output.foo.host).toEqual('github.com'); - expect(output.foo.owner).toEqual('owner'); - expect(output.foo.repo).toEqual('repo'); + expect(output.foo).toEqual({ + host: 'github.com', + owner: 'owner', + repo: 'repo', + }); }); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 30b855ac30..ab1525fe49 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -91,7 +91,8 @@ export class DefaultWorkflowRunner implements WorkflowRunner { // TODO(blam): let's work out how we can deprecate these. // We shouln't really need to be exposing these now we can deal with - // objects in the params block + // objects in the params block. + // Maybe we can expose a new RepoUrlPicker with secrets for V3 that provides an object already. this.nunjucks.addFilter('parseRepoUrl', repoUrl => { return JSON.stringify(parseRepoUrl(repoUrl, this.options.integrations)); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index 456836046c..035398d448 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -14,15 +14,11 @@ * limitations under the License. */ -import os from 'os'; import { getVoidLogger, DatabaseManager } from '@backstage/backend-common'; -import { ConfigReader, JsonObject } from '@backstage/config'; -import { createTemplateAction, TemplateActionRegistry } from '../actions'; -import { RepoSpec } from '../actions/builtin/publish/util'; +import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { StorageTaskBroker } from './StorageTaskBroker'; import { TaskWorker } from './TaskWorker'; -import { ScmIntegrations } from '@backstage/integration'; import { WorkflowRunner } from './types'; import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 08baaca7f0..4f11d37da6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -141,7 +141,7 @@ export interface TaskStore { }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; } -export type WorkflowResponse = { output: { [key: string]: JsonObject } }; +export type WorkflowResponse = { output: { [key: string]: JsonValue } }; export interface WorkflowRunner { execute(task: Task): Promise; } From f8355c99cb649d286354a83e07da0f77c542d536 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 13:58:31 +0200 Subject: [PATCH 25/60] chore: remove all tracked temp directories and make sure to emit the processing and skipped events Signed-off-by: blam --- .../src/scaffolder/tasks/DefaultWorkflowRunner.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index ab1525fe49..9efb3f24bf 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -129,7 +129,7 @@ export class DefaultWorkflowRunner implements WorkflowRunner { * This is a little bit of a hack / magic so that when we use nunjucks and we try to * pass through something other than a string from the parameters section. * When an accessor is used that is an object, it's toString is the JSON.stringify'd version of it's children - * Which makes it work really well in string templating as we can parse the result again after.yarn + * Which makes it work really well in string templating as we can parse the result again after. */ return JSON.parse( JSON.stringify(input), @@ -171,11 +171,17 @@ export class DefaultWorkflowRunner implements WorkflowRunner { if (!isTruthy(ifResult)) { await task.emitLog( `Skipping step ${step.id} because it's if condition was false`, + { stepId: step.id, status: 'skipped' }, ); continue; } } + await task.emitLog(`Beginning step ${step.name}`, { + stepId: step.id, + status: 'processing', + }); + const action = this.options.actionRegistry.get(step.action); const { taskLogger, streamLogger } = createStepLogger({ task, step }); @@ -220,6 +226,11 @@ export class DefaultWorkflowRunner implements WorkflowRunner { }, }); + // Remove all temporary directories that were created when executing the action + for (const tmpDir of tmpDirs) { + await fs.remove(tmpDir); + } + context.steps[step.id] = { output: stepOutput }; await task.emitLog(`Finished step ${step.name}`, { From 5a0d2e9fa55addf41ded8258138bf8b2cdf40cf6 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 14:01:51 +0200 Subject: [PATCH 26/60] feat(catalog-model/docs): Updating `api-report` for `catalog-model` Signed-off-by: blam Signed-off-by: blam --- packages/catalog-model/api-report.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 1e7eb8fd52..1b6c3029b7 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -553,6 +553,33 @@ export interface TemplateEntityV1beta2 extends Entity { // @public (undocumented) export const templateEntityV1beta2Validator: KindValidator; +// @public (undocumented) +export interface TemplateEntityV1beta3 extends Entity { + // (undocumented) + apiVersion: 'backstage.io/v1beta3'; + // (undocumented) + kind: 'Template'; + // (undocumented) + spec: { + type: string; + parameters?: JsonObject | JsonObject[]; + steps: Array<{ + id?: string; + name?: string; + action: string; + input?: JsonObject; + if?: string | boolean; + }>; + output?: { + [name: string]: string; + }; + owner?: string; + }; +} + +// @public (undocumented) +export const templateEntityV1beta3Validator: KindValidator; + // @alpha export type UNSTABLE_EntityStatus = { items?: UNSTABLE_EntityStatusItem[]; From 18083d18213fbe8d70e10e5d5036ee3ef9061df1 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 14:08:35 +0200 Subject: [PATCH 27/60] chore: add changeset Signed-off-by: blam --- .changeset/polite-timers-watch.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/polite-timers-watch.md diff --git a/.changeset/polite-timers-watch.md b/.changeset/polite-timers-watch.md new file mode 100644 index 0000000000..d8f2ea80fb --- /dev/null +++ b/.changeset/polite-timers-watch.md @@ -0,0 +1,7 @@ +--- +'@backstage/catalog-model': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Introduce the new `backstage.io/v1beta3` template kind with nunjucks support 🥋 From cbe2803100e1d4d00037db77e40a8e2051c0b02b Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 15:39:31 +0200 Subject: [PATCH 28/60] chore: remove the hackery and wrap up the template stringify Signed-off-by: blam --- .../tasks/DefaultWorkflowRunner.test.ts | 30 ++++++++- .../scaffolder/tasks/DefaultWorkflowRunner.ts | 61 +++++++++++-------- 2 files changed, 64 insertions(+), 27 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index 0217236e92..542cbb4b2f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -235,6 +235,34 @@ describe('DefaultWorkflowRunner', () => { ); }); + it('should keep the original types for the input and not parse things that arent meant to be parsed', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'jest-mock-action', + input: { + number: '${{parameters.number}}', + string: '${{parameters.string}}', + }, + }, + ], + output: {}, + parameters: { + number: 0, + string: '1', + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { number: 0, string: '1' } }), + ); + }); + it('should template complex values into the action', async () => { const task = createMockTaskWithSpec({ apiVersion: 'backstage.io/v1beta3', @@ -355,7 +383,7 @@ describe('DefaultWorkflowRunner', () => { }, ], output: { - foo: '${{parameters.repoUrl | parseRepoUrl}}', + foo: '${{ parameters.repoUrl | parseRepoUrl }}', }, parameters: { repoUrl: 'github.com?repo=repo&owner=owner', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 9efb3f24bf..da59951d36 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -94,7 +94,7 @@ export class DefaultWorkflowRunner implements WorkflowRunner { // objects in the params block. // Maybe we can expose a new RepoUrlPicker with secrets for V3 that provides an object already. this.nunjucks.addFilter('parseRepoUrl', repoUrl => { - return JSON.stringify(parseRepoUrl(repoUrl, this.options.integrations)); + return parseRepoUrl(repoUrl, this.options.integrations); }); this.nunjucks.addFilter('projectSlug', repoUrl => { @@ -107,15 +107,43 @@ export class DefaultWorkflowRunner implements WorkflowRunner { return JSON.parse(JSON.stringify(input), (_key, value) => { try { if (typeof value === 'string') { + try { + // Let's assume that we're dealing with a template string. + if (value.startsWith('${{') && value.endsWith('}}')) { + // Lets convert ${{ parameters.bob }} to ${{ (parameters.bob) | dump }} so we can keep the input type + const wrappedDumped = value.replace( + /\${{(.+)}}/g, + '${{ ( $1 ) | dump }}', + ); + + // Run the templating + const templated = this.nunjucks.renderString( + wrappedDumped, + context, + ); + + // If there's emtpy string returned, then it's undefined + if (templated === '') { + return undefined; + } + + // Reparse the dumped string + return JSON.parse(templated); + } + } catch (ex) { + this.options.logger.debug( + `Failed to parse template string: ${value} with error ${ex.message}`, + ); + } + + // Fallback to default behaviour const templated = this.nunjucks.renderString(value, context); + if (templated === '') { return undefined; } - try { - return JSON.parse(templated); - } catch { - return templated; - } + + return templated; } } catch { return value; @@ -124,25 +152,6 @@ export class DefaultWorkflowRunner implements WorkflowRunner { }); } - private makeStringifyableParams(input: T): T { - /** - * This is a little bit of a hack / magic so that when we use nunjucks and we try to - * pass through something other than a string from the parameters section. - * When an accessor is used that is an object, it's toString is the JSON.stringify'd version of it's children - * Which makes it work really well in string templating as we can parse the result again after. - */ - return JSON.parse( - JSON.stringify(input), - (key: string, value: JsonObject) => { - if (typeof value === 'object' && key) { - value.toString = () => JSON.stringify(value); - } - - return value; - }, - ); - } - async execute(task: Task): Promise { if (!isValidTaskSpec(task.spec)) { throw new InputError( @@ -160,7 +169,7 @@ export class DefaultWorkflowRunner implements WorkflowRunner { ); const context: TemplateContext = { - parameters: this.makeStringifyableParams(task.spec.parameters), + parameters: task.spec.parameters, steps: {}, }; From f39de106b5677e3dbb71047f52f9ab5fdc6930f0 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 17:51:25 +0200 Subject: [PATCH 29/60] chore: only dump when there's only one param ref Signed-off-by: blam --- .../scaffolder/tasks/DefaultWorkflowRunner.ts | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index da59951d36..63ba392dcc 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -80,17 +80,19 @@ const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { export class DefaultWorkflowRunner implements WorkflowRunner { private readonly nunjucks: nunjucks.Environment; + private readonly nunjucksOptions: nunjucks.ConfigureOptions = { + autoescape: false, + tags: { + variableStart: '${{', + variableEnd: '}}', + }, + }; + constructor(private readonly options: Options) { - this.nunjucks = nunjucks.configure({ - autoescape: false, - tags: { - variableStart: '${{', - variableEnd: '}}', - }, - }); + this.nunjucks = nunjucks.configure(this.nunjucksOptions); // TODO(blam): let's work out how we can deprecate these. - // We shouln't really need to be exposing these now we can deal with + // We shouldn't really need to be exposing these now we can deal with // objects in the params block. // Maybe we can expose a new RepoUrlPicker with secrets for V3 that provides an object already. this.nunjucks.addFilter('parseRepoUrl', repoUrl => { @@ -103,13 +105,21 @@ export class DefaultWorkflowRunner implements WorkflowRunner { }); } + private isSingleTemplateString(input: string) { + const { parser, nodes } = require('nunjucks'); + const parsed = parser.parse(input, {}, this.nunjucksOptions); + return ( + parsed.children.length === 1 && + !(parsed.children[0] instanceof nodes.TemplateData) + ); + } + private render(input: T, context: TemplateContext): T { return JSON.parse(JSON.stringify(input), (_key, value) => { try { if (typeof value === 'string') { try { - // Let's assume that we're dealing with a template string. - if (value.startsWith('${{') && value.endsWith('}}')) { + if (this.isSingleTemplateString(value)) { // Lets convert ${{ parameters.bob }} to ${{ (parameters.bob) | dump }} so we can keep the input type const wrappedDumped = value.replace( /\${{(.+)}}/g, @@ -122,7 +132,7 @@ export class DefaultWorkflowRunner implements WorkflowRunner { context, ); - // If there's emtpy string returned, then it's undefined + // If there's an empty string returned, then it's undefined if (templated === '') { return undefined; } @@ -131,7 +141,7 @@ export class DefaultWorkflowRunner implements WorkflowRunner { return JSON.parse(templated); } } catch (ex) { - this.options.logger.debug( + this.options.logger.error( `Failed to parse template string: ${value} with error ${ex.message}`, ); } From 808d30cc307bb2ec5118153718d99e9f508ad520 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 28 Sep 2021 20:29:19 +0200 Subject: [PATCH 30/60] docs: removing older documentation for now Signed-off-by: blam --- docs/features/software-templates/legacy.md | 117 ------ .../migrating-from-v1alpha1-to-v1beta2.md | 333 ------------------ .../migrating-from-v1beta2-to-v1beta3.md | 148 ++++++++ microsite/sidebars.json | 3 +- 4 files changed, 149 insertions(+), 452 deletions(-) delete mode 100644 docs/features/software-templates/legacy.md delete mode 100644 docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md create mode 100644 docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md diff --git a/docs/features/software-templates/legacy.md b/docs/features/software-templates/legacy.md deleted file mode 100644 index 49f3e21904..0000000000 --- a/docs/features/software-templates/legacy.md +++ /dev/null @@ -1,117 +0,0 @@ ---- -id: template-legacy -title: Writing Templates (Legacy) -# prettier-ignore -description: Old documentation describing the backstage.io/v1alpha1 format of the Template Schema ---- - -## Kind: Template - -Describes the following entity kind: - -| Field | Value | -| ------------ | ----------------------- | -| `apiVersion` | `backstage.io/v1alpha1` | -| `kind` | `Template` | - -A Template describes a skeleton for use with the Scaffolder. It is used for -describing what templating library is supported, and also for documenting the -variables that the template requires using -[JSON Forms Schema](https://jsonforms.io/). - -Descriptor files for this kind may look as follows. - -```yaml -apiVersion: backstage.io/v1alpha1 -kind: Template -metadata: - name: react-ssr-template - title: React SSR Template - description: - Next.js application skeleton for creating isomorphic web applications. - tags: - - recommended - - react -spec: - owner: web@example.com - templater: cookiecutter - type: website - path: '.' - schema: - required: - - component_id - - description - properties: - component_id: - title: Name - type: string - description: Unique name of the component - description: - title: Description - type: string - description: Description of the component -``` - -In addition to the [common envelope metadata](#common-to-all-kinds-the-metadata) -shape, this kind has the following structure. - -### `apiVersion` and `kind` [required] - -Exactly equal to `backstage.io/v1alpha1` and `Template`, respectively. - -### `metadata.title` [required] - -The nice display name for the template as a string, e.g. `React SSR Template`. -This field is required as is used to reference the template to the user instead -of the `metadata.name` field. - -### `metadata.tags` [optional] - -A list of strings that can be associated with the template, e.g. -`['recommended', 'react']`. - -This list will also be used in the frontend to display to the user so you can -potentially search and group templates by these tags. - -### `spec.type` [optional] - -The type of component as a string, e.g. `website`. This field is optional but -recommended. - -The software catalog accepts any type value, but an organization should take -great care to establish a proper taxonomy for these. Tools including Backstage -itself may read this field and behave differently depending on its value. For -example, a website type component may present tooling in the Backstage interface -that is specific to just websites. - -The current set of well-known and common values for this field is: - -- `service` - a backend service, typically exposing an API -- `website` - a website -- `library` - a software library, such as an npm module or a Java library - -### `spec.templater` [required] - -The templating library that is supported by the template skeleton as a string, -e.g `cookiecutter`. - -Different skeletons will use different templating syntax, so it's common that -the template will need to be run with a particular piece of software. - -This key will be used to identify the correct templater which is registered into -the `TemplatersBuilder`. - -The values which are available by default are: - -- `cookiecutter` - [cookiecutter](https://github.com/cookiecutter/cookiecutter). - -### `spec.path` [optional] - -The string location where the templater should be run if it is not on the same -level as the `template.yaml` definition, e.g. `./cookiecutter/skeleton`. - -This will set the `cwd` when running the templater to the folder path that you -specify relative to the `template.yaml` definition. - -This is also particularly useful when you have multiple template definitions in -the same repository but only a single `template.yaml` registered in backstage. diff --git a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md deleted file mode 100644 index 58d88e8e9e..0000000000 --- a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md +++ /dev/null @@ -1,333 +0,0 @@ ---- -id: migrating-from-v1alpha1-to-v1beta2 -title: Migrating to v1beta2 templates -# prettier-ignore -description: How to move your old templates from v1alpha1 to the more declarative v1beta2 ---- - -# What's new? - -Previously, the scaffolder was very restricted in what you could do when -creating new software components from templates. There were three scaffolding -steps which was pretty hard to extend and add new functionality to, difficult to -re-use logic between templates. There used to be a fixed pipeline of -`preparers`, `templaters`, and `publishers`, which were defined by the backend -and needed to be run for each template. This is now changed, to give the -template total control over what should be executed as part of the templating -run. This makes templates a little more declarative as you can now register -different `actions` or `functions` with the `scaffolder-backend` which you then -can decide how, and in what order, to run using the template definition YAML -file. - -We've also made some improvements, and added some helpers to work with -cookiecutter. The skeleton for a template can now be stored in a different place -to where your entity definition is: previously you needed to have your -`template.yaml` next to the skeleton source (`{{cookiecutter.component_id}}` -directory), but now that's not the case. Part of the changes with the `v1beta2` -syntax is that you can grab your template source from any repository, and re-use -them between templates. - -We've also renamed the `schema` property to `parameters` as this makes more -sense when using them as parameters to the actions or steps that you've setup -for your templates. There's the added benefit that you can now assign an array -to the `parameters` property, which will then give you multiple steps in the UI, -so you can split apart your input parameters and group them as needed rather -than having one long list of input fields. - -## The `parameters` property - -The `schema` key has now been renamed to `parameters` with a few more features. -You can pass an array now to break apart the input form into different steps in -the UI. You can also specify `ui:schema` fields that are passed along to -[`react-jsonschema-form`](https://rjsf-team.github.io/react-jsonschema-form/) -inline with the JSON schema. - -```yaml -spec: - parameters: - - title: Fill in some steps - required: - - name - properties: - name: - title: Name - type: string - description: Unique name of the component - ui:autofocus: true - ui:options: - rows: 5 -``` - -## The `steps` property - -`v1beta2` template syntax introduces the new `steps` property, which is an array -of `actions` that the scaffolder will run in combination with the user input -that is declared in the `schema`. Actions look like the following: - -```yaml -spec: - steps: - - id: publish # a unique id for the step, can be anything you like - name: Publish # a user friendly name for the step, this is what is shown in the frontend - action: publish:github # the action ID that has been registered with the scaffolder-backend - input: # parameters that are passed as input to the action handler function - allowedHosts: ['github.com'] - description: 'This is {{ parameters.name }}' # handlebars templating is supported with the values from the parameters section in the same file. - repoUrl: '{{ parameters.repoUrl }}' -``` - -# Migrating a `v1alpha1` template - -## The template definition (.yaml) - -### `parameters` - -Because of the changes to invert the control to the `template.yaml` definition -for running the workflow, we need to adjust the `schema` property and we also -now need to define what the template is actually going to do as part of the -template run. - -A simple migration would move the following yaml: - -```yaml -apiVersion: backstage.io/v1alpha1 -kind: Template -metadata: - name: react-ssr-template - title: React SSR Template - description: Create a website powered with Next.js - tags: - - recommended - - react -spec: - owner: web@example.com - templater: cookiecutter - type: website - path: '.' - schema: - required: - - component_id - - description - properties: - component_id: - title: Name - type: string - description: Unique name of the component - description: - title: Description - type: string - description: Help others understand what this website is for. -``` - -To something that looks like the following: - -```yaml -apiVersion: backstage.io/v1beta2 -kind: Template -metadata: - name: react-ssr-template - title: React SSR Template - description: Create a website powered with Next.js - tags: - - recommended - - react -spec: - owner: web@example.com - type: website - parameters: - - title: Add some input - required: - - component_id - - description - properties: - component_id: - title: Name - type: string - description: Unique name of the component - description: - title: Description - type: string - description: Help others understand what this website is for. - - title: Some more additional info that was previously provided automatically - required: - - owner - - repoUrl - properties: - owner: - title: Owner - type: string - description: Owner of the component - ui:field: OwnerPicker - ui:options: - allowedKinds: - - Group - - title: Choose a location - repoUrl: - title: Repository Location - type: string - ui:field: RepoUrlPicker - ui:options: - allowedHosts: - - github.com -``` - -There are a few things to note here. On the `alpha` version, the second step of -the template flow in the frontend was provided by Backstage for free, so we used -to collect the user input for the `owner` field and the `repositoryUrl` that you -were going to publish to. Now because `actions` can have any workflow they like, -it doesn't make sense to still provide these fields for every scaffolding -workflow, as you might not need these anymore. That's why we now manually add -those fields back into the template parameters that are shown to the user: - -```yaml - - title: Some more additional info that was previously provided automatically - required: - - owner - - repoUrl - properties: - owner: - title: Owner - type: string - description: Owner of the component - ui:field: OwnerPicker - ui:options: - allowedKinds: - - Group - - title: Choose a location - repoUrl: - title: Repository Location - type: string - ui:field: RepoUrlPicker - ui:options: - allowedHosts: - - github.com -``` - -Maybe you also don't need to publish to `github.com`, you should replace this -with your VCS provider URL that is listed in your `integrations` config instead. - -### `steps` - -So now we should have all the required information that we need from the user in -a much more extensible way. We now need to tell the scaffolder what to do with -these parameters and what to do with the user input. - -We've made templating using `cookiecutter` a little simpler. You don't need to -store the `cookiecutter` skeleton in the same directory as the `template.yaml` -definition, it can live wherever you like - maybe a shared repository somewhere -so you can re-use the skeletons but apply different actions for different -templates depending on your use case. - -We also no longer need to have a directory called -`{{cookiecutter.component_id}}`. This is because now we can't ensure that -`component_id` will be a parameter that is provided from the frontend, this -could break `cookiecutter`. If your directory structure used to look like this: - -``` -my-awesome-template - -> {{cookiecutter.component_id}} - -> file.txt - -> some_more_files.ts - -> hooks - -> post_gen_project.sh - -> template.yaml -``` - -We now recommend that you move to the following structure: - -``` -my-awesome-template - -> skeleton - -> file.txt - -> some_more_files.ts - -> template.yaml -``` - -This migration renames the skeleton folder to something more semantic, and also -drops support for `cookiecutter` hooks. We've dropped support for `cookiecutter` -hooks for now, as hopefully everything that is stored in these hooks can be -moved to `actions` instead, and for security reasons, it's more secure to run -trusted code that you ship with Backstage as an action rather than some script -that can be pulled in from anywhere which doesn't get vetted first. It's a -pretty big security risk that those scripts will be run on Backstage instances -inside your infrastructure, especially `.sh` files. - -If you really need hooks and can't find a suitable solution by using actions -please reach out to us through a ticket and we'll see what we can do to assist -:) - -You'll notice that we removed the `templater` property from the `spec` -definition in the template `yaml`, so there's no way to define that this is a -`cookiecutter` `templater`. - -We've created a built-in action that you can use which will when run, go grab a -directory from anywhere and run `cookiecutter` on top of it, and then extract -the contents into the working directory for the scaffolder. - -Adding the `steps` for a simple template should look something like the -following: - -```yaml -spec: - steps: - # this action will go use cookiecutter to template some files into the working directory - - id: template # an ID for the templating step - name: Create skeleton # A user friendly name for the action - action: fetch:cookiecutter - input: - url: ./skeleton # this is the directory for your skeleton files. - # If it's located next to the `template.yaml` then you can use a relative path, - # otherwise you can use absolute URLs that point at the VCS: https://github.com/backstage/backstage/tree/master/some_folder_somewhere - values: - # for each value that you need to pass to cookiecutter, they should be listed here and set in this values object. - # You can use the handlebars templating syntax to pull them from the input parameters listed in the same file - name: '{{ parameters.name }}' - owner: '{{ parameters.owner }}' - destination: '{{ parseRepoUrl parameters.repoUrl }}' - - # this action is for publishing the working directory to the VCS - - id: publish - name: Publish - action: publish:github - input: - allowedHosts: ['github.com'] - description: 'This is {{ parameters.name }}' - repoUrl: '{{ parameters.repoUrl }}' - - # this action will then register the created component in Backstage - - id: register - name: Register - action: catalog:register - input: - repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' - catalogInfoPath: '/catalog-info.yaml' -``` - -### `output` - -Steps can output values, and so can the template itself. This is good for -returning values to the frontend, so we can make the buttons like -`Go to catalog` and `Go to repo` work correctly. You can add the following to -your `template.yaml` to make sure you return the right values from the steps: - -```yaml -spec: - output: - remoteUrl: '{{ steps.publish.output.remoteUrl }}' - entityRef: '{{ steps.register.output.entityRef }}' -``` - -Or you can return a `links` array with text and a URL explicitly: - -```yaml -spec: - output: - links: - - url: '{{steps.publish.output.remoteUrl}}' - title: 'Go to Repo' -``` - -## Questions? - -If you have any questions or feedback, please reach out to us on GitHub or -Discord and we will do our best to help! diff --git a/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md new file mode 100644 index 0000000000..8ccacfa301 --- /dev/null +++ b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md @@ -0,0 +1,148 @@ +--- +id: migrating-from-v1beta2-to-v1beta3 +title: Migrating to v1beta3 templates +# prettier-ignore +description: How to migrate your existing templates to beta3 syntax +--- + +# What's new? + +Well then, here we are! 🚀 + +Backstage has had many forms of templating languages throughout different +plugins and different systems. We've had `cookiecutter` syntax in templates, and +we also had `handlebars` templating in the `kind: Template`. Then we wanted to +remove the additional dependency on `cookiecutter` for `Software Templates` out +of the box, so we introduced `nunjucks` as an alternative in `fetch:template` +action which is based on the `jinja2` syntax so they're pretty similar. In an +effort to reduce confusion and unify on to one templating language, we're +officially deprecating support for `handlebars` templating in the +`kind: Template` entities with version `backstage.io/v1beta3` and moving to +using `nunjucks` instead. + +This provides us a lot of built in `filters` (`handlebars` helpers), that as +Template authors will give you much more flexibility out of the box, and also +open up sharing of filters in the `entity` and the actual `skeleton` too, and +removing the slight differences between the two languages. + +We've also removed a lot of the built in helpers that we shipped with +`handlebars`, as they're now supported as first class citizens by either +`nunjucks` or the new `scaffolder` when using `backstage.io/v1beta3` +`apiVersion` + +The migration path is pretty simple, and we've removed some of the pain points +from writing the `handlebars` templates too. Let's go through what's new and how +to upgrade. + +## `${{ }}` instead of `"{{ }}"` + +One really big readability and cause for confusing was the fact that with +`handlebars` and `yaml` was that you always had to wrap your templating strings +in quotes in `yaml` so that it didn't try to parse it as a `json` object and +fail. This was pretty annoying, as it also meant that all things look like +strings. Now that's no longer the case, you can now remove the `""` and take +advantage of writing nice `yaml` files that just work. + +```diff + spec: + steps: + input: + allowedHosts: ['github.com'] +- description: 'This is {{ parameters.name }}' ++ description: This is ${{ parameters.name }} +- repoUrl: '{{ parameters.repoUrl }}' ++ repoUrl: ${{ parameters.repoUrl }} +``` + +## No more `eq` or `not` helper + +These helpers are no longer needed with the more expressive `api` that +`nunjucks` provides. You can simply use the built-in `nunjucks` and `jinja2` +style operators. + +```diff + spec: + steps: + input: +- if: '{{ eq parameters.value "backstage" }}' ++ if: ${{ parameters.value === "backstage" }} + ... + +``` + +And then for the `not` + +```diff + spec: + steps: + input: +- if: '{{ not parameters.value "backstage" }}' ++ if: ${{ parameters.value !== "backstage" }} + ... + +``` + +Much better right? ✨ + +## No more `json` helper + +This helper is no longer needed, as we've added support for complex values and +supporting the additional primitive values now rather than everything being a +`string`. This means that now that you can pass around `parameters` and it +should all work as expected and keep the type that has been declared in the +input schema. + +```diff + spec: + parameters: + test: + type: number + name: Test Number + address: + type: object + required: + - line1 + properties: + line1:🙏 + type: string + name: Line 1 + line2: + type: string + name: Line 2 + + steps: + - id: test step + action: run:something + input: +- address: '{{ json parameters.address }}' ++ address: ${{ parameters.address }} +- number: '{{ parameters.number }}' ++ number: ${{ parameters.number }} # this will now make sure that the type of number is a number now 🙏 + +``` + +## `parseRepoUrl` is now a `filter` + +All calls to `parseRepoUrl` are now a `jinja2` `filter`, which means you'll need +to update the syntax. + +```diff + spec: + steps: + input: +- repoUrl: '{{ parseRepoUrl parameters.repoUrl }}' ++ repoUrl: ${{ parameters.repoUrl | parseRepoUrl }} + ... +``` + +Now we have complex value support here too, expect that this `filter` will go +away in future versions and the `RepoUrlPicker` will return an object so +`parameters.repoUrl` will already be a +`{ host: string; owner: string; repo: string }` 🚀 + +### Summary + +Of course, we're always available on [discord](https://discord.gg/MUpMjP2) if +you're stuck or something's not working as expected. You can also +[raise an issue](https://github.com/backstage/backstage/issues/new/choose) with +feedback or bugs! diff --git a/microsite/sidebars.json b/microsite/sidebars.json index c269f8ac8f..cdbfdccec3 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -78,8 +78,7 @@ "features/software-templates/builtin-actions", "features/software-templates/writing-custom-actions", "features/software-templates/writing-custom-field-extensions", - "features/software-templates/template-legacy", - "features/software-templates/migrating-from-v1alpha1-to-v1beta2" + "features/software-templates/migrating-from-v1beta2-to-v1beta3" ] }, { From a61204bc4ba1f8cb14d743fb3557b914b6364948 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 29 Sep 2021 10:53:34 +0200 Subject: [PATCH 31/60] chore: remove the link to the legacy docs Signed-off-by: blam --- docs/features/software-catalog/descriptor-format.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 7a9aa099e9..c174b94557 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -620,9 +620,6 @@ The following describes the following entity kind: | `apiVersion` | `backstage.io/v1beta2` | | `kind` | `Template` | -If you're looking for docs on `v1alpha1` you can find them -[here](../software-templates/legacy.md) - A template definition describes both the parameters that are rendered in the frontend part of the scaffolding wizard, and the steps that are executed when scaffolding that component. From ca9c69f3a88463474abafe91cfdc7848b46235e8 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 30 Sep 2021 17:16:08 +0200 Subject: [PATCH 32/60] chore: fix code review comments Signed-off-by: blam --- .../src/scaffolder/tasks/DefaultWorkflowRunner.ts | 2 +- plugins/scaffolder-backend/src/service/router.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 63ba392dcc..354092ee06 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -14,7 +14,6 @@ * limitations under the License. */ import { ScmIntegrations } from '@backstage/integration'; -import { TemplateActionRegistry } from '..'; import { Task, TaskSpec, @@ -33,6 +32,7 @@ import { PassThrough } from 'stream'; import { isTruthy } from './helper'; import { validate as validateJsonSchema } from 'jsonschema'; import { parseRepoUrl } from '../actions/builtin/publish/util'; +import { TemplateActionRegistry } from '../actions'; type Options = { workingDirectory: string; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index b75a5d0368..8e71dde461 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -200,8 +200,7 @@ export async function createRouter( } const baseUrl = getEntityBaseUrl(template); - // TODO: need to make sure that the TaskSpec is the right format here. - // If it's beta2 use values, beta3 uses parameters to clear that up. + taskSpec = template.apiVersion === 'backstage.io/v1beta2' ? { From 8893feca7306facecee44ca5737619a41442c45f Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 1 Oct 2021 11:21:59 +0200 Subject: [PATCH 33/60] chore: docs updates Signed-off-by: blam --- .../migrating-from-v1beta2-to-v1beta3.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md index 8ccacfa301..84d76b9fb1 100644 --- a/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md +++ b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md @@ -36,12 +36,12 @@ to upgrade. ## `${{ }}` instead of `"{{ }}"` -One really big readability and cause for confusing was the fact that with -`handlebars` and `yaml` was that you always had to wrap your templating strings -in quotes in `yaml` so that it didn't try to parse it as a `json` object and -fail. This was pretty annoying, as it also meant that all things look like -strings. Now that's no longer the case, you can now remove the `""` and take -advantage of writing nice `yaml` files that just work. +One really big readability issue and cause for confusion was the fact that with +`handlebars` and `yaml`you always had to wrap your templating strings in quotes +in `yaml` so that it didn't try to parse it as a `json` object and fail. This +was pretty annoying, as it also meant that all things look like strings. Now +that's no longer the case, you can now remove the `""` and take advantage of +writing nice `yaml` files that just work. ```diff spec: @@ -54,7 +54,7 @@ advantage of writing nice `yaml` files that just work. + repoUrl: ${{ parameters.repoUrl }} ``` -## No more `eq` or `not` helper +## No more `eq` or `not` helpers These helpers are no longer needed with the more expressive `api` that `nunjucks` provides. You can simply use the built-in `nunjucks` and `jinja2` @@ -117,7 +117,7 @@ input schema. - address: '{{ json parameters.address }}' + address: ${{ parameters.address }} - number: '{{ parameters.number }}' -+ number: ${{ parameters.number }} # this will now make sure that the type of number is a number now 🙏 ++ number: ${{ parameters.number }} # this will now make sure that the type of number is a number 🙏 ``` From 18404c795b11013613399c31d3aef96cf66cb6aa Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 11 Oct 2021 14:31:53 +0200 Subject: [PATCH 34/60] chore: reset the changes to the catalog-model Signed-off-by: blam Signed-off-by: blam --- packages/catalog-model/api-report.md | 27 ----------------------- packages/catalog-model/src/kinds/index.ts | 2 -- 2 files changed, 29 deletions(-) diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 1b6c3029b7..1e7eb8fd52 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -553,33 +553,6 @@ export interface TemplateEntityV1beta2 extends Entity { // @public (undocumented) export const templateEntityV1beta2Validator: KindValidator; -// @public (undocumented) -export interface TemplateEntityV1beta3 extends Entity { - // (undocumented) - apiVersion: 'backstage.io/v1beta3'; - // (undocumented) - kind: 'Template'; - // (undocumented) - spec: { - type: string; - parameters?: JsonObject | JsonObject[]; - steps: Array<{ - id?: string; - name?: string; - action: string; - input?: JsonObject; - if?: string | boolean; - }>; - output?: { - [name: string]: string; - }; - owner?: string; - }; -} - -// @public (undocumented) -export const templateEntityV1beta3Validator: KindValidator; - // @alpha export type UNSTABLE_EntityStatus = { items?: UNSTABLE_EntityStatusItem[]; diff --git a/packages/catalog-model/src/kinds/index.ts b/packages/catalog-model/src/kinds/index.ts index edac9c505a..be9f7a0d6a 100644 --- a/packages/catalog-model/src/kinds/index.ts +++ b/packages/catalog-model/src/kinds/index.ts @@ -52,8 +52,6 @@ export type { } from './SystemEntityV1alpha1'; export { templateEntityV1beta2Validator } from './TemplateEntityV1beta2'; export type { TemplateEntityV1beta2 } from './TemplateEntityV1beta2'; -export { templateEntityV1beta3Validator } from './TemplateEntityV1beta3'; -export type { TemplateEntityV1beta3 } from './TemplateEntityV1beta3'; export type { KindValidator } from './types'; export { userEntityV1alpha1Validator } from './UserEntityV1alpha1'; export type { From 5118da771f7fbfc77a20c41e847db7c9e55f1f94 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 11 Oct 2021 14:34:55 +0200 Subject: [PATCH 35/60] chore: remove the changes for catalog-backend too Signed-off-by: blam --- .../ingestion/processors/BuiltinKindsEntityProcessor.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts index 18a8ad2a27..bcabc00f32 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts @@ -48,8 +48,6 @@ import { systemEntityV1alpha1Validator, TemplateEntityV1beta2, templateEntityV1beta2Validator, - TemplateEntityV1beta3, - templateEntityV1beta3Validator, UserEntity, userEntityV1alpha1Validator, } from '@backstage/catalog-model'; @@ -63,9 +61,6 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { resourceEntityV1alpha1Validator, groupEntityV1alpha1Validator, locationEntityV1alpha1Validator, - templateEntityV1beta3Validator, - - // TODO: remove once beta3 is stable templateEntityV1beta2Validator, userEntityV1alpha1Validator, systemEntityV1alpha1Validator, @@ -139,7 +134,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { * Emit relations for the Template kind */ if (entity.kind === 'Template') { - const template = entity as TemplateEntityV1beta2 | TemplateEntityV1beta3; + const template = entity as TemplateEntityV1beta2; doEmit( template.spec.owner, { defaultKind: 'Group', defaultNamespace: selfRef.namespace }, From b1d5d584e57d2f5f2551020fe47a03dacdf69288 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 11 Oct 2021 14:43:05 +0200 Subject: [PATCH 36/60] chore: updating the version string Signed-off-by: blam --- .../src/scaffolder/tasks/types.ts | 2 +- plugins/scaffolder-backend/src/service/router.ts | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 4f11d37da6..cd93404165 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -65,7 +65,7 @@ export interface TaskStep { if?: string | boolean; } export interface TaskSpecV1beta3 { - apiVersion: 'backstage.io/v1beta3'; + apiVersion: 'scaffolder.backstage.io/v1beta3'; baseUrl?: string; parameters: JsonObject; steps: TaskStep[]; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 8e71dde461..f8ba99336f 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -34,16 +34,15 @@ import { } from '@backstage/backend-common'; import { InputError, NotFoundError } from '@backstage/errors'; import { CatalogApi } from '@backstage/catalog-client'; -import { - TemplateEntityV1beta2, - Entity, - TemplateEntityV1beta3, -} from '@backstage/catalog-model'; +import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; +import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; + import { ScmIntegrations } from '@backstage/integration'; import { TemplateAction } from '../scaffolder/actions'; import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; +import { TaskSpec } from '../scaffolder/tasks/types'; export interface RouterOptions { logger: Logger; @@ -61,7 +60,7 @@ function isSupportedTemplate( ) { return ( entity.apiVersion === 'backstage.io/v1beta2' || - entity.apiVersion === 'backstage.io/v1beta3' + entity.apiVersion === 'scaffolder.backstage.io/v1beta3' ); } @@ -187,7 +186,7 @@ export async function createRouter( token, }); - let taskSpec; + let taskSpec: TaskSpec; if (isSupportedTemplate(template)) { for (const parameters of [template.spec.parameters ?? []].flat()) { From 17d7f5bc57dfb5d57db11927694de2d6dfcb9f25 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 11 Oct 2021 14:51:43 +0200 Subject: [PATCH 37/60] feat: removing the older stuff and updating to work with the new common packages Signed-off-by: blam --- .../src/kinds/TemplateEntityV1beta3.test.ts | 156 ------------------ .../src/kinds/TemplateEntityV1beta3.ts | 43 ----- .../tasks/DefaultWorkflowRunner.test.ts | 26 +-- .../scaffolder/tasks/DefaultWorkflowRunner.ts | 2 +- .../src/scaffolder/tasks/TaskWorker.test.ts | 4 +- .../src/scaffolder/tasks/TaskWorker.ts | 2 +- 6 files changed, 17 insertions(+), 216 deletions(-) delete mode 100644 packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts delete mode 100644 packages/catalog-model/src/kinds/TemplateEntityV1beta3.ts diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts b/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts deleted file mode 100644 index cc275435e8..0000000000 --- a/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - TemplateEntityV1beta3, - templateEntityV1beta3Validator as validator, -} from './TemplateEntityV1beta3'; - -describe('templateEntityV1beta3Validator', () => { - let entity: TemplateEntityV1beta3; - - beforeEach(() => { - entity = { - apiVersion: 'backstage.io/v1beta3', - kind: 'Template', - metadata: { - name: 'test', - }, - spec: { - type: 'website', - parameters: { - required: ['storePath', 'owner'], - properties: { - owner: { - type: 'string', - title: 'Owner', - description: 'Who is going to own this component', - }, - }, - }, - steps: [ - { - id: 'fetch', - name: 'Fetch', - action: 'fetch:plan', - input: { - url: './template', - }, - if: '${{ parameters.owner }}', - }, - ], - output: { - fetchUrl: '${{ steps.fetch.output.targetUrl }}', - }, - owner: 'team-b@example.com', - }, - }; - }); - - it('happy path: accepts valid data', async () => { - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('ignores unknown apiVersion', async () => { - (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(validator.check(entity)).resolves.toBe(false); - }); - - it('ignores unknown kind', async () => { - (entity as any).kind = 'Wizard'; - await expect(validator.check(entity)).resolves.toBe(false); - }); - - it('rejects missing type', async () => { - delete (entity as any).spec.type; - await expect(validator.check(entity)).rejects.toThrow(/type/); - }); - - it('accepts any other type', async () => { - (entity as any).spec.type = 'hallo'; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('accepts missing parameters', async () => { - delete (entity as any).spec.parameters; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('accepts missing outputs', async () => { - delete (entity as any).spec.outputs; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('rejects empty type', async () => { - (entity as any).spec.type = ''; - await expect(validator.check(entity)).rejects.toThrow(/type/); - }); - - it('rejects missing steps', async () => { - delete (entity as any).spec.steps; - await expect(validator.check(entity)).rejects.toThrow(/steps/); - }); - - it('accepts step with missing id', async () => { - delete (entity as any).spec.steps[0].id; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('accepts step with missing name', async () => { - delete (entity as any).spec.steps[0].name; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('rejects step with missing action', async () => { - delete (entity as any).spec.steps[0].action; - await expect(validator.check(entity)).rejects.toThrow(/action/); - }); - - it('accepts missing owner', async () => { - delete (entity as any).spec.owner; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('rejects empty owner', async () => { - (entity as any).spec.owner = ''; - await expect(validator.check(entity)).rejects.toThrow(/owner/); - }); - - it('rejects wrong type owner', async () => { - (entity as any).spec.owner = 5; - await expect(validator.check(entity)).rejects.toThrow(/owner/); - }); - - it('accepts missing if', async () => { - delete (entity as any).spec.steps[0].if; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('accepts boolean in if', async () => { - (entity as any).spec.steps[0].if = true; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('accepts empty if', async () => { - (entity as any).spec.steps[0].if = ''; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('rejects wrong type if', async () => { - (entity as any).spec.steps[0].if = 5; - await expect(validator.check(entity)).rejects.toThrow(/if/); - }); -}); diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1beta3.ts b/packages/catalog-model/src/kinds/TemplateEntityV1beta3.ts deleted file mode 100644 index 2da9a0b6b8..0000000000 --- a/packages/catalog-model/src/kinds/TemplateEntityV1beta3.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { JsonObject } from '@backstage/config'; -import type { Entity } from '../entity/Entity'; -import schema from '../schema/kinds/Template.v1beta3.schema.json'; -import { ajvCompiledJsonSchemaValidator } from './util'; - -/** @public */ -export interface TemplateEntityV1beta3 extends Entity { - apiVersion: 'backstage.io/v1beta3'; - kind: 'Template'; - spec: { - type: string; - parameters?: JsonObject | JsonObject[]; - steps: Array<{ - id?: string; - name?: string; - action: string; - input?: JsonObject; - if?: string | boolean; - }>; - output?: { [name: string]: string }; - owner?: string; - }; -} - -/** @public */ -export const templateEntityV1beta3Validator = - ajvCompiledJsonSchemaValidator(schema); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index 542cbb4b2f..3ecdd6dcd8 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -91,7 +91,7 @@ describe('DefaultWorkflowRunner', () => { it('should throw an error if the action does not exist', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', parameters: {}, output: {}, steps: [{ id: 'test', name: 'name', action: 'does-not-exist' }], @@ -105,7 +105,7 @@ describe('DefaultWorkflowRunner', () => { describe('validation', () => { it('should throw an error if the action has a schema and the input does not match', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', parameters: {}, output: {}, steps: [{ id: 'test', name: 'name', action: 'jest-validated-action' }], @@ -118,7 +118,7 @@ describe('DefaultWorkflowRunner', () => { it('should run the action when the validation passes', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', parameters: {}, output: {}, steps: [ @@ -140,7 +140,7 @@ describe('DefaultWorkflowRunner', () => { describe('conditionals', () => { it('should execute steps conditionally', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [ { id: 'test', name: 'test', action: 'output-action' }, { @@ -163,7 +163,7 @@ describe('DefaultWorkflowRunner', () => { it('should skips steps conditionally', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [ { id: 'test', name: 'test', action: 'output-action' }, { @@ -186,7 +186,7 @@ describe('DefaultWorkflowRunner', () => { it('should skips steps using the negating equals operator', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [ { id: 'test', name: 'test', action: 'output-action' }, { @@ -211,7 +211,7 @@ describe('DefaultWorkflowRunner', () => { describe('templating', () => { it('should template the input to an action', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [ { id: 'test', @@ -237,7 +237,7 @@ describe('DefaultWorkflowRunner', () => { it('should keep the original types for the input and not parse things that arent meant to be parsed', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [ { id: 'test', @@ -265,7 +265,7 @@ describe('DefaultWorkflowRunner', () => { it('should template complex values into the action', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [ { id: 'test', @@ -291,7 +291,7 @@ describe('DefaultWorkflowRunner', () => { it('supports really complex structures', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [ { id: 'test', @@ -320,7 +320,7 @@ describe('DefaultWorkflowRunner', () => { it('supports numbers as first class too', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [ { id: 'test', @@ -349,7 +349,7 @@ describe('DefaultWorkflowRunner', () => { it('should template the output from simple actions', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [ { id: 'test', @@ -373,7 +373,7 @@ describe('DefaultWorkflowRunner', () => { describe('filters', () => { it('provides the parseRepoUrl filter', async () => { const task = createMockTaskWithSpec({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [ { id: 'test', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 354092ee06..4a94b9bf73 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -49,7 +49,7 @@ type TemplateContext = { }; const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta3 => { - return taskSpec.apiVersion === 'backstage.io/v1beta3'; + return taskSpec.apiVersion === 'scaffolder.backstage.io/v1beta3'; }; const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index 035398d448..87a0229b5d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -92,7 +92,7 @@ describe('TaskWorker', () => { }); await broker.dispatch({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [{ id: 'test', name: 'test', action: 'not-found-action' }], output: { result: '{{ steps.test.output.testOutput }}', @@ -121,7 +121,7 @@ describe('TaskWorker', () => { }); const { taskId } = await broker.dispatch({ - apiVersion: 'backstage.io/v1beta3', + apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [{ id: 'test', name: 'test', action: 'not-found-action' }], output: { result: '{{ steps.test.output.testOutput }}', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 5b69ecc6df..7122c2fc3e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -38,7 +38,7 @@ export class TaskWorker { async runOneTask(task: Task) { try { const { output } = - task.spec.apiVersion === 'backstage.io/v1beta3' + task.spec.apiVersion === 'scaffolder.backstage.io/v1beta3' ? await this.options.runners.workflowRunner.execute(task) : await this.options.runners.legacyWorkflowRunner.execute(task); From ac29021b53c5730404c9aba291214683ed799151 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 11 Oct 2021 14:55:16 +0200 Subject: [PATCH 38/60] chore: re-running things Signed-off-by: blam --- .changeset/polite-timers-watch.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.changeset/polite-timers-watch.md b/.changeset/polite-timers-watch.md index d8f2ea80fb..dccae47437 100644 --- a/.changeset/polite-timers-watch.md +++ b/.changeset/polite-timers-watch.md @@ -1,7 +1,5 @@ --- -'@backstage/catalog-model': patch -'@backstage/plugin-catalog-backend': patch '@backstage/plugin-scaffolder-backend': patch --- -Introduce the new `backstage.io/v1beta3` template kind with nunjucks support 🥋 +Introduce the new `scaffolder.backstage.io/v1beta3` template kind with nunjucks support 🥋 From ca16be0611964b9e5e74c43b7f030929f13812e6 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 11 Oct 2021 15:23:29 +0200 Subject: [PATCH 39/60] chore: removing the older json schema not required anymore Signed-off-by: blam --- .../schema/kinds/Template.v1beta3.schema.json | 186 ------------------ 1 file changed, 186 deletions(-) delete mode 100644 packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json diff --git a/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json b/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json deleted file mode 100644 index e87a436d34..0000000000 --- a/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "TemplateV1beta3", - "description": "A Template describes a scaffolding task for use with the Scaffolder. It describes the required parameters as well as a series of steps that will be taken to execute the scaffolding task.", - "examples": [ - { - "apiVersion": "backstage.io/v1beta3", - "kind": "Template", - "metadata": { - "name": "react-ssr-template", - "title": "React SSR Template", - "description": "Next.js application skeleton for creating isomorphic web applications.", - "tags": ["recommended", "react"] - }, - "spec": { - "owner": "artist-relations-team", - "parameters": { - "required": ["name", "description", "repoUrl"], - "properties": { - "name": { - "title": "Name", - "type": "string", - "description": "Unique name of the component" - }, - "description": { - "title": "Description", - "type": "string", - "description": "Description of the component" - }, - "repoUrl": { - "title": "Pick a repository", - "type": "string", - "ui:field": "RepoUrlPicker" - } - } - }, - "steps": [ - { - "id": "fetch", - "name": "Fetch", - "action": "fetch:plain", - "parameters": { - "url": "./template" - } - }, - { - "id": "publish", - "name": "Publish to GitHub", - "action": "publish:github", - "parameters": { - "repoUrl": "${{ parameters.repoUrl }}" - }, - "if": "${{ parameters.repoUrl }}" - } - ], - "output": { - "catalogInfoUrl": "${{ steps.publish.output.catalogInfoUrl }}" - } - } - } - ], - "allOf": [ - { - "$ref": "Entity" - }, - { - "type": "object", - "required": ["spec"], - "properties": { - "apiVersion": { - "enum": ["backstage.io/v1beta3"] - }, - "kind": { - "enum": ["Template"] - }, - "spec": { - "type": "object", - "required": ["type", "steps"], - "properties": { - "type": { - "type": "string", - "description": "The type of component created by the template. The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, a website type component may present tooling in the Backstage interface that is specific to just websites.", - "examples": ["service", "website", "library"], - "minLength": 1 - }, - "parameters": { - "oneOf": [ - { - "type": "object", - "description": "The JSONSchema describing the inputs for the template." - }, - { - "type": "array", - "description": "A list of separate forms to collect parameters.", - "items": { - "type": "object", - "description": "The JSONSchema describing the inputs for the template." - } - } - ] - }, - "steps": { - "type": "array", - "description": "A list of steps to execute.", - "items": { - "type": "object", - "description": "A description of the step to execute.", - "required": ["action"], - "properties": { - "id": { - "type": "string", - "description": "The ID of the step, which can be used to refer to its outputs." - }, - "name": { - "type": "string", - "description": "The name of the step, which will be displayed in the UI during the scaffolding process." - }, - "action": { - "type": "string", - "description": "The name of the action to execute." - }, - "input": { - "type": "object", - "description": "A templated object describing the inputs to the action." - }, - "if": { - "type": ["string", "boolean"], - "description": "A templated condition that skips the step when evaluated to false. If the condition is true or not defined, the step is executed. The condition is true, if the input is not `false`, `undefined`, `null`, `\"\"`, `0`, or `[]`." - } - } - } - }, - "output": { - "type": "object", - "description": "A templated object describing the outputs of the scaffolding task.", - "properties": { - "links": { - "type": "array", - "description": "A list of external hyperlinks, typically pointing to resources created or updated by the template", - "items": { - "type": "object", - "required": [], - "properties": { - "url": { - "type": "string", - "description": "A url in a standard uri format.", - "examples": ["https://github.com/my-org/my-new-repo"], - "minLength": 1 - }, - "entityRef": { - "type": "string", - "description": "An entity reference to an entity in the catalog.", - "examples": ["Component:default/my-app"], - "minLength": 1 - }, - "title": { - "type": "string", - "description": "A user friendly display name for the link.", - "examples": ["View new repo"], - "minLength": 1 - }, - "icon": { - "type": "string", - "description": "A key representing a visual icon to be displayed in the UI.", - "examples": ["dashboard"], - "minLength": 1 - } - } - } - } - }, - "additionalProperties": { - "type": "string" - } - }, - "owner": { - "type": "string", - "description": "The user (or group) owner of the template", - "minLength": 1 - } - } - } - } - } - ] -} From 7fe6c0bb70232818e74d5bcd1845ab2627cbe6a5 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 12 Oct 2021 16:48:56 +0200 Subject: [PATCH 40/60] chore: reworking the docs a little bit Signed-off-by: blam --- .../migrating-from-v1beta2-to-v1beta3.md | 28 ++++++++++++------- .../fixtures/test-v1beta3/template.yaml | 4 ++- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md index 84d76b9fb1..280eab083d 100644 --- a/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md +++ b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md @@ -12,32 +12,43 @@ Well then, here we are! 🚀 Backstage has had many forms of templating languages throughout different plugins and different systems. We've had `cookiecutter` syntax in templates, and we also had `handlebars` templating in the `kind: Template`. Then we wanted to -remove the additional dependency on `cookiecutter` for `Software Templates` out -of the box, so we introduced `nunjucks` as an alternative in `fetch:template` +remove the additional dependency on `cookiecutter` for Software Templates out of +the box, so we introduced `nunjucks` as an alternative in `fetch:template` action which is based on the `jinja2` syntax so they're pretty similar. In an effort to reduce confusion and unify on to one templating language, we're officially deprecating support for `handlebars` templating in the -`kind: Template` entities with version `backstage.io/v1beta3` and moving to -using `nunjucks` instead. +`kind: Template` entities with `apiVersion` `scaffolder.backstage.io/v1beta3` +and moving to using `nunjucks` instead. This provides us a lot of built in `filters` (`handlebars` helpers), that as Template authors will give you much more flexibility out of the box, and also -open up sharing of filters in the `entity` and the actual `skeleton` too, and +open up sharing of filters in the Entity and the actual `skeleton` too, and removing the slight differences between the two languages. We've also removed a lot of the built in helpers that we shipped with `handlebars`, as they're now supported as first class citizens by either -`nunjucks` or the new `scaffolder` when using `backstage.io/v1beta3` +`nunjucks` or the new `scaffolder` when using `scaffolder.backstage.io/v1beta3` `apiVersion` The migration path is pretty simple, and we've removed some of the pain points from writing the `handlebars` templates too. Let's go through what's new and how to upgrade. +## `backstage.io/v1beta2` -> `scaffolder.backstage.io/v1beta3` + +The most important change is that you'll need to switch over the `apiVersion` in +your templates to the new one. + +```diff + kind: Template +- apiVersion: backstage.io/v1beta2 ++ apiVersion: scaffolder.backstage.io/v1beta3 +``` + ## `${{ }}` instead of `"{{ }}"` One really big readability issue and cause for confusion was the fact that with -`handlebars` and `yaml`you always had to wrap your templating strings in quotes +`handlebars` and `yaml` you always had to wrap your templating strings in quotes in `yaml` so that it didn't try to parse it as a `json` object and fail. This was pretty annoying, as it also meant that all things look like strings. Now that's no longer the case, you can now remove the `""` and take advantage of @@ -67,7 +78,6 @@ style operators. - if: '{{ eq parameters.value "backstage" }}' + if: ${{ parameters.value === "backstage" }} ... - ``` And then for the `not` @@ -79,7 +89,6 @@ And then for the `not` - if: '{{ not parameters.value "backstage" }}' + if: ${{ parameters.value !== "backstage" }} ... - ``` Much better right? ✨ @@ -118,7 +127,6 @@ input schema. + address: ${{ parameters.address }} - number: '{{ parameters.number }}' + number: ${{ parameters.number }} # this will now make sure that the type of number is a number 🙏 - ``` ## `parseRepoUrl` is now a `filter` diff --git a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml index 01bd5c2e4f..10d4a39ff7 100644 --- a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml +++ b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml @@ -1,4 +1,4 @@ -apiVersion: backstage.io/v1beta3 +apiVersion: scaffolder.backstage.io/v1beta3 kind: Template metadata: name: test-v1beta3 @@ -6,6 +6,7 @@ metadata: description: Test V1 Beta 3 Demo Templates spec: type: website + owner: team-a parameters: - name: Enter some stuff description: Enter some stuff @@ -16,6 +17,7 @@ spec: inputObject: type: object title: object input test + description: a little nested thing never hurt anyone right? properties: first: type: string From f71fda7600dbaba982b0aeda6b7ce739c0e5ca2f Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 14 Oct 2021 10:39:00 +0200 Subject: [PATCH 41/60] chore: removing double log Lines Signed-off-by: blam --- .../src/scaffolder/tasks/DefaultWorkflowRunner.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 4a94b9bf73..2ac309a1d4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -222,11 +222,6 @@ export class DefaultWorkflowRunner implements WorkflowRunner { const tmpDirs = new Array(); const stepOutput: { [outputName: string]: JsonValue } = {}; - await task.emitLog(`Beginning step ${step.name}`, { - stepId: step.id, - status: 'processing', - }); - await action.handler({ baseUrl: task.spec.baseUrl, input, From 54bbe25c344606dcc28a64e72774e705bc9b04d3 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Mon, 27 Sep 2021 11:32:24 +0200 Subject: [PATCH 42/60] Store the namespaced bucket storage for each instance that was created with `MockStorage.create()` instead of global variable Signed-off-by: Dominik Henneke --- .changeset/stupid-elephants-invent.md | 5 ++++ .../apis/StorageApi/MockStorageApi.test.ts | 17 +++++++++++-- .../apis/StorageApi/MockStorageApi.ts | 24 ++++++++++++------- 3 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 .changeset/stupid-elephants-invent.md diff --git a/.changeset/stupid-elephants-invent.md b/.changeset/stupid-elephants-invent.md new file mode 100644 index 0000000000..11b87a667d --- /dev/null +++ b/.changeset/stupid-elephants-invent.md @@ -0,0 +1,5 @@ +--- +'@backstage/test-utils': patch +--- + +Store the namespaced bucket storage for each instance that was created with `MockStorage.create()` instead of global variable. diff --git a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts index fabfa989e2..a3028f8b57 100644 --- a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts +++ b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { MockStorageApi } from './MockStorageApi'; import { StorageApi } from '@backstage/core-plugin-api'; +import { MockStorageApi } from './MockStorageApi'; describe('WebStorage Storage API', () => { const createMockStorage = (): StorageApi => { @@ -124,7 +124,7 @@ describe('WebStorage Storage API', () => { expect(secondStorage.get(keyName)).toBe('deerp'); }); - it('should not clash with other namesapces when creating buckets', async () => { + it('should not clash with other namespaces when creating buckets', async () => { const rootStorage = createMockStorage(); // when getting key test2 it will translate to /profile/something/deep/test2 @@ -139,4 +139,17 @@ describe('WebStorage Storage API', () => { expect(secondStorage.get('deep/test2')).toBe(undefined); }); + + it('should not reuse storage instances between different rootStorages', async () => { + const rootStorage1 = createMockStorage(); + const rootStorage2 = createMockStorage(); + + const firstStorage = rootStorage1.forBucket('something'); + const secondStorage = rootStorage2.forBucket('something'); + + await firstStorage.set('test2', true); + + expect(firstStorage.get('test2')).toBe(true); + expect(secondStorage.get('test2')).toBe(undefined); + }); }); diff --git a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts index 9f21b4e4c8..ab5d26afc5 100644 --- a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts +++ b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts @@ -23,29 +23,37 @@ import ObservableImpl from 'zen-observable'; export type MockStorageBucket = { [key: string]: any }; -const bucketStorageApis = new Map(); - export class MockStorageApi implements StorageApi { private readonly namespace: string; private readonly data: MockStorageBucket; + private readonly bucketStorageApis: Map; - private constructor(namespace: string, data?: MockStorageBucket) { + private constructor( + namespace: string, + bucketStorageApis: Map, + data?: MockStorageBucket, + ) { this.namespace = namespace; + this.bucketStorageApis = bucketStorageApis; this.data = { ...data }; } static create(data?: MockStorageBucket) { - return new MockStorageApi('', data); + return new MockStorageApi('', new Map(), data); } forBucket(name: string): StorageApi { - if (!bucketStorageApis.has(name)) { - bucketStorageApis.set( + if (!this.bucketStorageApis.has(name)) { + this.bucketStorageApis.set( name, - new MockStorageApi(`${this.namespace}/${name}`, this.data), + new MockStorageApi( + `${this.namespace}/${name}`, + this.bucketStorageApis, + this.data, + ), ); } - return bucketStorageApis.get(name)!; + return this.bucketStorageApis.get(name)!; } get(key: string): T | undefined { From 82fbda923e16a028a68fac9eb888ec7113a1636d Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Thu, 30 Sep 2021 15:09:38 +0200 Subject: [PATCH 43/60] Introduce a new `StarredEntitiesApi` that is used in the `useStarredEntities` hook Signed-off-by: Dominik Henneke --- .changeset/cool-deers-cough.md | 7 + .../components/catalog/EntityPage.test.tsx | 21 ++- .../ApiExplorerPage/ApiExplorerPage.test.tsx | 37 +++-- plugins/catalog-react/api-report.md | 40 +++++ plugins/catalog-react/package.json | 4 +- .../DefaultStarredEntitiesApi.test.ts | 145 ++++++++++++++++++ .../DefaultStarredEntitiesApi.ts | 116 ++++++++++++++ .../StarredEntitiesApi/StarredEntitiesApi.ts | 71 +++++++++ .../src/apis/StarredEntitiesApi/index.ts | 22 +++ plugins/catalog-react/src/apis/index.ts | 17 ++ .../src/hooks/useEntityListProvider.test.tsx | 30 ++-- .../src/hooks/useStarredEntities.test.tsx | 67 +++++--- .../src/hooks/useStarredEntities.ts | 50 ++---- plugins/catalog-react/src/index.ts | 1 + .../CatalogPage/CatalogPage.test.tsx | 41 +++-- .../CatalogTable/CatalogTable.test.tsx | 72 +++++---- .../EntityLayout/EntityLayout.test.tsx | 24 +-- plugins/catalog/src/plugin.ts | 10 ++ .../components/DefaultTechDocsHome.test.tsx | 19 ++- 19 files changed, 637 insertions(+), 157 deletions(-) create mode 100644 .changeset/cool-deers-cough.md create mode 100644 plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts create mode 100644 plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts create mode 100644 plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts create mode 100644 plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts create mode 100644 plugins/catalog-react/src/apis/index.ts diff --git a/.changeset/cool-deers-cough.md b/.changeset/cool-deers-cough.md new file mode 100644 index 0000000000..6d3e923d7d --- /dev/null +++ b/.changeset/cool-deers-cough.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-react': minor +'@backstage/plugin-catalog': patch +--- + +Introduce a new `StarredEntitiesApi` that is used in the `useStarredEntities` hook. +The `@backstage/plugin-catalog` installs a default implementation that is backed by the `StorageApi`, but one can also override the `starredEntitiesApiRef`. diff --git a/packages/app/src/components/catalog/EntityPage.test.tsx b/packages/app/src/components/catalog/EntityPage.test.tsx index 955e477f3c..089cb9c7cf 100644 --- a/packages/app/src/components/catalog/EntityPage.test.tsx +++ b/packages/app/src/components/catalog/EntityPage.test.tsx @@ -14,13 +14,17 @@ * limitations under the License. */ -import React from 'react'; -import { EntityLayout } from '@backstage/plugin-catalog'; -import { EntityProvider } from '@backstage/plugin-catalog-react'; -import { renderInTestApp } from '@backstage/test-utils'; -import { cicdContent } from './EntityPage'; -import { githubActionsApiRef } from '@backstage/plugin-github-actions'; import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; +import { EntityLayout } from '@backstage/plugin-catalog'; +import { + DefaultStarredEntitiesApi, + EntityProvider, + starredEntitiesApiRef, +} from '@backstage/plugin-catalog-react'; +import { githubActionsApiRef } from '@backstage/plugin-github-actions'; +import { MockStorageApi, renderInTestApp } from '@backstage/test-utils'; +import React from 'react'; +import { cicdContent } from './EntityPage'; describe('EntityPage Test', () => { const entity = { @@ -48,7 +52,10 @@ describe('EntityPage Test', () => { downloadJobLogsForWorkflowRun: jest.fn(), } as jest.Mocked; - const apis = ApiRegistry.with(githubActionsApiRef, mockedApi); + const apis = ApiRegistry.with(githubActionsApiRef, mockedApi).with( + starredEntitiesApiRef, + new DefaultStarredEntitiesApi({ storageApi: MockStorageApi.create() }), + ); describe('cicdContent', () => { it('Should render GitHub Actions View', async () => { diff --git a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx index d0b7523ead..292d03c838 100644 --- a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx +++ b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx @@ -15,30 +15,31 @@ */ import { Entity, RELATION_MEMBER_OF } from '@backstage/catalog-model'; -import { - CatalogApi, - catalogApiRef, - entityRouteRef, -} from '@backstage/plugin-catalog-react'; -import { MockStorageApi, wrapInTestApp } from '@backstage/test-utils'; -import { render } from '@testing-library/react'; -import React from 'react'; -import { apiDocsConfigRef } from '../../config'; -import { ApiExplorerPage } from './ApiExplorerPage'; - import { ApiProvider, ApiRegistry, ConfigReader, } from '@backstage/core-app-api'; +import { TableColumn, TableProps } from '@backstage/core-components'; import { - storageApiRef, ConfigApi, configApiRef, + storageApiRef, } from '@backstage/core-plugin-api'; -import { TableColumn, TableProps } from '@backstage/core-components'; -import DashboardIcon from '@material-ui/icons/Dashboard'; import { CatalogTableRow } from '@backstage/plugin-catalog'; +import { + CatalogApi, + catalogApiRef, + DefaultStarredEntitiesApi, + entityRouteRef, +} from '@backstage/plugin-catalog-react'; +import { starredEntitiesApiRef } from '@backstage/plugin-catalog-react/src/apis'; +import { MockStorageApi, wrapInTestApp } from '@backstage/test-utils'; +import DashboardIcon from '@material-ui/icons/Dashboard'; +import { render } from '@testing-library/react'; +import React from 'react'; +import { apiDocsConfigRef } from '../../config'; +import { ApiExplorerPage } from './ApiExplorerPage'; describe('ApiCatalogPage', () => { const catalogApi: Partial = { @@ -82,6 +83,8 @@ describe('ApiCatalogPage', () => { getApiDefinitionWidget: () => undefined, }; + const storageApi = MockStorageApi.create(); + const renderWrapped = (children: React.ReactNode) => render( wrapInTestApp( @@ -89,7 +92,11 @@ describe('ApiCatalogPage', () => { apis={ApiRegistry.from([ [catalogApiRef, catalogApi], [configApiRef, configApi], - [storageApiRef, MockStorageApi.create()], + [storageApiRef, storageApi], + [ + starredEntitiesApiRef, + new DefaultStarredEntitiesApi({ storageApi }), + ], [apiDocsConfigRef, apiDocsConfig], ])} > diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index ce7282276e..e176d56129 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -16,11 +16,13 @@ import { Entity } from '@backstage/catalog-model'; import { EntityName } from '@backstage/catalog-model'; import { IconButton } from '@material-ui/core'; import { LinkProps } from '@backstage/core-components'; +import { Observable } from '@backstage/core-plugin-api'; import { PropsWithChildren } from 'react'; import { default as React_2 } from 'react'; import { ReactNode } from 'react'; import { RouteRef } from '@backstage/core-plugin-api'; import { ScmIntegrationRegistry } from '@backstage/integration'; +import { StorageApi } from '@backstage/core-plugin-api'; import { SystemEntity } from '@backstage/catalog-model'; import { TableColumn } from '@backstage/core-components'; import { UserEntity } from '@backstage/catalog-model'; @@ -131,6 +133,23 @@ export type DefaultEntityFilters = { text?: EntityTextFilter; }; +// Warning: (ae-missing-release-tag) "DefaultStarredEntitiesApi" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class DefaultStarredEntitiesApi implements StarredEntitiesApi { + constructor(opts: { storageApi: StorageApi }); + // (undocumented) + isStarred(entity: Entity): boolean; + // (undocumented) + star(entity: Entity): Promise; + // (undocumented) + starredEntities$(): Observable; + // (undocumented) + toggleStarred(entity: Entity): Promise; + // (undocumented) + unstar(entity: Entity): Promise; +} + // Warning: (ae-forgotten-export) The symbol "EntityLoadingStatus" needs to be exported by the entry point index.d.ts // // @public @deprecated (undocumented) @@ -742,6 +761,27 @@ export function reduceEntityFilters( // @public (undocumented) export const rootRoute: RouteRef; +// @public +export interface StarredEntitiesApi { + star(entity: Entity): Promise; + starredEntities$(): Observable; + toggleStarred(entity: Entity): Promise; + unstar(entity: Entity): Promise; +} + +// Warning: (ae-missing-release-tag) "StarredEntitiesApiObservable" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type StarredEntitiesApiObservable = { + starredEntities: Set; + isStarred: (entity: Entity) => boolean; +}; + +// Warning: (ae-missing-release-tag) "starredEntitiesApiRef" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export const starredEntitiesApiRef: ApiRef; + // Warning: (ae-forgotten-export) The symbol "Props" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "UnregisterEntityDialog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/plugins/catalog-react/package.json b/plugins/catalog-react/package.json index f6b36a7191..9ddee679c3 100644 --- a/plugins/catalog-react/package.json +++ b/plugins/catalog-react/package.json @@ -45,7 +45,8 @@ "qs": "^6.9.4", "react": "^16.13.1", "react-router": "6.0.0-beta.0", - "react-use": "^17.2.4" + "react-use": "^17.2.4", + "zen-observable": "^0.8.15" }, "devDependencies": { "@backstage/cli": "^0.7.15", @@ -56,6 +57,7 @@ "@testing-library/user-event": "^13.1.8", "@types/jest": "^26.0.7", "@types/jwt-decode": "^3.1.0", + "@types/zen-observable": "^0.8.0", "cross-fetch": "^3.0.6", "react-test-renderer": "^16.13.1" }, diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts new file mode 100644 index 0000000000..6a4f90200b --- /dev/null +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts @@ -0,0 +1,145 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { StorageApi } from '@backstage/core-plugin-api'; +import { MockStorageApi } from '@backstage/test-utils'; +import { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; + +describe('DefaultStarredEntitiesApi', () => { + let mockStorage: StorageApi; + let starredEntitiesApi: DefaultStarredEntitiesApi; + + const mockEntity: Entity = { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'mock', + }, + }; + + beforeEach(() => { + mockStorage = MockStorageApi.create(); + starredEntitiesApi = new DefaultStarredEntitiesApi({ + storageApi: mockStorage, + }); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('toggleStarred', () => { + it('should star unstarred entity', async () => { + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); + + await starredEntitiesApi.toggleStarred(mockEntity); + + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); + }); + + it('should unstar starred entity', async () => { + const bucket = mockStorage.forBucket('settings'); + await bucket.set('starredEntities', ['entity:Component:default:mock']); + + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); + + await starredEntitiesApi.toggleStarred(mockEntity); + + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); + }); + }); + + describe('star', () => { + it('should star unstarred entity', async () => { + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); + + await starredEntitiesApi.star(mockEntity); + + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); + }); + + it('should keep starred entity', async () => { + const bucket = mockStorage.forBucket('settings'); + await bucket.set('starredEntities', ['entity:Component:default:mock']); + + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); + + await starredEntitiesApi.star(mockEntity); + + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); + }); + }); + + describe('unstar', () => { + it('should unstar starred entity', async () => { + const bucket = mockStorage.forBucket('settings'); + await bucket.set('starredEntities', ['entity:Component:default:mock']); + + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); + + await starredEntitiesApi.unstar(mockEntity); + + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); + }); + + it('should keep unstarred entity', async () => { + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); + + await starredEntitiesApi.unstar(mockEntity); + + expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); + }); + }); + + describe('starredEntities$', () => { + const handler = jest.fn(); + + beforeEach(async () => { + await new Promise(resolve => { + starredEntitiesApi.starredEntities$().subscribe({ + next: (...args) => { + handler(...args); + + if (handler.mock.calls.length >= 2) { + resolve(); + } + }, + }); + + const bucket = mockStorage.forBucket('settings'); + bucket.set('starredEntities', ['entity:Component:default:mock']).then(); + }); + }); + + it('should receive updates', async () => { + expect(handler).toBeCalledTimes(2); + expect(handler).toBeCalledWith({ + starredEntities: new Set(), + isStarred: expect.any(Function), + }); + expect(handler).toBeCalledWith({ + starredEntities: new Set(['entity:Component:default:mock']), + isStarred: expect.any(Function), + }); + }); + + it('should receive isStarred function that operates on the latest state', async () => { + expect(handler.mock.calls[0][0].isStarred(mockEntity)).toBe(true); + expect(handler.mock.calls[1][0].isStarred(mockEntity)).toBe(true); + }); + }); +}); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts new file mode 100644 index 0000000000..fce6541dca --- /dev/null +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -0,0 +1,116 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { Observable, StorageApi } from '@backstage/core-plugin-api'; +import ObservableImpl from 'zen-observable'; +import { + StarredEntitiesApi, + StarredEntitiesApiObservable, +} from './StarredEntitiesApi'; + +const buildEntityKey = (component: Entity) => + `entity:${component.kind}:${component.metadata.namespace ?? 'default'}:${ + component.metadata.name + }`; + +export class DefaultStarredEntitiesApi implements StarredEntitiesApi { + private readonly settingsStore: StorageApi; + private starredEntities: Set; + + constructor(opts: { storageApi: StorageApi }) { + this.settingsStore = opts.storageApi.forBucket('settings'); + + this.starredEntities = new Set( + this.settingsStore.get('starredEntities') ?? [], + ); + + this.settingsStore.observe$('starredEntities').subscribe({ + next: next => { + this.starredEntities = new Set(next.newValue ?? []); + this.notifyChanges(); + }, + }); + } + + async toggleStarred(entity: Entity): Promise { + const entityKey = buildEntityKey(entity); + + if (this.starredEntities.has(entityKey)) { + await this.unstar(entity); + } else { + await this.star(entity); + } + } + + async star(entity: Entity): Promise { + const entityKey = buildEntityKey(entity); + + this.starredEntities.add(entityKey); + + await this.settingsStore.set( + 'starredEntities', + Array.from(this.starredEntities), + ); + } + + async unstar(entity: Entity): Promise { + const entityKey = buildEntityKey(entity); + + this.starredEntities.delete(entityKey); + + await this.settingsStore.set( + 'starredEntities', + Array.from(this.starredEntities), + ); + } + + starredEntities$(): Observable { + return this.observable; + } + + isStarred(entity: Entity): boolean { + const entityKey = buildEntityKey(entity); + return this.starredEntities.has(entityKey); + } + + private readonly subscribers = new Set< + ZenObservable.SubscriptionObserver + >(); + + private readonly observable = + new ObservableImpl(subscriber => { + // forward the the latest value + subscriber.next({ + starredEntities: this.starredEntities, + isStarred: e => this.isStarred(e), + }); + + this.subscribers.add(subscriber); + return () => { + this.subscribers.delete(subscriber); + }; + }); + + private notifyChanges() { + for (const subscription of this.subscribers) { + subscription.next({ + starredEntities: this.starredEntities, + isStarred: e => this.isStarred(e), + }); + } + } +} diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts new file mode 100644 index 0000000000..bf29a425ed --- /dev/null +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { ApiRef, createApiRef, Observable } from '@backstage/core-plugin-api'; + +export const starredEntitiesApiRef: ApiRef = createApiRef({ + id: 'catalog-react.starred-entities', +}); + +export type StarredEntitiesApiObservable = { + /** + * A set of starred entities + */ + starredEntities: Set; + + /** + * A function to check if an entity is starred. + * + * @param entity - the entity to check + * @returns true, if the entity is starred. + */ + isStarred: (entity: Entity) => boolean; +}; + +/** + * An API to store and retrieve starred entities + * + * @public + */ +export interface StarredEntitiesApi { + /** + * Toggle the star state of the entity + * + * @param entity - the entity to be toggled + */ + toggleStarred(entity: Entity): Promise; + + /** + * Star the entity + * + * @param entity - the entity to be starred + */ + star(entity: Entity): Promise; + + /** + * Unstar the entity + * + * @param entity - the entity to be unstarred + */ + unstar(entity: Entity): Promise; + + /** + * Observe the state of starred entities and receive a handler + * to check the star state of an entity. + */ + starredEntities$(): Observable; +} diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts new file mode 100644 index 0000000000..c71a795b9e --- /dev/null +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; +export { starredEntitiesApiRef } from './StarredEntitiesApi'; +export type { + StarredEntitiesApi, + StarredEntitiesApiObservable, +} from './StarredEntitiesApi'; diff --git a/plugins/catalog-react/src/apis/index.ts b/plugins/catalog-react/src/apis/index.ts new file mode 100644 index 0000000000..5c7e980890 --- /dev/null +++ b/plugins/catalog-react/src/apis/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './StarredEntitiesApi'; diff --git a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx index 5d711016a4..e448ee73f1 100644 --- a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx +++ b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx @@ -14,21 +14,8 @@ * limitations under the License. */ -import React, { PropsWithChildren } from 'react'; -import qs from 'qs'; -import { act, renderHook } from '@testing-library/react-hooks'; -import { MockStorageApi } from '@backstage/test-utils'; import { CatalogApi } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; -import { - EntityListProvider, - useEntityListProvider, -} from './useEntityListProvider'; -import { catalogApiRef } from '../api'; -import { UserListFilterKind } from '../types'; -import { EntityKindFilter, EntityTypeFilter, UserListFilter } from '../filters'; -import { EntityKindPicker, UserListPicker } from '../components'; - import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; import { ConfigApi, @@ -37,6 +24,19 @@ import { identityApiRef, storageApiRef, } from '@backstage/core-plugin-api'; +import { MockStorageApi } from '@backstage/test-utils'; +import { act, renderHook } from '@testing-library/react-hooks'; +import qs from 'qs'; +import React, { PropsWithChildren } from 'react'; +import { catalogApiRef } from '../api'; +import { DefaultStarredEntitiesApi, starredEntitiesApiRef } from '../apis'; +import { EntityKindPicker, UserListPicker } from '../components'; +import { EntityKindFilter, EntityTypeFilter, UserListFilter } from '../filters'; +import { UserListFilterKind } from '../types'; +import { + EntityListProvider, + useEntityListProvider, +} from './useEntityListProvider'; const entities: Entity[] = [ { @@ -81,6 +81,10 @@ const apis = ApiRegistry.from([ [catalogApiRef, mockCatalogApi], [identityApiRef, mockIdentityApi], [storageApiRef, MockStorageApi.create()], + [ + starredEntitiesApiRef, + new DefaultStarredEntitiesApi({ storageApi: MockStorageApi.create() }), + ], ]); const wrapper = ({ diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx b/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx index 50662312da..0379e49351 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx +++ b/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx @@ -14,16 +14,18 @@ * limitations under the License. */ -import React, { PropsWithChildren } from 'react'; -import { renderHook, act } from '@testing-library/react-hooks'; -import { useStarredEntities } from './useStarredEntities'; -import { storageApiRef, StorageApi } from '@backstage/core-plugin-api'; -import { MockErrorApi } from '@backstage/test-utils'; import { Entity } from '@backstage/catalog-model'; -import { ApiProvider, ApiRegistry, WebStorage } from '@backstage/core-app-api'; +import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; +import { StorageApi } from '@backstage/core-plugin-api'; +import { MockStorageApi } from '@backstage/test-utils'; +import { act, renderHook } from '@testing-library/react-hooks'; +import React, { PropsWithChildren } from 'react'; +import { DefaultStarredEntitiesApi, starredEntitiesApiRef } from '../apis'; +import { useStarredEntities } from './useStarredEntities'; describe('useStarredEntities', () => { - let mockStorage: StorageApi | undefined; + let mockStorage: StorageApi; + let wrapper: React.ComponentType; const mockEntity: Entity = { apiVersion: '1', @@ -42,41 +44,56 @@ describe('useStarredEntities', () => { }, }; - const wrapper = ({ children }: PropsWithChildren<{}>) => { - return ( - + beforeEach(() => { + mockStorage = MockStorageApi.create(); + wrapper = ({ children }: PropsWithChildren<{}>) => ( + {children} ); - }; - - beforeEach(() => { - mockStorage = new WebStorage('@backstage', new MockErrorApi()).forBucket( - Date.now().toString(), // TODO(blam): need something that changes every test run for now until the MockStorage is implemented - ); }); + it('should return an empty set for when there is no items in storage', async () => { - const { result } = renderHook(() => useStarredEntities(), { wrapper }); + const { result, waitForNextUpdate } = renderHook( + () => useStarredEntities(), + { wrapper }, + ); + + await waitForNextUpdate(); expect(result.current.starredEntities.size).toBe(0); }); + it('should return a set with the current items when there are items in storage', async () => { const expectedIds = ['i', 'am', 'some', 'test', 'ids']; const store = mockStorage?.forBucket('settings'); await store?.set('starredEntities', expectedIds); - const { result } = renderHook(() => useStarredEntities(), { wrapper }); + const { result, waitForNextUpdate } = renderHook( + () => useStarredEntities(), + { wrapper }, + ); + + await waitForNextUpdate(); for (const item of expectedIds) { expect(result.current.starredEntities.has(item)).toBeTruthy(); } }); + it('should listen to changes when the storage is set elsewhere', async () => { const { result, waitForNextUpdate } = renderHook( () => useStarredEntities(), { wrapper }, ); + await waitForNextUpdate(); + expect(result.current.starredEntities.size).toBe(0); expect(result.current.isStarredEntity(mockEntity)).toBeFalsy(); @@ -91,18 +108,26 @@ describe('useStarredEntities', () => { }); it('should write new entries to the local store when adding a toggling entity', async () => { - const { result } = renderHook(() => useStarredEntities(), { wrapper }); + const { result, waitForNextUpdate } = renderHook( + () => useStarredEntities(), + { wrapper }, + ); act(() => { result.current.toggleStarredEntity(mockEntity); }); + await waitForNextUpdate(); + expect(result.current.isStarredEntity(mockEntity)).toBeTruthy(); expect(result.current.isStarredEntity(secondMockEntity)).toBeFalsy(); }); it('should remove an existing entity when toggling entries', async () => { - const { result } = renderHook(() => useStarredEntities(), { wrapper }); + const { result, waitForNextUpdate } = renderHook( + () => useStarredEntities(), + { wrapper }, + ); act(() => { result.current.toggleStarredEntity(mockEntity); @@ -110,6 +135,8 @@ describe('useStarredEntities', () => { result.current.toggleStarredEntity(mockEntity); }); + await waitForNextUpdate(); + expect(result.current.isStarredEntity(mockEntity)).toBeFalsy(); expect(result.current.isStarredEntity(secondMockEntity)).toBeTruthy(); }); diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.ts b/plugins/catalog-react/src/hooks/useStarredEntities.ts index 6f7b895fc8..56a0ee6c3c 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.ts +++ b/plugins/catalog-react/src/hooks/useStarredEntities.ts @@ -15,56 +15,24 @@ */ import { Entity } from '@backstage/catalog-model'; -import { storageApiRef, useApi } from '@backstage/core-plugin-api'; -import { useCallback, useEffect, useState } from 'react'; +import { useApi } from '@backstage/core-plugin-api'; +import { useCallback } from 'react'; import { useObservable } from 'react-use'; - -const buildEntityKey = (component: Entity) => - `entity:${component.kind}:${component.metadata.namespace ?? 'default'}:${ - component.metadata.name - }`; +import { starredEntitiesApiRef } from '../apis'; export const useStarredEntities = () => { - const storageApi = useApi(storageApiRef); - const settingsStore = storageApi.forBucket('settings'); - const rawStarredEntityKeys = - settingsStore.get('starredEntities') ?? []; + const starredEntitiesApi = useApi(starredEntitiesApiRef); - const [starredEntities, setStarredEntities] = useState( - new Set(rawStarredEntityKeys), + const { starredEntities, isStarred: isStarredEntity } = useObservable( + starredEntitiesApi.starredEntities$(), + { starredEntities: new Set(), isStarred: _ => false }, ); - const observedItems = useObservable( - settingsStore.observe$('starredEntities'), - ); - - useEffect(() => { - if (observedItems?.newValue) { - const currentValue = observedItems?.newValue ?? []; - setStarredEntities(new Set(currentValue)); - } - }, [observedItems?.newValue]); - const toggleStarredEntity = useCallback( (entity: Entity) => { - const entityKey = buildEntityKey(entity); - if (starredEntities.has(entityKey)) { - starredEntities.delete(entityKey); - } else { - starredEntities.add(entityKey); - } - - settingsStore.set('starredEntities', Array.from(starredEntities)); + starredEntitiesApi.toggleStarred(entity).then(); }, - [starredEntities, settingsStore], - ); - - const isStarredEntity = useCallback( - (entity: Entity) => { - const entityKey = buildEntityKey(entity); - return starredEntities.has(entityKey); - }, - [starredEntities], + [starredEntitiesApi], ); return { diff --git a/plugins/catalog-react/src/index.ts b/plugins/catalog-react/src/index.ts index 1ac67d514b..cf7a679f7a 100644 --- a/plugins/catalog-react/src/index.ts +++ b/plugins/catalog-react/src/index.ts @@ -23,6 +23,7 @@ export type { CatalogApi } from '@backstage/catalog-client'; export { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; export { catalogApiRef } from './api'; +export * from './apis'; export * from './components'; export * from './hooks'; export * from './filters'; diff --git a/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx b/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx index 48ec1ef176..6b33b33b08 100644 --- a/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx +++ b/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx @@ -20,28 +20,32 @@ import { RELATION_MEMBER_OF, RELATION_OWNED_BY, } from '@backstage/catalog-model'; -import { TableColumn, TableProps } from '@backstage/core-components'; -import { catalogApiRef, entityRouteRef } from '@backstage/plugin-catalog-react'; -import { - MockStorageApi, - renderWithEffects, - wrapInTestApp, - mockBreakpoint, -} from '@backstage/test-utils'; -import { fireEvent, screen } from '@testing-library/react'; -import React from 'react'; -import { createComponentRouteRef } from '../../routes'; -import { EntityRow } from '../CatalogTable/types'; -import { CatalogPage } from './CatalogPage'; -import DashboardIcon from '@material-ui/icons/Dashboard'; - import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; +import { TableColumn, TableProps } from '@backstage/core-components'; import { IdentityApi, identityApiRef, ProfileInfo, storageApiRef, } from '@backstage/core-plugin-api'; +import { + catalogApiRef, + DefaultStarredEntitiesApi, + entityRouteRef, + starredEntitiesApiRef, +} from '@backstage/plugin-catalog-react'; +import { + mockBreakpoint, + MockStorageApi, + renderWithEffects, + wrapInTestApp, +} from '@backstage/test-utils'; +import DashboardIcon from '@material-ui/icons/Dashboard'; +import { fireEvent, screen } from '@testing-library/react'; +import React from 'react'; +import { createComponentRouteRef } from '../../routes'; +import { EntityRow } from '../CatalogTable'; +import { CatalogPage } from './CatalogPage'; describe('CatalogPage', () => { const origReplaceState = window.history.replaceState; @@ -120,6 +124,7 @@ describe('CatalogPage', () => { getIdToken: async () => undefined, getProfile: () => testProfile, }; + const storageApi = MockStorageApi.create(); const renderWrapped = (children: React.ReactNode) => renderWithEffects( @@ -128,7 +133,11 @@ describe('CatalogPage', () => { apis={ApiRegistry.from([ [catalogApiRef, catalogApi], [identityApiRef, identityApi], - [storageApiRef, MockStorageApi.create()], + [storageApiRef, storageApi], + [ + starredEntitiesApiRef, + new DefaultStarredEntitiesApi({ storageApi }), + ], ])} > {children} diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx index 6c80e64fa9..822913a385 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx @@ -15,19 +15,22 @@ */ import { + EDIT_URL_ANNOTATION, Entity, VIEW_URL_ANNOTATION, - EDIT_URL_ANNOTATION, } from '@backstage/catalog-model'; -import { act, fireEvent } from '@testing-library/react'; -import { renderInTestApp } from '@backstage/test-utils'; -import * as React from 'react'; -import { CatalogTable } from './CatalogTable'; +import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; import { entityRouteRef, + DefaultStarredEntitiesApi, MockEntityListContextProvider, + starredEntitiesApiRef, UserListFilter, } from '@backstage/plugin-catalog-react'; +import { MockStorageApi, renderInTestApp } from '@backstage/test-utils'; +import { act, fireEvent } from '@testing-library/react'; +import * as React from 'react'; +import { CatalogTable } from './CatalogTable'; const entities: Entity[] = [ { @@ -48,6 +51,11 @@ const entities: Entity[] = [ ]; describe('CatalogTable component', () => { + const mockApis = ApiRegistry.with( + starredEntitiesApiRef, + new DefaultStarredEntitiesApi({ storageApi: MockStorageApi.create() }), + ); + beforeEach(() => { window.open = jest.fn(); }); @@ -58,9 +66,11 @@ describe('CatalogTable component', () => { it('should render error message', async () => { const rendered = await renderInTestApp( - - - , + + + + + , { mountedRoutes: { '/catalog/:namespace/:kind/:name': entityRouteRef, @@ -75,20 +85,22 @@ describe('CatalogTable component', () => { it('should display entity names when loading has finished and no error occurred', async () => { const rendered = await renderInTestApp( - false, - () => false, - ), - }, - }} - > - - , + + false, + () => false, + ), + }, + }} + > + + + , { mountedRoutes: { '/catalog/:namespace/:kind/:name': entityRouteRef, @@ -112,9 +124,11 @@ describe('CatalogTable component', () => { }; const { getByTitle } = await renderInTestApp( - - - , + + + + + , { mountedRoutes: { '/catalog/:namespace/:kind/:name': entityRouteRef, @@ -142,9 +156,11 @@ describe('CatalogTable component', () => { }; const { getByTitle } = await renderInTestApp( - - - , + + + + + , { mountedRoutes: { '/catalog/:namespace/:kind/:name': entityRouteRef, diff --git a/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx b/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx index 7722f6ba17..2216070984 100644 --- a/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx +++ b/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx @@ -13,24 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { CatalogApi } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; +import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; +import { AlertApi, alertApiRef } from '@backstage/core-plugin-api'; import { - catalogApiRef, - EntityProvider, AsyncEntityProvider, + catalogApiRef, + DefaultStarredEntitiesApi, + EntityProvider, entityRouteRef, + starredEntitiesApiRef, } from '@backstage/plugin-catalog-react'; -import { renderInTestApp } from '@backstage/test-utils'; +import { MockStorageApi, renderInTestApp } from '@backstage/test-utils'; import { fireEvent } from '@testing-library/react'; import React from 'react'; import { act } from 'react-dom/test-utils'; import { Route, Routes } from 'react-router'; import { EntityLayout } from './EntityLayout'; -import { AlertApi, alertApiRef } from '@backstage/core-plugin-api'; -import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; - const mockEntity = { kind: 'MyKind', metadata: { @@ -38,10 +40,12 @@ const mockEntity = { }, } as Entity; -const mockApis = ApiRegistry.with(catalogApiRef, {} as CatalogApi).with( - alertApiRef, - {} as AlertApi, -); +const mockApis = ApiRegistry.with(catalogApiRef, {} as CatalogApi) + .with(alertApiRef, {} as AlertApi) + .with( + starredEntitiesApiRef, + new DefaultStarredEntitiesApi({ storageApi: MockStorageApi.create() }), + ); describe('EntityLayout', () => { it('renders simplest case', async () => { diff --git a/plugins/catalog/src/plugin.ts b/plugins/catalog/src/plugin.ts index 9b90a44aff..20f937b2e7 100644 --- a/plugins/catalog/src/plugin.ts +++ b/plugins/catalog/src/plugin.ts @@ -18,7 +18,9 @@ import { CatalogClient } from '@backstage/catalog-client'; import { catalogApiRef, catalogRouteRef, + DefaultStarredEntitiesApi, entityRouteRef, + starredEntitiesApiRef, } from '@backstage/plugin-catalog-react'; import { CatalogClientWrapper } from './CatalogClientWrapper'; import { createComponentRouteRef, viewTechDocRouteRef } from './routes'; @@ -29,6 +31,7 @@ import { createRoutableExtension, discoveryApiRef, identityApiRef, + storageApiRef, } from '@backstage/core-plugin-api'; export const catalogPlugin = createPlugin({ @@ -43,6 +46,13 @@ export const catalogPlugin = createPlugin({ identityApi, }), }), + + createApiFactory({ + api: starredEntitiesApiRef, + deps: { storageApi: storageApiRef }, + factory: ({ storageApi }) => + new DefaultStarredEntitiesApi({ storageApi }), + }), ], routes: { catalogIndex: catalogRouteRef, diff --git a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx index 48ea67585b..0ba1f510cd 100644 --- a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx +++ b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx @@ -14,12 +14,6 @@ * limitations under the License. */ -import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog-react'; -import { MockStorageApi, renderInTestApp } from '@backstage/test-utils'; -import { screen } from '@testing-library/react'; -import React from 'react'; -import { DefaultTechDocsHome } from './DefaultTechDocsHome'; - import { ApiProvider, ApiRegistry, @@ -30,7 +24,17 @@ import { configApiRef, storageApiRef, } from '@backstage/core-plugin-api'; +import { + CatalogApi, + catalogApiRef, + DefaultStarredEntitiesApi, + starredEntitiesApiRef, +} from '@backstage/plugin-catalog-react'; +import { MockStorageApi, renderInTestApp } from '@backstage/test-utils'; +import { screen } from '@testing-library/react'; +import React from 'react'; import { rootDocsRouteRef } from '../../routes'; +import { DefaultTechDocsHome } from './DefaultTechDocsHome'; jest.mock('@backstage/plugin-catalog-react', () => { const actual = jest.requireActual('@backstage/plugin-catalog-react'); @@ -63,10 +67,13 @@ describe('TechDocs Home', () => { }, }); + const storageApi = MockStorageApi.create(); + const apiRegistry = ApiRegistry.from([ [catalogApiRef, mockCatalogApi], [configApiRef, configApi], [storageApiRef, MockStorageApi.create()], + [starredEntitiesApiRef, new DefaultStarredEntitiesApi({ storageApi })], ]); it('should render a TechDocs home page', async () => { From 126daa5ec34391b4e2a4389981a188b30425b53d Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Mon, 11 Oct 2021 11:22:13 +0200 Subject: [PATCH 44/60] Resolve PR comments * rename starredEntities$ to starredEntitie$ * delete star and unstar from the public interface Signed-off-by: Dominik Henneke --- plugins/catalog-react/api-report.md | 6 ++---- .../DefaultStarredEntitiesApi.test.ts | 2 +- .../DefaultStarredEntitiesApi.ts | 2 +- .../StarredEntitiesApi/StarredEntitiesApi.ts | 16 +--------------- .../src/hooks/useStarredEntities.ts | 2 +- 5 files changed, 6 insertions(+), 22 deletions(-) diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index e176d56129..0accacce3e 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -143,7 +143,7 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { // (undocumented) star(entity: Entity): Promise; // (undocumented) - starredEntities$(): Observable; + starredEntitie$(): Observable; // (undocumented) toggleStarred(entity: Entity): Promise; // (undocumented) @@ -763,10 +763,8 @@ export const rootRoute: RouteRef; // @public export interface StarredEntitiesApi { - star(entity: Entity): Promise; - starredEntities$(): Observable; + starredEntitie$(): Observable; toggleStarred(entity: Entity): Promise; - unstar(entity: Entity): Promise; } // Warning: (ae-missing-release-tag) "StarredEntitiesApiObservable" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts index 6a4f90200b..c182bdad39 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts @@ -110,7 +110,7 @@ describe('DefaultStarredEntitiesApi', () => { beforeEach(async () => { await new Promise(resolve => { - starredEntitiesApi.starredEntities$().subscribe({ + starredEntitiesApi.starredEntitie$().subscribe({ next: (...args) => { handler(...args); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts index fce6541dca..6080cc4e39 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -78,7 +78,7 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { ); } - starredEntities$(): Observable { + starredEntitie$(): Observable { return this.observable; } diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts index bf29a425ed..f64298227e 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts @@ -49,23 +49,9 @@ export interface StarredEntitiesApi { */ toggleStarred(entity: Entity): Promise; - /** - * Star the entity - * - * @param entity - the entity to be starred - */ - star(entity: Entity): Promise; - - /** - * Unstar the entity - * - * @param entity - the entity to be unstarred - */ - unstar(entity: Entity): Promise; - /** * Observe the state of starred entities and receive a handler * to check the star state of an entity. */ - starredEntities$(): Observable; + starredEntitie$(): Observable; } diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.ts b/plugins/catalog-react/src/hooks/useStarredEntities.ts index 56a0ee6c3c..cb94d465bc 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.ts +++ b/plugins/catalog-react/src/hooks/useStarredEntities.ts @@ -24,7 +24,7 @@ export const useStarredEntities = () => { const starredEntitiesApi = useApi(starredEntitiesApiRef); const { starredEntities, isStarred: isStarredEntity } = useObservable( - starredEntitiesApi.starredEntities$(), + starredEntitiesApi.starredEntitie$(), { starredEntities: new Set(), isStarred: _ => false }, ); From 8f55fb815b5e9ba8d75c96b1b9bf504c40a0ff3d Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Tue, 12 Oct 2021 10:37:50 +0200 Subject: [PATCH 45/60] Fix test Signed-off-by: Dominik Henneke --- .../techdocs/src/home/components/DefaultTechDocsHome.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx index 0ba1f510cd..1d8d7d2b04 100644 --- a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx +++ b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx @@ -72,7 +72,7 @@ describe('TechDocs Home', () => { const apiRegistry = ApiRegistry.from([ [catalogApiRef, mockCatalogApi], [configApiRef, configApi], - [storageApiRef, MockStorageApi.create()], + [storageApiRef, storageApi], [starredEntitiesApiRef, new DefaultStarredEntitiesApi({ storageApi })], ]); From 56f0ef983e6bcaa1221beaa9cdfd1a94d51db1ee Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Tue, 12 Oct 2021 10:16:30 +0200 Subject: [PATCH 46/60] Remove the star and unstar method from DefaultStarredEntitiesApi Signed-off-by: Dominik Henneke --- plugins/catalog-react/api-report.md | 4 -- .../DefaultStarredEntitiesApi.test.ts | 42 ------------------- .../DefaultStarredEntitiesApi.ts | 21 +--------- 3 files changed, 2 insertions(+), 65 deletions(-) diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 0accacce3e..12ae143d8a 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -141,13 +141,9 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { // (undocumented) isStarred(entity: Entity): boolean; // (undocumented) - star(entity: Entity): Promise; - // (undocumented) starredEntitie$(): Observable; // (undocumented) toggleStarred(entity: Entity): Promise; - // (undocumented) - unstar(entity: Entity): Promise; } // Warning: (ae-forgotten-export) The symbol "EntityLoadingStatus" needs to be exported by the entry point index.d.ts diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts index c182bdad39..1edcdf706b 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts @@ -63,48 +63,6 @@ describe('DefaultStarredEntitiesApi', () => { }); }); - describe('star', () => { - it('should star unstarred entity', async () => { - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); - - await starredEntitiesApi.star(mockEntity); - - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); - }); - - it('should keep starred entity', async () => { - const bucket = mockStorage.forBucket('settings'); - await bucket.set('starredEntities', ['entity:Component:default:mock']); - - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); - - await starredEntitiesApi.star(mockEntity); - - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); - }); - }); - - describe('unstar', () => { - it('should unstar starred entity', async () => { - const bucket = mockStorage.forBucket('settings'); - await bucket.set('starredEntities', ['entity:Component:default:mock']); - - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); - - await starredEntitiesApi.unstar(mockEntity); - - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); - }); - - it('should keep unstarred entity', async () => { - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); - - await starredEntitiesApi.unstar(mockEntity); - - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); - }); - }); - describe('starredEntities$', () => { const handler = jest.fn(); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts index 6080cc4e39..4c5ea63cf9 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -50,27 +50,10 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { const entityKey = buildEntityKey(entity); if (this.starredEntities.has(entityKey)) { - await this.unstar(entity); + this.starredEntities.delete(entityKey); } else { - await this.star(entity); + this.starredEntities.add(entityKey); } - } - - async star(entity: Entity): Promise { - const entityKey = buildEntityKey(entity); - - this.starredEntities.add(entityKey); - - await this.settingsStore.set( - 'starredEntities', - Array.from(this.starredEntities), - ); - } - - async unstar(entity: Entity): Promise { - const entityKey = buildEntityKey(entity); - - this.starredEntities.delete(entityKey); await this.settingsStore.set( 'starredEntities', From 615f668d317afbce4466f98cc811afa218e31c06 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Tue, 12 Oct 2021 11:02:30 +0200 Subject: [PATCH 47/60] Use entity refs as a storage format instead of the custom one Signed-off-by: Dominik Henneke --- .changeset/cool-deers-cough.md | 2 ++ plugins/catalog-react/api-report.md | 4 +--- .../DefaultStarredEntitiesApi.test.ts | 6 +++--- .../DefaultStarredEntitiesApi.ts | 16 ++++++++-------- .../StarredEntitiesApi/StarredEntitiesApi.ts | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.changeset/cool-deers-cough.md b/.changeset/cool-deers-cough.md index 6d3e923d7d..c2619d1fa2 100644 --- a/.changeset/cool-deers-cough.md +++ b/.changeset/cool-deers-cough.md @@ -5,3 +5,5 @@ Introduce a new `StarredEntitiesApi` that is used in the `useStarredEntities` hook. The `@backstage/plugin-catalog` installs a default implementation that is backed by the `StorageApi`, but one can also override the `starredEntitiesApiRef`. + +**BREAKING** All previously stored entities get lost with this update, because the storage format has been migrated from a custom string to an entity reference. diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 12ae143d8a..ea4c51adda 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -133,9 +133,7 @@ export type DefaultEntityFilters = { text?: EntityTextFilter; }; -// Warning: (ae-missing-release-tag) "DefaultStarredEntitiesApi" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export class DefaultStarredEntitiesApi implements StarredEntitiesApi { constructor(opts: { storageApi: StorageApi }); // (undocumented) diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts index 1edcdf706b..eec484863f 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts @@ -53,7 +53,7 @@ describe('DefaultStarredEntitiesApi', () => { it('should unstar starred entity', async () => { const bucket = mockStorage.forBucket('settings'); - await bucket.set('starredEntities', ['entity:Component:default:mock']); + await bucket.set('starredEntities', ['component:default/mock']); expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); @@ -79,7 +79,7 @@ describe('DefaultStarredEntitiesApi', () => { }); const bucket = mockStorage.forBucket('settings'); - bucket.set('starredEntities', ['entity:Component:default:mock']).then(); + bucket.set('starredEntities', ['component:default/mock']).then(); }); }); @@ -90,7 +90,7 @@ describe('DefaultStarredEntitiesApi', () => { isStarred: expect.any(Function), }); expect(handler).toBeCalledWith({ - starredEntities: new Set(['entity:Component:default:mock']), + starredEntities: new Set(['component:default/mock']), isStarred: expect.any(Function), }); }); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts index 4c5ea63cf9..5783954fbc 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Observable, StorageApi } from '@backstage/core-plugin-api'; import ObservableImpl from 'zen-observable'; import { @@ -22,11 +22,11 @@ import { StarredEntitiesApiObservable, } from './StarredEntitiesApi'; -const buildEntityKey = (component: Entity) => - `entity:${component.kind}:${component.metadata.namespace ?? 'default'}:${ - component.metadata.name - }`; - +/** + * Default implementation of the StarredEntitiesApi that is backed by the StorageApi. + * + * @public + */ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { private readonly settingsStore: StorageApi; private starredEntities: Set; @@ -47,7 +47,7 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { } async toggleStarred(entity: Entity): Promise { - const entityKey = buildEntityKey(entity); + const entityKey = stringifyEntityRef(entity); if (this.starredEntities.has(entityKey)) { this.starredEntities.delete(entityKey); @@ -66,7 +66,7 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { } isStarred(entity: Entity): boolean { - const entityKey = buildEntityKey(entity); + const entityKey = stringifyEntityRef(entity); return this.starredEntities.has(entityKey); } diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts index f64298227e..0c8ec52445 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts @@ -23,7 +23,7 @@ export const starredEntitiesApiRef: ApiRef = createApiRef({ export type StarredEntitiesApiObservable = { /** - * A set of starred entities + * A set of entity references that are starred */ starredEntities: Set; From 78fcb898c492001e1ebbabe2b8e2f7fe9a67c2e5 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Tue, 12 Oct 2021 10:37:07 +0200 Subject: [PATCH 48/60] Rename the storage location of the starred entities from `/settings/starredEntities` to `/starredEntitites/entitifyRefs` Signed-off-by: Dominik Henneke --- .changeset/cool-deers-cough.md | 5 ++++- .../StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts | 8 ++++---- .../apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts | 8 ++++---- .../catalog-react/src/hooks/useStarredEntities.test.tsx | 4 ++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.changeset/cool-deers-cough.md b/.changeset/cool-deers-cough.md index c2619d1fa2..0c07c91b96 100644 --- a/.changeset/cool-deers-cough.md +++ b/.changeset/cool-deers-cough.md @@ -6,4 +6,7 @@ Introduce a new `StarredEntitiesApi` that is used in the `useStarredEntities` hook. The `@backstage/plugin-catalog` installs a default implementation that is backed by the `StorageApi`, but one can also override the `starredEntitiesApiRef`. -**BREAKING** All previously stored entities get lost with this update, because the storage format has been migrated from a custom string to an entity reference. +**BREAKING** All previously stored entities get lost with this update, because: + +1. The storage format has been migrated from a custom string to an entity reference. +2. The storage key in the local storage has been changed. diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts index eec484863f..95375aa0f6 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts @@ -52,8 +52,8 @@ describe('DefaultStarredEntitiesApi', () => { }); it('should unstar starred entity', async () => { - const bucket = mockStorage.forBucket('settings'); - await bucket.set('starredEntities', ['component:default/mock']); + const bucket = mockStorage.forBucket('starredEntities'); + await bucket.set('entityRefs', ['component:default/mock']); expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); @@ -78,8 +78,8 @@ describe('DefaultStarredEntitiesApi', () => { }, }); - const bucket = mockStorage.forBucket('settings'); - bucket.set('starredEntities', ['component:default/mock']).then(); + const bucket = mockStorage.forBucket('starredEntities'); + bucket.set('entityRefs', ['component:default/mock']).then(); }); }); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts index 5783954fbc..fa775679ca 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -32,13 +32,13 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { private starredEntities: Set; constructor(opts: { storageApi: StorageApi }) { - this.settingsStore = opts.storageApi.forBucket('settings'); + this.settingsStore = opts.storageApi.forBucket('starredEntities'); this.starredEntities = new Set( - this.settingsStore.get('starredEntities') ?? [], + this.settingsStore.get('entityRefs') ?? [], ); - this.settingsStore.observe$('starredEntities').subscribe({ + this.settingsStore.observe$('entityRefs').subscribe({ next: next => { this.starredEntities = new Set(next.newValue ?? []); this.notifyChanges(); @@ -56,7 +56,7 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { } await this.settingsStore.set( - 'starredEntities', + 'entityRefs', Array.from(this.starredEntities), ); } diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx b/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx index 0379e49351..4f41ef090f 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx +++ b/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx @@ -71,8 +71,8 @@ describe('useStarredEntities', () => { it('should return a set with the current items when there are items in storage', async () => { const expectedIds = ['i', 'am', 'some', 'test', 'ids']; - const store = mockStorage?.forBucket('settings'); - await store?.set('starredEntities', expectedIds); + const store = mockStorage?.forBucket('starredEntities'); + await store?.set('entityRefs', expectedIds); const { result, waitForNextUpdate } = renderHook( () => useStarredEntities(), From c1d50273de5674f1e211cd148c15366eb323741d Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Tue, 12 Oct 2021 10:54:49 +0200 Subject: [PATCH 49/60] Use entity references in the StarredEntitiesApi Signed-off-by: Dominik Henneke --- plugins/catalog-react/api-report.md | 8 +++---- .../DefaultStarredEntitiesApi.test.ts | 22 +++++++++---------- .../DefaultStarredEntitiesApi.ts | 16 +++++--------- .../StarredEntitiesApi/StarredEntitiesApi.ts | 9 ++++---- .../src/hooks/useStarredEntities.ts | 11 +++++++--- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index ea4c51adda..8670830584 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -137,11 +137,11 @@ export type DefaultEntityFilters = { export class DefaultStarredEntitiesApi implements StarredEntitiesApi { constructor(opts: { storageApi: StorageApi }); // (undocumented) - isStarred(entity: Entity): boolean; + isStarred(entityRef: string): boolean; // (undocumented) starredEntitie$(): Observable; // (undocumented) - toggleStarred(entity: Entity): Promise; + toggleStarred(entityRef: string): Promise; } // Warning: (ae-forgotten-export) The symbol "EntityLoadingStatus" needs to be exported by the entry point index.d.ts @@ -758,7 +758,7 @@ export const rootRoute: RouteRef; // @public export interface StarredEntitiesApi { starredEntitie$(): Observable; - toggleStarred(entity: Entity): Promise; + toggleStarred(entityRef: string): Promise; } // Warning: (ae-missing-release-tag) "StarredEntitiesApiObservable" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -766,7 +766,7 @@ export interface StarredEntitiesApi { // @public (undocumented) export type StarredEntitiesApiObservable = { starredEntities: Set; - isStarred: (entity: Entity) => boolean; + isStarred: (entityRef: string) => boolean; }; // Warning: (ae-missing-release-tag) "starredEntitiesApiRef" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts index 95375aa0f6..e4f7c4ffa1 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; +import { stringifyEntityRef } from '@backstage/catalog-model'; import { StorageApi } from '@backstage/core-plugin-api'; import { MockStorageApi } from '@backstage/test-utils'; import { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; @@ -23,13 +23,13 @@ describe('DefaultStarredEntitiesApi', () => { let mockStorage: StorageApi; let starredEntitiesApi: DefaultStarredEntitiesApi; - const mockEntity: Entity = { + const mockEntityRef = stringifyEntityRef({ apiVersion: '1', kind: 'Component', metadata: { name: 'mock', }, - }; + }); beforeEach(() => { mockStorage = MockStorageApi.create(); @@ -44,22 +44,22 @@ describe('DefaultStarredEntitiesApi', () => { describe('toggleStarred', () => { it('should star unstarred entity', async () => { - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); + expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(false); - await starredEntitiesApi.toggleStarred(mockEntity); + await starredEntitiesApi.toggleStarred(mockEntityRef); - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); + expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(true); }); it('should unstar starred entity', async () => { const bucket = mockStorage.forBucket('starredEntities'); await bucket.set('entityRefs', ['component:default/mock']); - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(true); + expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(true); - await starredEntitiesApi.toggleStarred(mockEntity); + await starredEntitiesApi.toggleStarred(mockEntityRef); - expect(starredEntitiesApi.isStarred(mockEntity)).toBe(false); + expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(false); }); }); @@ -96,8 +96,8 @@ describe('DefaultStarredEntitiesApi', () => { }); it('should receive isStarred function that operates on the latest state', async () => { - expect(handler.mock.calls[0][0].isStarred(mockEntity)).toBe(true); - expect(handler.mock.calls[1][0].isStarred(mockEntity)).toBe(true); + expect(handler.mock.calls[0][0].isStarred(mockEntityRef)).toBe(true); + expect(handler.mock.calls[1][0].isStarred(mockEntityRef)).toBe(true); }); }); }); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts index fa775679ca..8486d180b7 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Observable, StorageApi } from '@backstage/core-plugin-api'; import ObservableImpl from 'zen-observable'; import { @@ -46,13 +45,11 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { }); } - async toggleStarred(entity: Entity): Promise { - const entityKey = stringifyEntityRef(entity); - - if (this.starredEntities.has(entityKey)) { - this.starredEntities.delete(entityKey); + async toggleStarred(entityRef: string): Promise { + if (this.starredEntities.has(entityRef)) { + this.starredEntities.delete(entityRef); } else { - this.starredEntities.add(entityKey); + this.starredEntities.add(entityRef); } await this.settingsStore.set( @@ -65,9 +62,8 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { return this.observable; } - isStarred(entity: Entity): boolean { - const entityKey = stringifyEntityRef(entity); - return this.starredEntities.has(entityKey); + isStarred(entityRef: string): boolean { + return this.starredEntities.has(entityRef); } private readonly subscribers = new Set< diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts index 0c8ec52445..68eae0b0a1 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; import { ApiRef, createApiRef, Observable } from '@backstage/core-plugin-api'; export const starredEntitiesApiRef: ApiRef = createApiRef({ @@ -30,10 +29,10 @@ export type StarredEntitiesApiObservable = { /** * A function to check if an entity is starred. * - * @param entity - the entity to check + * @param entityRef - an entity reference to check * @returns true, if the entity is starred. */ - isStarred: (entity: Entity) => boolean; + isStarred: (entityRef: string) => boolean; }; /** @@ -45,9 +44,9 @@ export interface StarredEntitiesApi { /** * Toggle the star state of the entity * - * @param entity - the entity to be toggled + * @param entityRef - an entity reference to toggle */ - toggleStarred(entity: Entity): Promise; + toggleStarred(entityRef: string): Promise; /** * Observe the state of starred entities and receive a handler diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.ts b/plugins/catalog-react/src/hooks/useStarredEntities.ts index cb94d465bc..e79e605877 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.ts +++ b/plugins/catalog-react/src/hooks/useStarredEntities.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { useApi } from '@backstage/core-plugin-api'; import { useCallback } from 'react'; import { useObservable } from 'react-use'; @@ -23,14 +23,19 @@ import { starredEntitiesApiRef } from '../apis'; export const useStarredEntities = () => { const starredEntitiesApi = useApi(starredEntitiesApiRef); - const { starredEntities, isStarred: isStarredEntity } = useObservable( + const { starredEntities, isStarred } = useObservable( starredEntitiesApi.starredEntitie$(), { starredEntities: new Set(), isStarred: _ => false }, ); + const isStarredEntity = useCallback( + (entity: Entity) => isStarred(stringifyEntityRef(entity)), + [isStarred], + ); + const toggleStarredEntity = useCallback( (entity: Entity) => { - starredEntitiesApi.toggleStarred(entity).then(); + starredEntitiesApi.toggleStarred(stringifyEntityRef(entity)).then(); }, [starredEntitiesApi], ); From bd924fece8b82161dc2e84b079dced5223aa81de Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Tue, 12 Oct 2021 11:09:51 +0200 Subject: [PATCH 50/60] Add additional docs Signed-off-by: Dominik Henneke --- plugins/catalog-react/api-report.md | 6 +----- .../src/apis/StarredEntitiesApi/StarredEntitiesApi.ts | 8 ++++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 8670830584..1a37f84df3 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -761,17 +761,13 @@ export interface StarredEntitiesApi { toggleStarred(entityRef: string): Promise; } -// Warning: (ae-missing-release-tag) "StarredEntitiesApiObservable" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public (undocumented) export type StarredEntitiesApiObservable = { starredEntities: Set; isStarred: (entityRef: string) => boolean; }; -// Warning: (ae-missing-release-tag) "starredEntitiesApiRef" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const starredEntitiesApiRef: ApiRef; // Warning: (ae-forgotten-export) The symbol "Props" needs to be exported by the entry point index.d.ts diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts index 68eae0b0a1..93ff55aa54 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts @@ -16,10 +16,18 @@ import { ApiRef, createApiRef, Observable } from '@backstage/core-plugin-api'; +/** + * An API to store starred entities + * + * @public + */ export const starredEntitiesApiRef: ApiRef = createApiRef({ id: 'catalog-react.starred-entities', }); +/** + * @public + */ export type StarredEntitiesApiObservable = { /** * A set of entity references that are starred From 1dbaaf7801070c208bcad17ef531c85dbad753f2 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Wed, 13 Oct 2021 12:14:10 +0200 Subject: [PATCH 51/60] Simplify the StarredEntitiesApi interface Signed-off-by: Dominik Henneke --- plugins/catalog-react/api-report.md | 16 +++------ .../DefaultStarredEntitiesApi.test.ts | 15 ++------ .../DefaultStarredEntitiesApi.ts | 34 +++++++------------ .../StarredEntitiesApi/StarredEntitiesApi.ts | 23 ++----------- .../src/apis/StarredEntitiesApi/index.ts | 5 +-- .../src/hooks/useStarredEntities.ts | 34 +++++++++++++------ 6 files changed, 46 insertions(+), 81 deletions(-) diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 1a37f84df3..2486cdd6e8 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -139,7 +139,7 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { // (undocumented) isStarred(entityRef: string): boolean; // (undocumented) - starredEntitie$(): Observable; + starredEntitie$(): Observable>; // (undocumented) toggleStarred(entityRef: string): Promise; } @@ -757,16 +757,10 @@ export const rootRoute: RouteRef; // @public export interface StarredEntitiesApi { - starredEntitie$(): Observable; + starredEntitie$(): Observable>; toggleStarred(entityRef: string): Promise; } -// @public (undocumented) -export type StarredEntitiesApiObservable = { - starredEntities: Set; - isStarred: (entityRef: string) => boolean; -}; - // @public export const starredEntitiesApiRef: ApiRef; @@ -894,10 +888,10 @@ export const UserListPicker: ({ // Warning: (ae-missing-release-tag) "useStarredEntities" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const useStarredEntities: () => { +export function useStarredEntities(): { starredEntities: Set; - toggleStarredEntity: (entity: Entity) => void; - isStarredEntity: (entity: Entity) => boolean; + toggleStarredEntity: (entityOrRef: Entity | EntityName | string) => void; + isStarredEntity: (entityOrRef: Entity | EntityName | string) => boolean; }; // Warnings were encountered during analysis: diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts index e4f7c4ffa1..be67b03dbb 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts @@ -85,19 +85,8 @@ describe('DefaultStarredEntitiesApi', () => { it('should receive updates', async () => { expect(handler).toBeCalledTimes(2); - expect(handler).toBeCalledWith({ - starredEntities: new Set(), - isStarred: expect.any(Function), - }); - expect(handler).toBeCalledWith({ - starredEntities: new Set(['component:default/mock']), - isStarred: expect.any(Function), - }); - }); - - it('should receive isStarred function that operates on the latest state', async () => { - expect(handler.mock.calls[0][0].isStarred(mockEntityRef)).toBe(true); - expect(handler.mock.calls[1][0].isStarred(mockEntityRef)).toBe(true); + expect(handler).toBeCalledWith(new Set()); + expect(handler).toBeCalledWith(new Set(['component:default/mock'])); }); }); }); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts index 8486d180b7..92a0eaebf2 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -16,10 +16,7 @@ import { Observable, StorageApi } from '@backstage/core-plugin-api'; import ObservableImpl from 'zen-observable'; -import { - StarredEntitiesApi, - StarredEntitiesApiObservable, -} from './StarredEntitiesApi'; +import { StarredEntitiesApi } from './StarredEntitiesApi'; /** * Default implementation of the StarredEntitiesApi that is backed by the StorageApi. @@ -58,7 +55,7 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { ); } - starredEntitie$(): Observable { + starredEntitie$(): Observable> { return this.observable; } @@ -67,29 +64,22 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { } private readonly subscribers = new Set< - ZenObservable.SubscriptionObserver + ZenObservable.SubscriptionObserver> >(); - private readonly observable = - new ObservableImpl(subscriber => { - // forward the the latest value - subscriber.next({ - starredEntities: this.starredEntities, - isStarred: e => this.isStarred(e), - }); + private readonly observable = new ObservableImpl>(subscriber => { + // forward the the latest value + subscriber.next(this.starredEntities); - this.subscribers.add(subscriber); - return () => { - this.subscribers.delete(subscriber); - }; - }); + this.subscribers.add(subscriber); + return () => { + this.subscribers.delete(subscriber); + }; + }); private notifyChanges() { for (const subscription of this.subscribers) { - subscription.next({ - starredEntities: this.starredEntities, - isStarred: e => this.isStarred(e), - }); + subscription.next(this.starredEntities); } } } diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts index 93ff55aa54..4c4c68d66a 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/StarredEntitiesApi.ts @@ -25,24 +25,6 @@ export const starredEntitiesApiRef: ApiRef = createApiRef({ id: 'catalog-react.starred-entities', }); -/** - * @public - */ -export type StarredEntitiesApiObservable = { - /** - * A set of entity references that are starred - */ - starredEntities: Set; - - /** - * A function to check if an entity is starred. - * - * @param entityRef - an entity reference to check - * @returns true, if the entity is starred. - */ - isStarred: (entityRef: string) => boolean; -}; - /** * An API to store and retrieve starred entities * @@ -57,8 +39,7 @@ export interface StarredEntitiesApi { toggleStarred(entityRef: string): Promise; /** - * Observe the state of starred entities and receive a handler - * to check the star state of an entity. + * Observe the set of starred entity references. */ - starredEntitie$(): Observable; + starredEntitie$(): Observable>; } diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts index c71a795b9e..e9f9c8923a 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts @@ -16,7 +16,4 @@ export { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; export { starredEntitiesApiRef } from './StarredEntitiesApi'; -export type { - StarredEntitiesApi, - StarredEntitiesApiObservable, -} from './StarredEntitiesApi'; +export type { StarredEntitiesApi } from './StarredEntitiesApi'; diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.ts b/plugins/catalog-react/src/hooks/useStarredEntities.ts index e79e605877..8f6b09614f 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.ts +++ b/plugins/catalog-react/src/hooks/useStarredEntities.ts @@ -14,29 +14,43 @@ * limitations under the License. */ -import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { + Entity, + EntityName, + stringifyEntityRef, +} from '@backstage/catalog-model'; import { useApi } from '@backstage/core-plugin-api'; import { useCallback } from 'react'; import { useObservable } from 'react-use'; import { starredEntitiesApiRef } from '../apis'; -export const useStarredEntities = () => { +function getEntityRef(entityOrRef: Entity | EntityName | string): string { + return typeof entityOrRef === 'string' + ? entityOrRef + : stringifyEntityRef(entityOrRef); +} + +export function useStarredEntities(): { + starredEntities: Set; + toggleStarredEntity: (entityOrRef: Entity | EntityName | string) => void; + isStarredEntity: (entityOrRef: Entity | EntityName | string) => boolean; +} { const starredEntitiesApi = useApi(starredEntitiesApiRef); - const { starredEntities, isStarred } = useObservable( + const starredEntities = useObservable( starredEntitiesApi.starredEntitie$(), - { starredEntities: new Set(), isStarred: _ => false }, + new Set(), ); const isStarredEntity = useCallback( - (entity: Entity) => isStarred(stringifyEntityRef(entity)), - [isStarred], + (entityOrRef: Entity | EntityName | string) => + starredEntities.has(getEntityRef(entityOrRef)), + [starredEntities], ); const toggleStarredEntity = useCallback( - (entity: Entity) => { - starredEntitiesApi.toggleStarred(stringifyEntityRef(entity)).then(); - }, + (entityOrRef: Entity | EntityName | string) => + starredEntitiesApi.toggleStarred(getEntityRef(entityOrRef)).then(), [starredEntitiesApi], ); @@ -45,4 +59,4 @@ export const useStarredEntities = () => { toggleStarredEntity, isStarredEntity, }; -}; +} From 0366c9b6671430c97efa5927286c5f9ba247bc72 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Wed, 13 Oct 2021 13:25:15 +0200 Subject: [PATCH 52/60] Introduce a `useStarredEntity` hook to check if a single entity is starred Signed-off-by: Dominik Henneke --- .changeset/gorgeous-laws-report.md | 7 ++ plugins/catalog-react/api-report.md | 8 ++ .../FavoriteEntity/FavoriteEntity.tsx | 21 ++-- plugins/catalog-react/src/hooks/index.ts | 1 + .../src/hooks/useStarredEntity.test.tsx | 103 ++++++++++++++++++ .../src/hooks/useStarredEntity.ts | 61 +++++++++++ .../FavouriteTemplate/FavouriteTemplate.tsx | 22 ++-- 7 files changed, 201 insertions(+), 22 deletions(-) create mode 100644 .changeset/gorgeous-laws-report.md create mode 100644 plugins/catalog-react/src/hooks/useStarredEntity.test.tsx create mode 100644 plugins/catalog-react/src/hooks/useStarredEntity.ts diff --git a/.changeset/gorgeous-laws-report.md b/.changeset/gorgeous-laws-report.md new file mode 100644 index 0000000000..47e5aa301f --- /dev/null +++ b/.changeset/gorgeous-laws-report.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-scaffolder': patch +--- + +Introduce a `useStarredEntity` hook to check if a single entity is starred. +It provides a more efficient implementation compared to the `useStarredEntities` hook, because the rendering is only triggered if the selected entity is starred, not if _any_ entity is starred. diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 2486cdd6e8..e28b8b0d32 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -894,6 +894,14 @@ export function useStarredEntities(): { isStarredEntity: (entityOrRef: Entity | EntityName | string) => boolean; }; +// Warning: (ae-missing-release-tag) "useStarredEntity" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export function useStarredEntity(entityOrRef: Entity | EntityName | string): { + toggleStarredEntity: () => void; + isStarredEntity: boolean; +}; + // Warnings were encountered during analysis: // // src/types.d.ts:6:49 - (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag diff --git a/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx b/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx index 95e617ab7f..db60c34b66 100644 --- a/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx +++ b/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx @@ -14,12 +14,12 @@ * limitations under the License. */ -import React, { ComponentProps } from 'react'; -import { useStarredEntities } from '../../hooks/useStarredEntities'; -import { IconButton, Tooltip, withStyles } from '@material-ui/core'; -import StarBorder from '@material-ui/icons/StarBorder'; -import Star from '@material-ui/icons/Star'; import { Entity } from '@backstage/catalog-model'; +import { IconButton, Tooltip, withStyles } from '@material-ui/core'; +import Star from '@material-ui/icons/Star'; +import StarBorder from '@material-ui/icons/StarBorder'; +import React, { ComponentProps } from 'react'; +import { useStarredEntity } from '../../hooks/useStarredEntity'; type Props = ComponentProps & { entity: Entity }; @@ -40,16 +40,17 @@ export const favoriteEntityIcon = (isStarred: boolean) => * @param props MaterialUI IconButton props extended by required `entity` prop */ export const FavoriteEntity = (props: Props) => { - const { toggleStarredEntity, isStarredEntity } = useStarredEntities(); - const isStarred = isStarredEntity(props.entity); + const { toggleStarredEntity, isStarredEntity } = useStarredEntity( + props.entity, + ); return ( toggleStarredEntity(props.entity)} + onClick={() => toggleStarredEntity()} > - - {favoriteEntityIcon(isStarred)} + + {favoriteEntityIcon(isStarredEntity)} ); diff --git a/plugins/catalog-react/src/hooks/index.ts b/plugins/catalog-react/src/hooks/index.ts index 47bca8892a..38fc58222d 100644 --- a/plugins/catalog-react/src/hooks/index.ts +++ b/plugins/catalog-react/src/hooks/index.ts @@ -36,4 +36,5 @@ export { useEntityKinds } from './useEntityKinds'; export { useOwnUser } from './useOwnUser'; export { useRelatedEntities } from './useRelatedEntities'; export { useStarredEntities } from './useStarredEntities'; +export { useStarredEntity } from './useStarredEntity'; export { useEntityOwnership } from './useEntityOwnership'; diff --git a/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx b/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx new file mode 100644 index 0000000000..b8577aec0c --- /dev/null +++ b/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx @@ -0,0 +1,103 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, EntityName } from '@backstage/catalog-model'; +import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; +import { renderHook } from '@testing-library/react-hooks'; +import React, { PropsWithChildren } from 'react'; +import Observable from 'zen-observable'; +import { StarredEntitiesApi, starredEntitiesApiRef } from '../apis'; +import { useStarredEntity } from './useStarredEntity'; + +describe('useStarredEntity', () => { + const mockStarredEntitiesApi: jest.Mocked = { + toggleStarred: jest.fn(), + starredEntitie$: jest.fn(), + }; + let wrapper: React.ComponentType; + + beforeEach(() => { + wrapper = ({ children }: PropsWithChildren<{}>) => ( + + {children} + + ); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe.each` + title | entityOrRef + ${'entity reference'} | ${'component:default/mock'} + ${'entity name'} | ${{ kind: 'component', namespace: 'default', name: 'mock' } as EntityName} + ${'entity'} | ${{ apiVersion: '1', kind: 'Component', metadata: { name: 'mock' } } as Entity} + `('with $title', ({ entityOrRef }) => { + describe('toggleStarredEntity', () => { + it('should toggle starred entity', () => { + mockStarredEntitiesApi.starredEntitie$.mockReturnValue(Observable.of()); + mockStarredEntitiesApi.toggleStarred.mockResolvedValue(); + + const { result } = renderHook(() => useStarredEntity(entityOrRef), { + wrapper, + }); + + result.current.toggleStarredEntity(); + + expect(mockStarredEntitiesApi.toggleStarred).toBeCalledTimes(1); + expect(mockStarredEntitiesApi.toggleStarred).toBeCalledWith( + 'component:default/mock', + ); + }); + }); + + describe('isStarredEntity', () => { + it('should return not starred entity', () => { + mockStarredEntitiesApi.starredEntitie$.mockReturnValue(Observable.of()); + mockStarredEntitiesApi.toggleStarred.mockResolvedValue(); + + const { result } = renderHook(() => useStarredEntity(entityOrRef), { + wrapper, + }); + + expect(result.current.isStarredEntity).toBe(false); + }); + + it('should return starred entity', async () => { + mockStarredEntitiesApi.starredEntitie$.mockReturnValue( + Observable.of(new Set(['component:default/mock'])), + ); + mockStarredEntitiesApi.toggleStarred.mockResolvedValue(); + + const { result, waitForNextUpdate } = renderHook( + () => useStarredEntity(entityOrRef), + { + wrapper, + }, + ); + + // the initial value will always be false because the observable triggers async + expect(result.current.isStarredEntity).toBe(false); + await waitForNextUpdate(); + + expect(result.current.isStarredEntity).toBe(true); + }); + }); + }); +}); diff --git a/plugins/catalog-react/src/hooks/useStarredEntity.ts b/plugins/catalog-react/src/hooks/useStarredEntity.ts new file mode 100644 index 0000000000..0233432196 --- /dev/null +++ b/plugins/catalog-react/src/hooks/useStarredEntity.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + EntityName, + stringifyEntityRef, +} from '@backstage/catalog-model'; +import { useApi } from '@backstage/core-plugin-api'; +import { useCallback, useEffect, useState } from 'react'; +import { starredEntitiesApiRef } from '../apis'; + +function getEntityRef(entityOrRef: Entity | EntityName | string): string { + return typeof entityOrRef === 'string' + ? entityOrRef + : stringifyEntityRef(entityOrRef); +} + +export function useStarredEntity(entityOrRef: Entity | EntityName | string): { + toggleStarredEntity: () => void; + isStarredEntity: boolean; +} { + const starredEntitiesApi = useApi(starredEntitiesApiRef); + + const [isStarredEntity, setIsStarredEntity] = useState(false); + + useEffect(() => { + const subscription = starredEntitiesApi.starredEntitie$().subscribe({ + next(starredEntities: Set) { + setIsStarredEntity(starredEntities.has(getEntityRef(entityOrRef))); + }, + }); + + return () => { + subscription.unsubscribe(); + }; + }, [entityOrRef, starredEntitiesApi]); + + const toggleStarredEntity = useCallback( + () => starredEntitiesApi.toggleStarred(getEntityRef(entityOrRef)).then(), + [entityOrRef, starredEntitiesApi], + ); + + return { + toggleStarredEntity, + isStarredEntity, + }; +} diff --git a/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx b/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx index d158315130..2c55ffdf88 100644 --- a/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx +++ b/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx @@ -14,12 +14,12 @@ * limitations under the License. */ -import React, { ComponentProps, useMemo } from 'react'; -import { useStarredEntities } from '@backstage/plugin-catalog-react'; -import { IconButton, makeStyles, Tooltip, withStyles } from '@material-ui/core'; -import StarBorder from '@material-ui/icons/StarBorder'; -import Star from '@material-ui/icons/Star'; import { Entity } from '@backstage/catalog-model'; +import { useStarredEntity } from '@backstage/plugin-catalog-react'; +import { IconButton, makeStyles, Tooltip, withStyles } from '@material-ui/core'; +import Star from '@material-ui/icons/Star'; +import StarBorder from '@material-ui/icons/StarBorder'; +import React, { ComponentProps } from 'react'; type Props = ComponentProps & { entity: Entity }; @@ -56,20 +56,18 @@ export const favouriteTemplateIcon = (isStarred: boolean) => */ export const FavouriteTemplate = (props: Props) => { const classes = useStyles(); - const { toggleStarredEntity, isStarredEntity } = useStarredEntities(); - const isStarred = useMemo( - () => isStarredEntity(props.entity), - [isStarredEntity, props.entity], + const { toggleStarredEntity, isStarredEntity } = useStarredEntity( + props.entity, ); return ( toggleStarredEntity(props.entity)} + onClick={() => toggleStarredEntity()} > - - {favouriteTemplateIcon(isStarred)} + + {favouriteTemplateIcon(isStarredEntity)} ); From 4ec21812bbd2ed199e23519f63f9a5abf6e73691 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Wed, 13 Oct 2021 16:07:27 +0200 Subject: [PATCH 53/60] Fix import Signed-off-by: Dominik Henneke --- .../src/components/ApiExplorerPage/ApiExplorerPage.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx index 292d03c838..cf40e5160c 100644 --- a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx +++ b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx @@ -32,8 +32,8 @@ import { catalogApiRef, DefaultStarredEntitiesApi, entityRouteRef, + starredEntitiesApiRef, } from '@backstage/plugin-catalog-react'; -import { starredEntitiesApiRef } from '@backstage/plugin-catalog-react/src/apis'; import { MockStorageApi, wrapInTestApp } from '@backstage/test-utils'; import DashboardIcon from '@material-ui/icons/Dashboard'; import { render } from '@testing-library/react'; From 5b57ba36f034765bd9f0fe14172f8b108cadd50d Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Thu, 14 Oct 2021 11:21:45 +0200 Subject: [PATCH 54/60] Provide a migration from the old storage location and format to the new one Signed-off-by: Dominik Henneke --- .changeset/cool-deers-cough.md | 6 +- .../DefaultStarredEntitiesApi.test.ts | 11 ++ .../DefaultStarredEntitiesApi.ts | 4 + .../apis/StarredEntitiesApi/migration.test.ts | 113 ++++++++++++++++++ .../src/apis/StarredEntitiesApi/migration.ts | 62 ++++++++++ 5 files changed, 192 insertions(+), 4 deletions(-) create mode 100644 plugins/catalog-react/src/apis/StarredEntitiesApi/migration.test.ts create mode 100644 plugins/catalog-react/src/apis/StarredEntitiesApi/migration.ts diff --git a/.changeset/cool-deers-cough.md b/.changeset/cool-deers-cough.md index 0c07c91b96..4f4921db8d 100644 --- a/.changeset/cool-deers-cough.md +++ b/.changeset/cool-deers-cough.md @@ -6,7 +6,5 @@ Introduce a new `StarredEntitiesApi` that is used in the `useStarredEntities` hook. The `@backstage/plugin-catalog` installs a default implementation that is backed by the `StorageApi`, but one can also override the `starredEntitiesApiRef`. -**BREAKING** All previously stored entities get lost with this update, because: - -1. The storage format has been migrated from a custom string to an entity reference. -2. The storage key in the local storage has been changed. +This change also updates the storage format from a custom string to an entity reference and moves the location in the local storage. +A migration will convert the previously starred entities to the location on the first load of Backstage. diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts index be67b03dbb..55b992fadf 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts @@ -18,6 +18,9 @@ import { stringifyEntityRef } from '@backstage/catalog-model'; import { StorageApi } from '@backstage/core-plugin-api'; import { MockStorageApi } from '@backstage/test-utils'; import { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; +import { performMigrationToTheNewBucket } from './migration'; + +jest.mock('./migration'); describe('DefaultStarredEntitiesApi', () => { let mockStorage: StorageApi; @@ -32,6 +35,8 @@ describe('DefaultStarredEntitiesApi', () => { }); beforeEach(() => { + (performMigrationToTheNewBucket as jest.Mock).mockResolvedValue(undefined); + mockStorage = MockStorageApi.create(); starredEntitiesApi = new DefaultStarredEntitiesApi({ storageApi: mockStorage, @@ -42,6 +47,12 @@ describe('DefaultStarredEntitiesApi', () => { jest.resetAllMocks(); }); + describe('constructor', () => { + it('should call migration', () => { + expect(performMigrationToTheNewBucket).toBeCalledTimes(1); + }); + }); + describe('toggleStarred', () => { it('should star unstarred entity', async () => { expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(false); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts index 92a0eaebf2..87d99c8904 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -16,6 +16,7 @@ import { Observable, StorageApi } from '@backstage/core-plugin-api'; import ObservableImpl from 'zen-observable'; +import { performMigrationToTheNewBucket } from './migration'; import { StarredEntitiesApi } from './StarredEntitiesApi'; /** @@ -28,6 +29,9 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { private starredEntities: Set; constructor(opts: { storageApi: StorageApi }) { + // no need to await. The updated content will be caught by the observe$ + performMigrationToTheNewBucket(opts).then(); + this.settingsStore = opts.storageApi.forBucket('starredEntities'); this.starredEntities = new Set( diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.test.ts new file mode 100644 index 0000000000..1ff3be811f --- /dev/null +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.test.ts @@ -0,0 +1,113 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { StorageApi } from '@backstage/core-plugin-api'; +import { MockStorageApi } from '@backstage/test-utils'; +import { performMigrationToTheNewBucket } from './migration'; + +describe('performMigrationToTheNewBucket', () => { + let mockStorage: StorageApi; + + beforeEach(() => { + mockStorage = MockStorageApi.create(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should migrate', async () => { + const oldBucket = mockStorage.forBucket('settings'); + const newBucket = mockStorage.forBucket('starredEntities'); + + // fill NEW bucket + await newBucket.set('entityRefs', ['component:default/c']); + + // fill OLD bucket + await oldBucket.set('starredEntities', [ + 'entity:Component:default:a', + 'entity:template:custom:b', + ]); + expect(oldBucket.get('starredEntities')).not.toBeUndefined(); + + await performMigrationToTheNewBucket({ storageApi: mockStorage }); + + // read NEW bucket + expect(await newBucket.get('entityRefs')).toEqual([ + 'component:default/c', + 'component:default/a', + 'template:custom/b', + ]); + + // OLD bucket should be removed + expect(oldBucket.get('starredEntities')).toBeUndefined(); + }); + + it('should ignore invalid entries', async () => { + const oldBucket = mockStorage.forBucket('settings'); + const newBucket = mockStorage.forBucket('starredEntities'); + + // fill OLD bucket + await oldBucket.set('starredEntities', [ + 'entity:Component:default:a', + 1, + 'entity:Component:a', + 'invalid', + ]); + expect(oldBucket.get('starredEntities')).not.toBeUndefined(); + + await performMigrationToTheNewBucket({ storageApi: mockStorage }); + + // read NEW bucket + expect(await newBucket.get('entityRefs')).toEqual(['component:default/a']); + + // OLD bucket should be removed + expect(oldBucket.get('starredEntities')).toBeUndefined(); + }); + + it('should skip migration without old starred entities', async () => { + const newBucket = mockStorage.forBucket('starredEntities'); + + // fill NEW bucket + const expectedEntries = ['component:default/a']; + await newBucket.set('entityRefs', expectedEntries); + + await performMigrationToTheNewBucket({ storageApi: mockStorage }); + + // read NEW bucket + expect(newBucket.get('entityRefs')).toBe(expectedEntries); + }); + + it('should skip migration with non-array old starred entities', async () => { + const oldBucket = mockStorage.forBucket('settings'); + const newBucket = mockStorage.forBucket('starredEntities'); + + // fill OLD bucket with invalid content + await oldBucket.set('starredEntities', 'invalid'); + + // fill NEW bucket + const expectedEntries = ['component:default/a']; + await newBucket.set('entityRefs', expectedEntries); + + await performMigrationToTheNewBucket({ storageApi: mockStorage }); + + // read NEW bucket + expect(newBucket.get('entityRefs')).toBe(expectedEntries); + + // OLD bucket should be unchanged + expect(oldBucket.get('starredEntities')).toBe('invalid'); + }); +}); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.ts new file mode 100644 index 0000000000..7147d449ab --- /dev/null +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { stringifyEntityRef } from '@backstage/catalog-model'; +import { StorageApi } from '@backstage/core-plugin-api'; +import { isArray, isString } from 'lodash'; + +/** + * Migrate the starred entities from the old format (entity:::) from the + * old storage location (/settings/starredEntities) to entity references in the new location + * (/starredEntities/entityRefs). + * + * This will only be executed once since the old location is cleared. + * + * @param storageApi - the StorageApi to migrate + */ +export async function performMigrationToTheNewBucket({ + storageApi, +}: { + storageApi: StorageApi; +}) { + const source = storageApi.forBucket('settings'); + const target = storageApi.forBucket('starredEntities'); + + const oldStarredEntities = source.get('starredEntities'); + + if (!isArray(oldStarredEntities)) { + // nothing to do + return; + } + + const targetEntities = new Set(target.get('entityRefs') ?? []); + + oldStarredEntities + .filter(isString) + // extract the old format 'entity:::' + .map(old => old.split(':')) + // check if the format is valid + .filter(split => split.length === 4 && split[0] === 'entity') + // convert to entity references + .map(([_, kind, namespace, name]) => + stringifyEntityRef({ kind, namespace, name }), + ) + .forEach(e => targetEntities.add(e)); + + await target.set('entityRefs', Array.from(targetEntities)); + + await source.remove('starredEntities'); +} From 34f9036a5ff418ee36efa03cb257d805388bc7a6 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Thu, 14 Oct 2021 12:31:15 +0200 Subject: [PATCH 55/60] chore: sync yarn.lock on master branch Signed-off-by: Himanshu Mishra --- yarn.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn.lock b/yarn.lock index 7ba8bdb9ef..c0329e7a4e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -25817,7 +25817,7 @@ svgo@^1.0.0, svgo@^1.2.2: unquote "~1.1.1" util.promisify "~1.0.0" -swagger-client@^3.16.1: +swagger-client@3.16.1, swagger-client@^3.16.1: version "3.16.1" resolved "https://registry.npmjs.org/swagger-client/-/swagger-client-3.16.1.tgz#df86c9d407ab52c00cb356e714b0ec732bb3ad40" integrity sha512-BcNRQzXHRGuXfhN0f80ptlr+bSaPvXwo8+gWbpmTnbKdAjcWOKAWwUx7rgGHjTKZh0qROr/GX9xOZIY8LrBuTg== From 4c840cebfa625304adfef6de557015dc4d74f05e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 14 Oct 2021 12:45:08 +0000 Subject: [PATCH 56/60] Version Packages --- .changeset/breezy-dolphins-lay.md | 8 -- .changeset/cool-deers-cough.md | 10 --- .changeset/dirty-balloons-develop.md | 5 -- .changeset/flat-camels-scream.md | 5 -- .changeset/funny-dolls-draw.md | 19 ----- .changeset/gorgeous-laws-report.md | 7 -- .changeset/happy-hotels-explain.md | 7 -- .changeset/hip-suns-fix.md | 5 -- .changeset/honest-drinks-eat.md | 5 -- .changeset/hungry-bugs-drop.md | 5 -- .changeset/khaki-planets-prove.md | 5 -- .changeset/large-pots-invent.md | 5 -- .changeset/lemon-actors-wait.md | 15 ---- .changeset/loud-bugs-carry.md | 5 -- .changeset/lucky-countries-smile.md | 5 -- .changeset/mean-eggs-knock.md | 6 -- .changeset/modern-carrots-occur.md | 5 -- .changeset/modern-clouds-guess.md | 9 --- .changeset/moody-shrimps-whisper.md | 5 -- .changeset/neat-cooks-sell.md | 5 -- .changeset/nine-laws-run.md | 11 --- .changeset/ninety-islands-compare.md | 5 -- .changeset/old-fishes-drum.md | 5 -- .changeset/olive-ants-allow.md | 5 -- .changeset/olive-mayflies-scream.md | 5 -- .changeset/plenty-bees-run.md | 5 -- .changeset/polite-timers-watch.md | 5 -- .changeset/popular-students-sniff.md | 5 -- .changeset/search-gentle-chairs-grab.md | 5 -- .changeset/silent-candles-remember.md | 5 -- .changeset/six-pumpkins-lie.md | 5 -- .changeset/slow-frogs-deliver.md | 5 -- .changeset/smart-seahorses-ring.md | 5 -- .changeset/spicy-goats-help.md | 7 -- .changeset/stale-seahorses-sell.md | 5 -- .changeset/stupid-elephants-invent.md | 5 -- .changeset/techdocs-dom-toretto.md | 5 -- .changeset/two-foxes-marry.md | 5 -- .changeset/unlucky-laws-doubt.md | 6 -- .changeset/wicked-clocks-kneel.md | 37 --------- .changeset/wise-clocks-accept.md | 5 -- .changeset/wise-kids-add.md | 6 -- .changeset/witty-dolls-walk.md | 5 -- .changeset/young-peas-unite.md | 25 ------ packages/app/CHANGELOG.md | 43 ++++++++++ packages/app/package.json | 78 +++++++++---------- packages/backend/CHANGELOG.md | 14 ++++ packages/backend/package.json | 20 ++--- packages/cli/CHANGELOG.md | 13 ++++ packages/cli/package.json | 12 +-- packages/codemods/CHANGELOG.md | 8 ++ packages/codemods/package.json | 2 +- packages/core-app-api/CHANGELOG.md | 9 +++ packages/core-app-api/package.json | 10 +-- packages/core-components/CHANGELOG.md | 25 ++++++ packages/core-components/package.json | 10 +-- packages/create-app/CHANGELOG.md | 20 +++++ packages/create-app/package.json | 2 +- packages/dev-utils/CHANGELOG.md | 12 +++ packages/dev-utils/package.json | 16 ++-- packages/integration-react/CHANGELOG.md | 9 +++ packages/integration-react/package.json | 14 ++-- packages/integration/CHANGELOG.md | 6 ++ packages/integration/package.json | 6 +- packages/test-utils/CHANGELOG.md | 9 +++ packages/test-utils/package.json | 8 +- packages/theme/CHANGELOG.md | 6 ++ packages/theme/package.json | 4 +- plugins/allure/CHANGELOG.md | 9 +++ plugins/allure/package.json | 16 ++-- plugins/analytics-module-ga/CHANGELOG.md | 9 +++ plugins/analytics-module-ga/package.json | 14 ++-- plugins/api-docs/CHANGELOG.md | 14 ++++ plugins/api-docs/package.json | 18 ++--- plugins/auth-backend/CHANGELOG.md | 13 ++++ plugins/auth-backend/package.json | 6 +- plugins/azure-devops-backend/CHANGELOG.md | 7 ++ plugins/azure-devops-backend/package.json | 4 +- plugins/azure-devops/CHANGELOG.md | 10 +++ plugins/azure-devops/package.json | 16 ++-- plugins/badges/CHANGELOG.md | 9 +++ plugins/badges/package.json | 16 ++-- plugins/bitrise/CHANGELOG.md | 9 +++ plugins/bitrise/package.json | 16 ++-- .../catalog-backend-module-ldap/CHANGELOG.md | 7 ++ .../catalog-backend-module-ldap/package.json | 6 +- .../CHANGELOG.md | 7 ++ .../package.json | 8 +- plugins/catalog-backend/CHANGELOG.md | 17 ++++ plugins/catalog-backend/package.json | 8 +- plugins/catalog-graph/CHANGELOG.md | 13 ++++ plugins/catalog-graph/package.json | 16 ++-- plugins/catalog-import/CHANGELOG.md | 10 +++ plugins/catalog-import/package.json | 18 ++--- plugins/catalog-react/CHANGELOG.md | 21 +++++ plugins/catalog-react/package.json | 12 +-- plugins/catalog/CHANGELOG.md | 18 +++++ plugins/catalog/package.json | 18 ++--- plugins/circleci/CHANGELOG.md | 9 +++ plugins/circleci/package.json | 16 ++-- plugins/cloudbuild/CHANGELOG.md | 9 +++ plugins/cloudbuild/package.json | 16 ++-- plugins/code-coverage-backend/CHANGELOG.md | 8 ++ plugins/code-coverage-backend/package.json | 6 +- plugins/code-coverage/CHANGELOG.md | 9 +++ plugins/code-coverage/package.json | 16 ++-- plugins/config-schema/CHANGELOG.md | 8 ++ plugins/config-schema/package.json | 14 ++-- plugins/cost-insights/CHANGELOG.md | 8 ++ plugins/cost-insights/package.json | 14 ++-- plugins/explore/CHANGELOG.md | 9 +++ plugins/explore/package.json | 16 ++-- plugins/firehydrant/CHANGELOG.md | 9 +++ plugins/firehydrant/package.json | 16 ++-- plugins/fossa/CHANGELOG.md | 9 +++ plugins/fossa/package.json | 16 ++-- plugins/gcp-projects/CHANGELOG.md | 8 ++ plugins/gcp-projects/package.json | 14 ++-- plugins/git-release-manager/CHANGELOG.md | 9 +++ plugins/git-release-manager/package.json | 16 ++-- plugins/github-actions/CHANGELOG.md | 10 +++ plugins/github-actions/package.json | 18 ++--- plugins/github-deployments/CHANGELOG.md | 11 +++ plugins/github-deployments/package.json | 20 ++--- plugins/gitops-profiles/CHANGELOG.md | 8 ++ plugins/gitops-profiles/package.json | 14 ++-- plugins/graphiql/CHANGELOG.md | 9 +++ plugins/graphiql/package.json | 14 ++-- plugins/home/CHANGELOG.md | 55 +++++++++++++ plugins/home/package.json | 14 ++-- plugins/ilert/CHANGELOG.md | 9 +++ plugins/ilert/package.json | 16 ++-- plugins/jenkins/CHANGELOG.md | 9 +++ plugins/jenkins/package.json | 16 ++-- plugins/kafka/CHANGELOG.md | 9 +++ plugins/kafka/package.json | 16 ++-- plugins/kubernetes-backend/CHANGELOG.md | 27 +++++++ plugins/kubernetes-backend/package.json | 4 +- plugins/kubernetes/CHANGELOG.md | 10 +++ plugins/kubernetes/package.json | 16 ++-- plugins/lighthouse/CHANGELOG.md | 9 +++ plugins/lighthouse/package.json | 16 ++-- plugins/newrelic/CHANGELOG.md | 8 ++ plugins/newrelic/package.json | 14 ++-- plugins/org/CHANGELOG.md | 9 +++ plugins/org/package.json | 16 ++-- plugins/pagerduty/CHANGELOG.md | 9 +++ plugins/pagerduty/package.json | 16 ++-- plugins/rollbar/CHANGELOG.md | 9 +++ plugins/rollbar/package.json | 16 ++-- plugins/scaffolder-backend/CHANGELOG.md | 10 +++ plugins/scaffolder-backend/package.json | 10 +-- plugins/scaffolder/CHANGELOG.md | 13 ++++ plugins/scaffolder/package.json | 20 ++--- plugins/search/CHANGELOG.md | 10 +++ plugins/search/package.json | 16 ++-- plugins/sentry/CHANGELOG.md | 9 +++ plugins/sentry/package.json | 16 ++-- plugins/shortcuts/CHANGELOG.md | 8 ++ plugins/shortcuts/package.json | 14 ++-- plugins/sonarqube/CHANGELOG.md | 9 +++ plugins/sonarqube/package.json | 16 ++-- plugins/splunk-on-call/CHANGELOG.md | 9 +++ plugins/splunk-on-call/package.json | 16 ++-- plugins/tech-radar/CHANGELOG.md | 8 ++ plugins/tech-radar/package.json | 14 ++-- plugins/techdocs/CHANGELOG.md | 14 ++++ plugins/techdocs/package.json | 24 +++--- plugins/todo/CHANGELOG.md | 9 +++ plugins/todo/package.json | 16 ++-- plugins/user-settings/CHANGELOG.md | 8 ++ plugins/user-settings/package.json | 14 ++-- plugins/welcome/CHANGELOG.md | 8 ++ plugins/welcome/package.json | 14 ++-- plugins/xcmetrics/CHANGELOG.md | 8 ++ plugins/xcmetrics/package.json | 14 ++-- yarn.lock | 66 ++++++++++++++++ 177 files changed, 1314 insertions(+), 805 deletions(-) delete mode 100644 .changeset/breezy-dolphins-lay.md delete mode 100644 .changeset/cool-deers-cough.md delete mode 100644 .changeset/dirty-balloons-develop.md delete mode 100644 .changeset/flat-camels-scream.md delete mode 100644 .changeset/funny-dolls-draw.md delete mode 100644 .changeset/gorgeous-laws-report.md delete mode 100644 .changeset/happy-hotels-explain.md delete mode 100644 .changeset/hip-suns-fix.md delete mode 100644 .changeset/honest-drinks-eat.md delete mode 100644 .changeset/hungry-bugs-drop.md delete mode 100644 .changeset/khaki-planets-prove.md delete mode 100644 .changeset/large-pots-invent.md delete mode 100644 .changeset/lemon-actors-wait.md delete mode 100644 .changeset/loud-bugs-carry.md delete mode 100644 .changeset/lucky-countries-smile.md delete mode 100644 .changeset/mean-eggs-knock.md delete mode 100644 .changeset/modern-carrots-occur.md delete mode 100644 .changeset/modern-clouds-guess.md delete mode 100644 .changeset/moody-shrimps-whisper.md delete mode 100644 .changeset/neat-cooks-sell.md delete mode 100644 .changeset/nine-laws-run.md delete mode 100644 .changeset/ninety-islands-compare.md delete mode 100644 .changeset/old-fishes-drum.md delete mode 100644 .changeset/olive-ants-allow.md delete mode 100644 .changeset/olive-mayflies-scream.md delete mode 100644 .changeset/plenty-bees-run.md delete mode 100644 .changeset/polite-timers-watch.md delete mode 100644 .changeset/popular-students-sniff.md delete mode 100644 .changeset/search-gentle-chairs-grab.md delete mode 100644 .changeset/silent-candles-remember.md delete mode 100644 .changeset/six-pumpkins-lie.md delete mode 100644 .changeset/slow-frogs-deliver.md delete mode 100644 .changeset/smart-seahorses-ring.md delete mode 100644 .changeset/spicy-goats-help.md delete mode 100644 .changeset/stale-seahorses-sell.md delete mode 100644 .changeset/stupid-elephants-invent.md delete mode 100644 .changeset/techdocs-dom-toretto.md delete mode 100644 .changeset/two-foxes-marry.md delete mode 100644 .changeset/unlucky-laws-doubt.md delete mode 100644 .changeset/wicked-clocks-kneel.md delete mode 100644 .changeset/wise-clocks-accept.md delete mode 100644 .changeset/wise-kids-add.md delete mode 100644 .changeset/witty-dolls-walk.md delete mode 100644 .changeset/young-peas-unite.md create mode 100644 plugins/analytics-module-ga/CHANGELOG.md create mode 100644 plugins/azure-devops/CHANGELOG.md diff --git a/.changeset/breezy-dolphins-lay.md b/.changeset/breezy-dolphins-lay.md deleted file mode 100644 index 0197979531..0000000000 --- a/.changeset/breezy-dolphins-lay.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -'@backstage/plugin-auth-backend': patch ---- - -AWS-ALB: update provider to the latest changes described [here](https://backstage.io/docs/auth/identity-resolver). - -This removes the `ExperimentalIdentityResolver` type in favor of `SignInResolver` and `AuthHandler`. -The AWS ALB provider can now be configured in the same way as the Google provider in the example. diff --git a/.changeset/cool-deers-cough.md b/.changeset/cool-deers-cough.md deleted file mode 100644 index 4f4921db8d..0000000000 --- a/.changeset/cool-deers-cough.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -'@backstage/plugin-catalog-react': minor -'@backstage/plugin-catalog': patch ---- - -Introduce a new `StarredEntitiesApi` that is used in the `useStarredEntities` hook. -The `@backstage/plugin-catalog` installs a default implementation that is backed by the `StorageApi`, but one can also override the `starredEntitiesApiRef`. - -This change also updates the storage format from a custom string to an entity reference and moves the location in the local storage. -A migration will convert the previously starred entities to the location on the first load of Backstage. diff --git a/.changeset/dirty-balloons-develop.md b/.changeset/dirty-balloons-develop.md deleted file mode 100644 index 6dd9005cca..0000000000 --- a/.changeset/dirty-balloons-develop.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-api-docs': patch ---- - -Bump `swagger-ui-react` to `^4.0.0-rc.3`. diff --git a/.changeset/flat-camels-scream.md b/.changeset/flat-camels-scream.md deleted file mode 100644 index 2c49af5010..0000000000 --- a/.changeset/flat-camels-scream.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/integration': patch ---- - -By replacing `\n` with a newline for GitHub Apps private keys, this allows users to store the private key as an environment variable and reference it in the YAML. diff --git a/.changeset/funny-dolls-draw.md b/.changeset/funny-dolls-draw.md deleted file mode 100644 index f194b30750..0000000000 --- a/.changeset/funny-dolls-draw.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -'@backstage/create-app': patch ---- - -The scaffolder plugin has just released the beta 3 version of software templates, which replaces the handlebars templating syntax. As part of this change, the template entity schema is no longer included in the core catalog-model as with previous versions. The decoupling of the template entities version will allow us to more easily make updates in the future. - -In order to use the new beta 3 templates, the following changes are **required** for any existing installation, inside `packages/backend/src/plugins/catalog.ts`: - -```diff -+import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; - -... - - const builder = await CatalogBuilder.create(env); -+ builder.addProcessor(new ScaffolderEntitiesProcessor()); - const { processingEngine, router } = await builder.build(); -``` - -If you're interested in learning more about creating custom kinds, please check out the [extending the model](https://backstage.io/docs/features/software-catalog/extending-the-model) documentation. diff --git a/.changeset/gorgeous-laws-report.md b/.changeset/gorgeous-laws-report.md deleted file mode 100644 index 47e5aa301f..0000000000 --- a/.changeset/gorgeous-laws-report.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@backstage/plugin-catalog-react': patch -'@backstage/plugin-scaffolder': patch ---- - -Introduce a `useStarredEntity` hook to check if a single entity is starred. -It provides a more efficient implementation compared to the `useStarredEntities` hook, because the rendering is only triggered if the selected entity is starred, not if _any_ entity is starred. diff --git a/.changeset/happy-hotels-explain.md b/.changeset/happy-hotels-explain.md deleted file mode 100644 index ede18c3187..0000000000 --- a/.changeset/happy-hotels-explain.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@backstage/core-app-api': patch -'@backstage/core-components': patch -'@backstage/theme': patch ---- - -Internal refactor to avoid importing all of `@material-ui/core`. diff --git a/.changeset/hip-suns-fix.md b/.changeset/hip-suns-fix.md deleted file mode 100644 index 4555c7b179..0000000000 --- a/.changeset/hip-suns-fix.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/core-components': patch ---- - -update the null check to use the optional chaining operator in case of non-null assertion operator is not working in function extractInitials(values: string) diff --git a/.changeset/honest-drinks-eat.md b/.changeset/honest-drinks-eat.md deleted file mode 100644 index b6cb168e00..0000000000 --- a/.changeset/honest-drinks-eat.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-graphiql': patch ---- - -Add experimental `experimentalInstallationRecipe` to `package.json`. diff --git a/.changeset/hungry-bugs-drop.md b/.changeset/hungry-bugs-drop.md deleted file mode 100644 index d13e21d446..0000000000 --- a/.changeset/hungry-bugs-drop.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/core-components': patch ---- - -Resolve a warning in `