From b6a2c1f255858848afcce01001b5c48067148e47 Mon Sep 17 00:00:00 2001 From: Kai Szybiak Date: Thu, 18 Nov 2021 15:43:37 +0100 Subject: [PATCH 01/50] Update AsyncAPI component to 1.0.0-x releases Signed-off-by: Kai Szybiak --- plugins/api-docs/package.json | 2 +- .../AsyncApiDefinition.test.tsx | 21 +- .../AsyncApiDefinition.tsx | 160 ++++++----- yarn.lock | 263 +++++++++++------- 4 files changed, 256 insertions(+), 190 deletions(-) diff --git a/plugins/api-docs/package.json b/plugins/api-docs/package.json index 16f2d0d188..f2e7119237 100644 --- a/plugins/api-docs/package.json +++ b/plugins/api-docs/package.json @@ -30,7 +30,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@asyncapi/react-component": "^0.23.0", + "@asyncapi/react-component": "^1.0.0-next.21", "@backstage/catalog-model": "^0.9.7", "@backstage/core-components": "^0.7.4", "@backstage/core-plugin-api": "^0.2.0", diff --git a/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinition.test.tsx b/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinition.test.tsx index cb89476c03..c9f979edad 100644 --- a/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinition.test.tsx +++ b/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinition.test.tsx @@ -18,6 +18,15 @@ import { renderInTestApp } from '@backstage/test-utils'; import React from 'react'; import { AsyncApiDefinition } from './AsyncApiDefinition'; +jest.mock('use-resize-observer', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(() => ({ + observe: jest.fn(), + unobserve: jest.fn(), + disconnect: jest.fn(), + })), +})); + describe('', () => { it('renders asyncapi spec', async () => { const definition = ` @@ -45,15 +54,7 @@ components: expect(getByText(/Account Service/i)).toBeInTheDocument(); expect(getByText(/user\/signedup/i)).toBeInTheDocument(); - expect(getByText(/UserSignedUp/i)).toBeInTheDocument(); - expect(getAllByText(/displayName/i)).toHaveLength(4); - }); - - it('renders error if definition is missing', async () => { - const { getByText } = await renderInTestApp( - , - ); - expect(getByText(/Error/i)).toBeInTheDocument(); - expect(getByText(/Document can't be null or falsey/i)).toBeInTheDocument(); + expect(getAllByText(/UserSignedUp/i)).toHaveLength(2); + expect(getAllByText(/displayName/i)).toHaveLength(3); }); }); diff --git a/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinition.tsx b/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinition.tsx index 1c993f67d7..3f9375921d 100644 --- a/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinition.tsx +++ b/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinition.tsx @@ -15,23 +15,72 @@ */ import AsyncApi from '@asyncapi/react-component'; -import '@asyncapi/react-component/lib/styles/fiori.css'; -import { alpha, makeStyles } from '@material-ui/core/styles'; +import '@asyncapi/react-component/styles/default.css'; +import { makeStyles, alpha, darken } from '@material-ui/core/styles'; +import { BackstageTheme } from '@backstage/theme'; import React from 'react'; +import { useTheme } from '@material-ui/core'; -const useStyles = makeStyles(theme => ({ +const useStyles = makeStyles((theme: BackstageTheme) => ({ root: { - '& .asyncapi': { - 'font-family': 'inherit', + fontFamily: 'inherit', + '& .bg-white': { background: 'none', }, - '& h2': { - ...theme.typography.h6, + '& .text-4xl': { + ...theme.typography.h3, }, - '& .text-teal': { - color: theme.palette.primary.main, + ' & h2': { + ...theme.typography.h4, + }, + '& .border': { + borderColor: alpha(theme.palette.border, 0.1), + }, + '& .min-w-min': { + minWidth: 'fit-content', + }, + '& .examples': { + padding: '1rem', + }, + '& .bg-teal-500': { + backgroundColor: theme.palette.status.ok, + }, + '& .bg-blue-500': { + backgroundColor: theme.palette.info.main, + }, + '& .bg-blue-400': { + backgroundColor: theme.palette.info.light, + }, + '& .bg-indigo-400': { + backgroundColor: theme.palette.warning.main, + }, + '& .text-teal-50': { + color: theme.palette.status.ok, + }, + '& .text-red-600': { + color: theme.palette.error.main, + }, + '& .text-orange-600': { + color: theme.palette.warning.main, + }, + '& .text-teal-500': { + color: theme.palette.status.ok, + }, + '& .text-blue-500': { + color: theme.palette.info.main, + }, + '& .-rotate-90': { + '--tw-rotate': '0deg', }, '& button': { + ...theme.typography.button, + borderRadius: theme.shape.borderRadius, + color: theme.palette.primary.main, + }, + '& a': { + color: theme.palette.link, + }, + '& a.no-underline': { ...theme.typography.button, background: 'none', boxSizing: 'border-box', @@ -48,84 +97,47 @@ const useStyles = makeStyles(theme => ({ border: `1px solid ${alpha(theme.palette.primary.main, 0.5)}`, '&:hover': { textDecoration: 'none', - '&.Mui-disabled': { - backgroundColor: 'transparent', - }, border: `1px solid ${theme.palette.primary.main}`, backgroundColor: alpha( theme.palette.primary.main, theme.palette.action.hoverOpacity, ), - // Reset on touch devices, it doesn't add specificity - '@media (hover: none)': { - backgroundColor: 'transparent', - }, - }, - '&.Mui-disabled': { - color: theme.palette.action.disabled, }, }, - '& .asyncapi__collapse-button:hover': { - color: theme.palette.primary.main, - }, - '& button.asyncapi__toggle-button': { - 'min-width': 'inherit', - }, - '& .asyncapi__info-list li': { - 'border-color': theme.palette.primary.main, - '&:hover': { - color: theme.palette.text.primary, - 'border-color': theme.palette.primary.main, - 'background-color': theme.palette.primary.main, - }, - }, - '& .asyncapi__info-list li a': { - color: theme.palette.primary.main, - '&:hover': { + '& li.no-underline': { + '& a': { + textDecoration: 'none', color: theme.palette.getContrastText(theme.palette.primary.main), }, }, - '& .asyncapi__enum': { - color: theme.palette.secondary.main, + }, + dark: { + '& svg': { + fill: theme.palette.text.primary, }, - '& .asyncapi__info, .asyncapi__channel, .asyncapi__channels > div, .asyncapi__schema, .asyncapi__channel-operations-list .asyncapi__messages-list-item .asyncapi__message, .asyncapi__message, .asyncapi__server, .asyncapi__servers > div, .asyncapi__messages > div, .asyncapi__schemas > div': - { - 'background-color': 'inherit', - }, - '& .asyncapi__channel-parameters-header, .asyncapi__channel-operations-header, .asyncapi__channel-operation-oneOf-subscribe-header, .asyncapi__channel-operation-oneOf-publish-header, .asyncapi__channel-operation-message-header, .asyncapi__message-header, .asyncapi__message-header-title, .asyncapi__message-header-title > h3, .asyncapi__bindings, .asyncapi__bindings-header, .asyncapi__bindings-header > h4': - { - 'background-color': 'inherit', + '& .prose': { + color: theme.palette.text.secondary, + '& h3': { color: theme.palette.text.primary, }, - '& .asyncapi__additional-properties-notice': { - color: theme.palette.text.hint, }, - '& .asyncapi__code, .asyncapi__code-pre': { - background: theme.palette.background.default, + '& .bg-gray-100, .bg-gray-200': { + backgroundColor: theme.palette.background.default, }, - '& .asyncapi__schema-example-header-title': { - color: theme.palette.text.secondary, + '& .text-gray-600': { + color: theme.palette.grey['50'], }, - '& .asyncapi__message-headers-header, .asyncapi__message-payload-header, .asyncapi__server-variables-header, .asyncapi__server-security-header': - { - 'background-color': 'inherit', - color: theme.palette.text.secondary, + '& .text-gray-700': { + color: theme.palette.grey['100'], + }, + '& .panel--right': { + background: darken(theme.palette.navigation.background, 0.1), + }, + '& .examples': { + backgroundColor: darken(theme.palette.navigation.background, 0.1), + '& pre': { + backgroundColor: darken(theme.palette.background.default, 0.2), }, - '& .asyncapi__table-header': { - background: theme.palette.background.default, - }, - '& .asyncapi__table-body': { - color: theme.palette.text.primary, - }, - '& .asyncapi__server-security-flow': { - background: theme.palette.background.default, - border: 'none', - }, - '& .asyncapi__server-security-flows-list a': { - color: theme.palette.primary.main, - }, - '& .asyncapi__table-row--nested': { - color: theme.palette.text.secondary, }, }, })); @@ -134,11 +146,15 @@ type Props = { definition: string; }; -export const AsyncApiDefinition = ({ definition }: Props) => { +export const AsyncApiDefinition = ({ definition }: Props): JSX.Element => { const classes = useStyles(); + const theme = useTheme(); + const classNames = `${classes.root} ${ + theme.palette.type === 'dark' ? classes.dark : '' + }`; return ( -
+
); diff --git a/yarn.lock b/yarn.lock index 9b934191d8..14907ef7eb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -69,25 +69,25 @@ dependencies: tslib "~2.0.1" -"@asyncapi/avro-schema-parser@^0.2.1": - version "0.2.1" - resolved "https://registry.npmjs.org/@asyncapi/avro-schema-parser/-/avro-schema-parser-0.2.1.tgz#0b7d9953e12084e6f556db261ae08bd8f6690faa" - integrity sha512-RZJaHsdYM4dChYSrb/TWrVCn/r2qcus+9/8iLL8+SMINHb0ECgH8tFZFJpr3Tq+LV2SBFaRQ+9kuecjQ8BNDSA== +"@asyncapi/avro-schema-parser@^0.3.0": + version "0.3.0" + resolved "https://registry.npmjs.org/@asyncapi/avro-schema-parser/-/avro-schema-parser-0.3.0.tgz#6922acc559ef999c57e81297d78ffe680fc92b3c" + integrity sha512-gWAqS2CKxbChdX8hZY+5EYQl6atP8FTSBvoG5mGGQ89XUoNdlLX14lsvbgvBnDj5sSwqfs+b5Mh5PUZMR/8maA== -"@asyncapi/openapi-schema-parser@^2.0.1": +"@asyncapi/openapi-schema-parser@^2.0.0": version "2.0.1" resolved "https://registry.npmjs.org/@asyncapi/openapi-schema-parser/-/openapi-schema-parser-2.0.1.tgz#4d6e82cced907b14e0ad6f98261ff2562d968d96" integrity sha512-algbtdM1gcAOa8+V8kp7WeBhdaNac82jmZUXx8YjyNfRVo02N2juDrjeBAGJd+FNva9Mb4MM7qfkJoAFpTL5VQ== dependencies: "@openapi-contrib/openapi-schema-to-json-schema" "^3.0.0" -"@asyncapi/parser@^1.5.0": - version "1.5.0" - resolved "https://registry.npmjs.org/@asyncapi/parser/-/parser-1.5.0.tgz#d70616a1e6081c7dd86957befd5dccc46b9a77df" - integrity sha512-HzrehCcT6R+iqtktNmrUM9wRUEMBqnCeXIrGJT0txBMS1QduNRmXGjvuDpxqwyaATPH/xu1gSp2l6pFP/hyVbA== +"@asyncapi/parser@1.10.0": + version "1.10.0" + resolved "https://registry.npmjs.org/@asyncapi/parser/-/parser-1.10.0.tgz#74c040328ce72af9adebb914382a84048412bfe2" + integrity sha512-rMUV6tOBqf/lO2JgIuqFb8y+qfkDHlTzFcmOZIMSJ7EbUHpfqdULqlkVva1Y0K7nzCpwlokYGyvbYe+ELlsueQ== dependencies: "@apidevtools/json-schema-ref-parser" "^9.0.6" - "@asyncapi/specs" "^2.7.7" + "@asyncapi/specs" "2.9.0" "@fmvilas/pseudo-yaml-ast" "^0.3.1" ajv "^6.10.1" js-yaml "^3.13.1" @@ -96,25 +96,24 @@ node-fetch "^2.6.0" tiny-merge-patch "^0.1.2" -"@asyncapi/react-component@^0.23.0": - version "0.23.1" - resolved "https://registry.npmjs.org/@asyncapi/react-component/-/react-component-0.23.1.tgz#6ae574aac86d835e6845268d49803c4808f37b97" - integrity sha512-xBH8F//sF4Sb7M95eVDnQDG+czklQieFKHkCr8SJmmb2j4NylAOTJBoYSZJ3QTC9r0O+SxdQglwh+u1mhMAU+A== +"@asyncapi/react-component@^1.0.0-next.21": + version "1.0.0-next.21" + resolved "https://registry.npmjs.org/@asyncapi/react-component/-/react-component-1.0.0-next.21.tgz#70e4f0d67e9830ece7481c7bba6383573bf6162d" + integrity sha512-nm6k1jW+pGJA6m14pYsad6kOemY0cRpUOrN1QoW6ovS3LmFfnzz+IcAmeX5J1cUHCwhK/H8lI/EUiDvr0XGrdQ== dependencies: - "@asyncapi/avro-schema-parser" "^0.2.1" - "@asyncapi/openapi-schema-parser" "^2.0.1" - "@asyncapi/parser" "^1.5.0" - constate "^1.2.0" - dompurify "^2.1.1" - markdown-it "^11.0.1" - merge "^2.1.0" - openapi-sampler "^1.0.0-beta.15" - react-use "^12.2.0" + "@asyncapi/avro-schema-parser" "^0.3.0" + "@asyncapi/openapi-schema-parser" "^2.0.0" + "@asyncapi/parser" "1.10.0" + highlight.js "^10.7.2" + isomorphic-dompurify "^0.13.0" + marked "^2.1.1" + openapi-sampler "^1.1.0" + use-resize-observer "^7.0.0" -"@asyncapi/specs@^2.7.7": - version "2.7.7" - resolved "https://registry.npmjs.org/@asyncapi/specs/-/specs-2.7.7.tgz#10f72c95153a3cc10039f6ba9c3a6f7c3b7fecfc" - integrity sha512-z8kj4GDJ640DU4msRsWprvmuC9n7vIeJW+D7Tp1xdefoLX5ZJrK7+4Xruna513wV0fSLpFzCmGz7McEP6CtKDg== +"@asyncapi/specs@2.9.0": + version "2.9.0" + resolved "https://registry.npmjs.org/@asyncapi/specs/-/specs-2.9.0.tgz#c5289b1e5853ecf9c0c04e53085d49aba09ca39e" + integrity sha512-23/mlTzC1O4yF9RyA8QAqUlZBcsd6Fc+kktWURNgOgEam/2cbYKGrib7d7WmbfAi1NIJk9P8DqX2s7PHJ/NZsw== "@azure/abort-controller@^1.0.0": version "1.0.2" @@ -3664,6 +3663,11 @@ resolved "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz#9df03bbd7c696a5c58885c34aa06da41c8543796" integrity sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg== +"@juggle/resize-observer@^3.3.1": + version "3.3.1" + resolved "https://registry.npmjs.org/@juggle/resize-observer/-/resize-observer-3.3.1.tgz#b50a781709c81e10701004214340f25475a171a0" + integrity sha512-zMM9Ds+SawiUkakS7y94Ymqx+S0ORzpG3frZirN3l+UlXUmSUR7hF4wxCVqW+ei94JzV5kt0uXBcoOEAuiydrw== + "@kubernetes/client-node@^0.15.0": version "0.15.0" resolved "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.15.0.tgz#aa5cfcfa9ba3055fe0b510c430d19bbda715d8e7" @@ -7100,6 +7104,13 @@ "@types/docker-modem" "*" "@types/node" "*" +"@types/dompurify@^2.1.0": + version "2.3.1" + resolved "https://registry.npmjs.org/@types/dompurify/-/dompurify-2.3.1.tgz#2934adcd31c4e6b02676f9c22f9756e5091c04dd" + integrity sha512-YJth9qa0V/E6/XPH1Jq4BC8uCMmO8V1fKWn8PCvuZcAhMn7q0ez9LW6naQT04UZzjFfAPhyRMZmI2a2rbMlEFA== + dependencies: + "@types/trusted-types" "*" + "@types/dompurify@^2.2.2": version "2.2.3" resolved "https://registry.npmjs.org/@types/dompurify/-/dompurify-2.2.3.tgz#6e89677a07902ac1b6821c345f34bd85da239b08" @@ -7881,13 +7892,6 @@ dependencies: "@types/react" "*" -"@types/react-wait@^0.3.0": - version "0.3.1" - resolved "https://registry.npmjs.org/@types/react-wait/-/react-wait-0.3.1.tgz#193cbd8fe86baa53b6f65dfa73f03d562f462a27" - integrity sha512-BS9AEjWZItDgpx6LlICcuf53M27zBFCsHx/llCbrmrt/WI7ecG2LGquCss3n8O8bwEDiTX4yYLjy8yLeIfgYTg== - dependencies: - "@types/react" "*" - "@types/react@*", "@types/react@>=16.9.0": version "16.14.18" resolved "https://registry.npmjs.org/@types/react/-/react-16.14.18.tgz#b2bcea05ee244fde92d409f91bd888ca8e54b20f" @@ -8758,6 +8762,11 @@ abab@^2.0.3: resolved "https://registry.npmjs.org/abab/-/abab-2.0.3.tgz#623e2075e02eb2d3f2475e49f99c91846467907a" integrity sha512-tsFzPpcttalNjFBCFMqsKYQcWxxen1pgJR56by//QwvJc4/OUS3kPOOttx2tSIfjsylB0pYu7f5D3K1RCxUnUg== +abab@^2.0.5: + version "2.0.5" + resolved "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a" + integrity sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q== + abbrev@1: version "1.1.1" resolved "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" @@ -8821,6 +8830,11 @@ acorn@^7.1.1, acorn@^7.4.0: resolved "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== +acorn@^8.2.4: + version "8.6.0" + resolved "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz#e3692ba0eb1a0c83eaa4f37f5fa7368dd7142895" + integrity sha512-U1riIR+lBSNi3IbxtaHOIKdH8sLFv3NYfNv8sg7ZsNhcfl4HF2++BfqqrNAxoCLQW1iiylOj76ecnaUxz+z9yw== + acorn@^8.4.1: version "8.4.1" resolved "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz#56c36251fc7cabc7096adc18f05afe814321a28c" @@ -11645,11 +11659,6 @@ constants-browserify@^1.0.0: resolved "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz#c20b96d8c617748aaf1c16021760cd27fcb8cb75" integrity sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U= -constate@^1.2.0: - version "1.3.2" - resolved "https://registry.npmjs.org/constate/-/constate-1.3.2.tgz#fa5f0fc292207f1ec21b46a5eb81f59c8b0a8b84" - integrity sha512-aaILV4vXwGTUZaQZHS5F1xBV8wRCR0Ow1505fdkS5/BPg6hbQrhNqdHL4wgxWgaDeEj43mu/Fb+LhqOKTMcrgQ== - contains-path@^0.1.0: version "0.1.0" resolved "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz#fe8cf184ff6670b6baef01a9d4861a5cbec4120a" @@ -11809,7 +11818,7 @@ copy-descriptor@^0.1.0: resolved "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d" integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40= -copy-to-clipboard@^3, copy-to-clipboard@^3.1.0, copy-to-clipboard@^3.2.0, copy-to-clipboard@^3.3.1: +copy-to-clipboard@^3, copy-to-clipboard@^3.2.0, copy-to-clipboard@^3.3.1: version "3.3.1" resolved "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.1.tgz#115aa1a9998ffab6196f93076ad6da3b913662ae" integrity sha512-i13qo6kIHTTpCm8/Wup+0b1mVWETvu2kIMzKoK8FpkLkFxlt0znUAHcMzox+T8sPlqtZXq3CulEjQHsYiGFJUw== @@ -12314,7 +12323,7 @@ cssom@~0.3.6: resolved "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz#9f1276f5b2b463f2114d3f2c75250af8c1a36f4a" integrity sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg== -cssstyle@^2.2.0: +cssstyle@^2.2.0, cssstyle@^2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz#ff665a0ddbdc31864b09647f34163443d90b0852" integrity sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A== @@ -12781,6 +12790,11 @@ decimal.js@^10.2.0: resolved "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.0.tgz#39466113a9e036111d02f82489b5fd6b0b5ed231" integrity sha512-vDPw+rDgn3bZe1+F/pyEwb1oMG2XTlRVgAa6B4KccTEpYgF8w6eQllVbQcfIJnZyvzFtFpxnpGtx8dd7DJp/Rw== +decimal.js@^10.2.1: + version "10.3.1" + resolved "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz#d8c3a444a9c6774ba60ca6ad7261c3a94fd5e783" + integrity sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ== + decode-uri-component@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" @@ -13254,7 +13268,7 @@ domhandler@^4.2.0: dependencies: domelementtype "^2.2.0" -dompurify@^2.1.1, dompurify@^2.2.9: +dompurify@^2.2.7, dompurify@^2.2.9: version "2.3.3" resolved "https://registry.npmjs.org/dompurify/-/dompurify-2.3.3.tgz#c1af3eb88be47324432964d8abc75cf4b98d634c" integrity sha512-dqnqRkPMAjOZE0FogZ+ceJNM2dZ3V/yNOuFB7+39qpO93hHhfRpHw3heYQC7DPK9FqbQTfBKUJhiSfz4MvXYwg== @@ -13573,11 +13587,6 @@ entities@^2.0.0, entities@~2.1.0: resolved "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5" integrity sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w== -entities@~2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/entities/-/entities-2.0.0.tgz#68d6084cab1b079767540d80e56a39b423e4abf4" - integrity sha512-D9f7V0JSRwIxlRI2mjMqufDrRDnx8p+eEOz7aUM9SuvF8gsBzra0/6tbjl1m8eQHrZlYj6PxqE00hZ1SAIKPLw== - env-paths@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/env-paths/-/env-paths-2.2.0.tgz#cdca557dc009152917d6166e2febe1f039685e43" @@ -13776,6 +13785,18 @@ escodegen@^1.14.1: optionalDependencies: source-map "~0.6.1" +escodegen@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz#5e32b12833e8aa8fa35e1bf0befa89380484c7dd" + integrity sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw== + dependencies: + esprima "^4.0.1" + estraverse "^5.2.0" + esutils "^2.0.2" + optionator "^0.8.1" + optionalDependencies: + source-map "~0.6.1" + eslint-config-prettier@^8.3.0: version "8.3.0" resolved "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.3.0.tgz#f7471b20b6fe8a9a9254cc684454202886a2dd7a" @@ -16189,7 +16210,7 @@ highlight.js@^10.1.0, highlight.js@^10.1.1, highlight.js@^10.4.1, highlight.js@^ resolved "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.2.tgz#89319b861edc66c48854ed1e6da21ea89f847360" integrity sha512-oFLl873u4usRM9K63j4ME9u3etNF0PLiJhSQ8rdfuL51Wn3zkD6drf9ZW0dOzjnZI22YYG24z30JcmfCZjMgYg== -highlight.js@~10.7.0: +highlight.js@^10.7.2, highlight.js@~10.7.0: version "10.7.3" resolved "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz#697272e3991356e40c3cac566a74eef681756531" integrity sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A== @@ -17386,6 +17407,11 @@ is-potential-custom-element-name@^1.0.0: resolved "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.0.tgz#0c52e54bcca391bb2c494b21e8626d7336c6e397" integrity sha1-DFLlS8yjkbssSUsh6GJtczbG45c= +is-potential-custom-element-name@^1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz#171ed6f19e3ac554394edf78caa05784a45bebb5" + integrity sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ== + is-promise@4.0.0, is-promise@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz#42ff9f84206c1991d26debf520dd5c01042dd2f3" @@ -17624,6 +17650,15 @@ isobject@^4.0.0: resolved "https://registry.npmjs.org/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== +isomorphic-dompurify@^0.13.0: + version "0.13.0" + resolved "https://registry.npmjs.org/isomorphic-dompurify/-/isomorphic-dompurify-0.13.0.tgz#a4dde357e8531018a85ebb2dd56c4794b6739ba3" + integrity sha512-j2/kt/PGbxvfeEm1uiRLlttZkQdn3hFe1rMr/wm3qFnMXSIw0Nmqu79k+TIoSj+KOwO98Sz9TbuNHU7ejv7IZA== + dependencies: + "@types/dompurify" "^2.1.0" + dompurify "^2.2.7" + jsdom "^16.5.2" + isomorphic-fetch@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz#0267b005049046d2421207215d45d6a262b8b8b4" @@ -18359,6 +18394,39 @@ jsdom@^16.4.0: ws "^7.2.3" xml-name-validator "^3.0.0" +jsdom@^16.5.2: + version "16.7.0" + resolved "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz#918ae71965424b197c819f8183a754e18977b710" + integrity sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw== + dependencies: + abab "^2.0.5" + acorn "^8.2.4" + acorn-globals "^6.0.0" + cssom "^0.4.4" + cssstyle "^2.3.0" + data-urls "^2.0.0" + decimal.js "^10.2.1" + domexception "^2.0.1" + escodegen "^2.0.0" + form-data "^3.0.0" + html-encoding-sniffer "^2.0.1" + http-proxy-agent "^4.0.1" + https-proxy-agent "^5.0.0" + is-potential-custom-element-name "^1.0.1" + nwsapi "^2.2.0" + parse5 "6.0.1" + saxes "^5.0.1" + symbol-tree "^3.2.4" + tough-cookie "^4.0.0" + w3c-hr-time "^1.0.2" + w3c-xmlserializer "^2.0.0" + webidl-conversions "^6.1.0" + whatwg-encoding "^1.0.5" + whatwg-mimetype "^2.3.0" + whatwg-url "^8.5.0" + ws "^7.4.6" + xml-name-validator "^3.0.0" + jsesc@^2.5.1: version "2.5.2" resolved "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" @@ -18396,7 +18464,7 @@ json-parse-even-better-errors@^2.3.0, json-parse-even-better-errors@^2.3.1: resolved "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== -json-pointer@^0.6.0: +json-pointer@^0.6.1: version "0.6.1" resolved "https://registry.npmjs.org/json-pointer/-/json-pointer-0.6.1.tgz#3c6caa6ac139e2599f5a1659d39852154015054d" integrity sha512-3OvjqKdCBvH41DLpV4iSt6v2XhZXV1bPB4OROuknvUXI7ZQNofieCPkmE26stEJ9zdQuvIxDHCuYhfgxFAAs+Q== @@ -19421,7 +19489,7 @@ lodash@4.17.15: resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== -lodash@4.17.21, lodash@^4.17.10, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4, lodash@^4.17.5, lodash@~4.17.0, lodash@~4.17.15, lodash@~4.17.4: +lodash@4.17.21, lodash@^4.17.10, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.7.0, lodash@~4.17.0, lodash@~4.17.15, lodash@~4.17.4: version "4.17.21" resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -19713,17 +19781,6 @@ markdown-escapes@^1.0.0: resolved "https://registry.npmjs.org/markdown-escapes/-/markdown-escapes-1.0.4.tgz#c95415ef451499d7602b91095f3c8e8975f78535" integrity sha512-8z4efJYk43E0upd0NbVXwgSTQs6cT3T06etieCMEg7dRbzCbxUCK/GHlX8mhHRDcp+OLlHkPKsvqQTCvsRl2cg== -markdown-it@^11.0.1: - version "11.0.1" - resolved "https://registry.npmjs.org/markdown-it/-/markdown-it-11.0.1.tgz#b54f15ec2a2193efa66dda1eb4173baea08993d6" - integrity sha512-aU1TzmBKcWNNYvH9pjq6u92BML+Hz3h5S/QpfTFwiQF852pLT+9qHsrhM9JYipkOXZxGn+sGH8oyJE9FD9WezQ== - dependencies: - argparse "^1.0.7" - entities "~2.0.0" - linkify-it "^3.0.1" - mdurl "^1.0.1" - uc.micro "^1.0.5" - markdown-it@^12.2.0: version "12.2.0" resolved "https://registry.npmjs.org/markdown-it/-/markdown-it-12.2.0.tgz#091f720fd5db206f80de7a8d1f1a7035fd0d38db" @@ -19760,6 +19817,11 @@ markdown-to-jsx@^7.1.3: resolved "https://registry.npmjs.org/markdown-to-jsx/-/markdown-to-jsx-7.1.3.tgz#f00bae66c0abe7dd2d274123f84cb6bd2a2c7c6a" integrity sha512-jtQ6VyT7rMT5tPV0g2EJakEnXLiPksnvlYtwQsVVZ611JsWGN8bQ1tVSDX4s6JllfEH6wmsYxNjTUAMrPmNA8w== +marked@^2.1.1: + version "2.1.3" + resolved "https://registry.npmjs.org/marked/-/marked-2.1.3.tgz#bd017cef6431724fd4b27e0657f5ceb14bff3753" + integrity sha512-/Q+7MGzaETqifOMWYEA7HVMaZb4XbcRfaOzcSsHZEith83KGlvaSG33u0SKu89Mj5h+T8V2hM+8O45Qc5XTgwA== + math-expression-evaluator@^1.2.14: version "1.2.22" resolved "https://registry.npmjs.org/math-expression-evaluator/-/math-expression-evaluator-1.2.22.tgz#c14dcb3d8b4d150e5dcea9c68c8dad80309b0d5e" @@ -20159,11 +20221,6 @@ merge2@^1.2.3, merge2@^1.3.0: resolved "https://registry.npmjs.org/merge2/-/merge2-1.3.0.tgz#5b366ee83b2f1582c48f87e47cf1a9352103ca81" integrity sha512-2j4DAdlBOkiSZIsaXk4mTE3sRS02yBHAtfy127xRV3bQUFqXkjHCHLW6Scv7DwNRbIWNHH8zpnz9zMaKXIdvYw== -merge@^2.1.0: - version "2.1.1" - resolved "https://registry.npmjs.org/merge/-/merge-2.1.1.tgz#59ef4bf7e0b3e879186436e8481c06a6c162ca98" - integrity sha512-jz+Cfrg9GWOZbQAnDQ4hlVnQky+341Yk5ru8bZSe6sIDTCIg8n9i/u7hSQGSVOF3C7lH6mGtqjkiT9G4wFLL0w== - meros@1.1.4, meros@^1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/meros/-/meros-1.1.4.tgz#c17994d3133db8b23807f62bec7f0cb276cfd948" @@ -20977,7 +21034,7 @@ nan@^2.14.1, nan@^2.15.0: resolved "https://registry.npmjs.org/nan/-/nan-2.15.0.tgz#3f34a473ff18e15c1b5626b62903b5ad6e665fee" integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ== -nano-css@^5.1.0, nano-css@^5.3.1: +nano-css@^5.3.1: version "5.3.1" resolved "https://registry.npmjs.org/nano-css/-/nano-css-5.3.1.tgz#b709383e07ad3be61f64edffacb9d98250b87a1f" integrity sha512-ENPIyNzANQRyYVvb62ajDd7PAyIgS2LIUnT9ewih4yrXSZX4hKoUwssy8WjUH++kEOA5wUTMgNnV7ko5n34kUA== @@ -21764,12 +21821,13 @@ open@^8.0.9: is-docker "^2.1.1" is-wsl "^2.2.0" -openapi-sampler@^1.0.0-beta.15: - version "1.0.0-beta.16" - resolved "https://registry.npmjs.org/openapi-sampler/-/openapi-sampler-1.0.0-beta.16.tgz#7813524d5b88d222efb772ceb5a809075d6d9174" - integrity sha512-05+GvwMagTY7GxoDQoWJfmAUFlxfebciiEzqKmu4iq6+MqBEn62AMUkn0CTxyKhnUGIaR2KXjTeslxIeJwVIOw== +openapi-sampler@^1.1.0: + version "1.1.1" + resolved "https://registry.npmjs.org/openapi-sampler/-/openapi-sampler-1.1.1.tgz#7bba7000a03cd8a4630bfbe5b3ef258990c78400" + integrity sha512-WAFsl5SPYuhQwaMTDFOcKhnEY1G1rmamrMiPmJdqwfl1lr81g63/befcsN9BNi0w5/R0L+hfcUj13PANEBeLgg== dependencies: - json-pointer "^0.6.0" + "@types/json-schema" "^7.0.7" + json-pointer "^0.6.1" openid-client@^4.1.1, openid-client@^4.2.1: version "4.9.0" @@ -22266,7 +22324,7 @@ parse5@5.1.1: resolved "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178" integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug== -parse5@^6.0.0: +parse5@6.0.1, parse5@^6.0.0: version "6.0.1" resolved "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b" integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== @@ -23976,11 +24034,6 @@ react-error-overlay@^6.0.9: resolved "https://registry.npmjs.org/react-error-overlay/-/react-error-overlay-6.0.9.tgz#3c743010c9359608c375ecd6bc76f35d93995b0a" integrity sha512-nQTTcUu+ATDbrSD1BZHr5kgSD4oF8OFjxun8uAaL8RwPBacGBNPf/yAuVVdx17N8XNzRDMrZ9XcKZHCjPW+9ew== -react-fast-compare@^2.0.4: - version "2.0.4" - resolved "https://registry.npmjs.org/react-fast-compare/-/react-fast-compare-2.0.4.tgz#e84b4d455b0fec113e0402c329352715196f81f9" - integrity sha512-suNP+J1VU1MWFKcyt7RtjiSWUjvidmQSlqu+eHslq+342xCbGTYmC0mEhPCOHxlW0CywylOC1u2DFAT+bv4dBw== - react-fast-compare@^3.0.1, react-fast-compare@^3.1.1, react-fast-compare@^3.2.0: version "3.2.0" resolved "https://registry.npmjs.org/react-fast-compare/-/react-fast-compare-3.2.0.tgz#641a9da81b6a6320f270e89724fb45a0b39e43bb" @@ -24298,23 +24351,6 @@ react-universal-interface@^0.6.2: resolved "https://registry.npmjs.org/react-universal-interface/-/react-universal-interface-0.6.2.tgz#5e8d438a01729a4dbbcbeeceb0b86be146fe2b3b" integrity sha512-dg8yXdcQmvgR13RIlZbTRQOoUrDciFVoSBZILwjE2LFISxZZ8loVJKAkuzswl5js8BHda79bIb2b84ehU8IjXw== -react-use@^12.2.0: - version "12.13.0" - resolved "https://registry.npmjs.org/react-use/-/react-use-12.13.0.tgz#dfefd8145552841f1c2213c2e79966b505a264ba" - integrity sha512-Kh0m9ezIn9xfRycx4jdAgvsYGstGfjTBO2ecIM3+G0RqrpMxTIL5jjYraHYfUzjGBHf7dUhNYBzm5vw5LItVZA== - dependencies: - "@types/react-wait" "^0.3.0" - copy-to-clipboard "^3.1.0" - nano-css "^5.1.0" - react-fast-compare "^2.0.4" - react-wait "^0.3.0" - resize-observer-polyfill "^1.5.1" - screenfull "^5.0.0" - set-harmonic-interval "^1.0.1" - throttle-debounce "^2.0.1" - ts-easing "^0.2.0" - tslib "^1.10.0" - react-use@^17.2.4: version "17.2.4" resolved "https://registry.npmjs.org/react-use/-/react-use-17.2.4.tgz#1f89be3db0a8237c79253db0a15e12bbe3cfeff1" @@ -24347,11 +24383,6 @@ react-virtualized@^9.21.0: prop-types "^15.6.0" react-lifecycles-compat "^3.0.4" -react-wait@^0.3.0: - version "0.3.0" - resolved "https://registry.npmjs.org/react-wait/-/react-wait-0.3.0.tgz#0cdd4d919012451a5bc3ab0a16d00c6fd9a8c10b" - integrity sha512-kB5x/kMKWcn0uVr9gBdNz21/oGbQwEQnF3P9p6E9yLfJ9DRcKS0fagbgYMFI0YFOoyKDj+2q6Rwax0kTYJF37g== - react@^16.0.0, react@^16.12.0, react@^16.13.1: version "16.13.1" resolved "https://registry.npmjs.org/react/-/react-16.13.1.tgz#2e818822f1a9743122c063d6410d85c1e3afe48e" @@ -25456,7 +25487,7 @@ sax@>=0.6.0, sax@^1.2.4, sax@~1.2.4: resolved "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== -saxes@^5.0.0: +saxes@^5.0.0, saxes@^5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz#eebab953fa3b7608dbe94e5dadb15c888fa6696d" integrity sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw== @@ -25512,7 +25543,7 @@ scoped-regex@^2.0.0: resolved "https://registry.npmjs.org/scoped-regex/-/scoped-regex-2.1.0.tgz#7b9be845d81fd9d21d1ec97c61a0b7cf86d2015f" integrity sha512-g3WxHrqSWCZHGHlSrF51VXFdjImhwvH8ZO/pryFH56Qi0cDsZfylQa/t0jCzVQFNbNvM00HfHjkDPEuarKDSWQ== -screenfull@^5.0.0, screenfull@^5.1.0: +screenfull@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/screenfull/-/screenfull-5.1.0.tgz#85c13c70f4ead4c1b8a935c70010dfdcd2c0e5c8" integrity sha512-dYaNuOdzr+kc6J6CFcBrzkLCfyGcMg+gWkJ8us93IQ7y1cevhQAugFsaCdMHb6lw8KV3xPzSxzH7zM1dQap9mA== @@ -27336,11 +27367,6 @@ throat@^5.0.0: resolved "https://registry.npmjs.org/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b" integrity sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA== -throttle-debounce@^2.0.1: - version "2.2.1" - resolved "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-2.2.1.tgz#fbd933ae6793448816f7d5b3cae259d464c98137" - integrity sha512-i9hAVld1f+woAiyNGqWelpDD5W1tpMroL3NofTz9xzwq6acWBlO2dC8k5EFSZepU6oOINtV5Q3aSPoRg7o4+fA== - throttle-debounce@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-3.0.1.tgz#32f94d84dfa894f786c9a1f290e7a645b6a19abb" @@ -27564,6 +27590,13 @@ tr46@^2.0.2: dependencies: punycode "^2.1.1" +tr46@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" + integrity sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw== + dependencies: + punycode "^2.1.1" + tr46@~0.0.3: version "0.0.3" resolved "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" @@ -28445,6 +28478,13 @@ use-memo-one@^1.1.1: resolved "https://registry.npmjs.org/use-memo-one/-/use-memo-one-1.1.1.tgz#39e6f08fe27e422a7d7b234b5f9056af313bd22c" integrity sha512-oFfsyun+bP7RX8X2AskHNTxu+R3QdE/RC5IefMbqptmACAA/gfol1KDD5KRzPsGMa62sWxGZw+Ui43u6x4ddoQ== +use-resize-observer@^7.0.0: + version "7.1.0" + resolved "https://registry.npmjs.org/use-resize-observer/-/use-resize-observer-7.1.0.tgz#709ea7540fbe0a60ceae41ee2bef933d7782e4d4" + integrity sha512-6DGWOnZpjAGP/MtslGg7OunZptyueQduMi0i8DC5nVKXtJ8Bdt0wR/1tSxugFRndzYCi/jtD+SlNs5PK8ijvXQ== + dependencies: + "@juggle/resize-observer" "^3.3.1" + use@^3.1.0: version "3.1.1" resolved "https://registry.npmjs.org/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" @@ -29052,6 +29092,15 @@ whatwg-url@^8.0.0, whatwg-url@^8.4.0: tr46 "^2.0.2" webidl-conversions "^6.1.0" +whatwg-url@^8.5.0: + version "8.7.0" + resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz#656a78e510ff8f3937bc0bcbe9f5c0ac35941b77" + integrity sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg== + dependencies: + lodash "^4.7.0" + tr46 "^2.1.0" + webidl-conversions "^6.1.0" + which-boxed-primitive@^1.0.1: version "1.0.2" resolved "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" From ef64a444caa88b7e20c417d50466065fb281c9a4 Mon Sep 17 00:00:00 2001 From: Kai Szybiak Date: Thu, 18 Nov 2021 15:46:07 +0100 Subject: [PATCH 02/50] Add changeset Signed-off-by: Kai Szybiak --- .changeset/rotten-candles-poke.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/rotten-candles-poke.md diff --git a/.changeset/rotten-candles-poke.md b/.changeset/rotten-candles-poke.md new file mode 100644 index 0000000000..b8a5b7138b --- /dev/null +++ b/.changeset/rotten-candles-poke.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-api-docs': patch +--- + +Update AsyncAPI component to 1.0.0-x releases From d5699813f79e9ca142b0c79f4af20528e8c70e35 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 3 Jul 2021 13:53:58 +0200 Subject: [PATCH 03/50] Proposed architecture. Signed-off-by: Eric Peterson --- .../architecture-recommended.drawio.svg | 225 +++++++++++++----- 1 file changed, 168 insertions(+), 57 deletions(-) diff --git a/docs/assets/techdocs/architecture-recommended.drawio.svg b/docs/assets/techdocs/architecture-recommended.drawio.svg index e3af4b6b5f..12892cfad7 100644 --- a/docs/assets/techdocs/architecture-recommended.drawio.svg +++ b/docs/assets/techdocs/architecture-recommended.drawio.svg @@ -1,4 +1,4 @@ - + @@ -7,7 +7,7 @@ - + @@ -67,8 +67,8 @@ - - + + @@ -105,7 +105,7 @@ - + @@ -124,7 +124,7 @@ - + @@ -232,34 +232,17 @@
- TechDocs plugin + TechDocs Plugin
- TechDocs plugin + TechDocs Plugin - - - - -
-
-
- TechDocs Backend plugin -
-
-
- - - TechDocs Backend plu... - - - - + @@ -268,21 +251,23 @@
- Request TechDocs site + + Request TechDocs Site +
- Request TechDocs site + Request TechDocs Site - + -
+
Fetch files to render @@ -290,7 +275,7 @@
- + Fetch files to render @@ -302,15 +287,15 @@ - + - + -
+
Source code hosting @@ -323,28 +308,9 @@ - - - - - -
-
-
- Caching -
- (Optional) -
-
-
- - - Caching... - - - - - + + + @@ -379,10 +345,155 @@ + + + + +
+
+
+ Cache Store +
+ (Optional) +
+
+
+ + + Cache Store... + + + + + + + +
+
+
+ Read/Write +
+ Objects +
+
+
+ + + Read/Write... + + + + + + + + + + + +
+
+
+ + Invalidate Objects (Optional) + +
+
+
+ + + Invalidate Objects (Optional) + + + + + + + + + + +
+
+
+ + + Memcache + + +
+
+
+ + + Memcache + + + + + + + + +
+
+
+ + TechDocs Service + +
+
+
+ + + TechDocs Service + + + + + + + +
+
+
+ + Cache Middleware +
+ (Optional) + +
+
+
+ + + Cache Middleware... + + + + + + + + + +
+
+
+ TechDocs Backend Plugin +
+
+
+ + + TechDocs Backend Plugin + + + - + Viewer does not support full SVG 1.1 From 3e443e8e31913e6de83568d5df2be5517c902287 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 3 Jul 2021 20:03:24 +0200 Subject: [PATCH 04/50] Express middleware for reading/writing data to cache. Signed-off-by: Eric Peterson --- .../src/cache/TechDocsCache.ts | 87 +++++++++++ .../src/cache/cacheMiddleware.ts | 139 ++++++++++++++++++ plugins/techdocs-backend/src/cache/index.ts | 17 +++ 3 files changed, 243 insertions(+) create mode 100644 plugins/techdocs-backend/src/cache/TechDocsCache.ts create mode 100644 plugins/techdocs-backend/src/cache/cacheMiddleware.ts create mode 100644 plugins/techdocs-backend/src/cache/index.ts diff --git a/plugins/techdocs-backend/src/cache/TechDocsCache.ts b/plugins/techdocs-backend/src/cache/TechDocsCache.ts new file mode 100644 index 0000000000..83497608cf --- /dev/null +++ b/plugins/techdocs-backend/src/cache/TechDocsCache.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { CacheClient } from '@backstage/backend-common'; +import { Logger } from 'winston'; + +export class CacheInvalidationError extends Error { + public readonly rejections: PromiseRejectedResult[]; + + constructor(rejections: PromiseRejectedResult[]) { + super(); + this.rejections = rejections; + } +} + +export class TechDocsCache { + protected readonly cache: CacheClient; + protected readonly logger: Logger; + + constructor({ cache, logger }: { cache: CacheClient; logger: Logger }) { + this.cache = cache; + this.logger = logger; + } + + async get(path: string): Promise { + try { + // Promise.race ensures we don't hang the client for long if the cache is + // temporarily unreachable. + const response = (await Promise.race([ + this.cache.get(path), + new Promise(cancelAfter => setTimeout(cancelAfter, 1000)), + ])) as string | undefined; + + if (response !== undefined) { + this.logger.debug(`Cache hit: ${path}`); + return Buffer.from(response, 'base64'); + } + + this.logger.debug(`Cache miss: ${path}`); + return response; + } catch (e) { + this.logger.warn(`Error getting cache entry ${path}: ${e.message}`); + this.logger.debug(e.stack); + return undefined; + } + } + + async set(path: string, data: Buffer): Promise { + this.logger.debug(`Writing cache entry for ${path}`); + this.cache + .set(path, data.toString('base64')) + .catch(e => this.logger.error('write error', e)); + } + + async invalidate(path: string): Promise { + return this.cache.delete(path); + } + + async invalidateMultiple( + paths: string[], + ): Promise[]> { + const settled = await Promise.allSettled( + paths.map(path => this.cache.delete(path)), + ); + const rejected = settled.filter( + s => s.status === 'rejected', + ) as PromiseRejectedResult[]; + + if (rejected.length) { + throw new CacheInvalidationError(rejected); + } + + return settled; + } +} diff --git a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts new file mode 100644 index 0000000000..2abbb24cf3 --- /dev/null +++ b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts @@ -0,0 +1,139 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Router, Request, json } from 'express'; +import router from 'express-promise-router'; +import { Logger } from 'winston'; +import { TechDocsCache } from '.'; +import { CacheInvalidationError } from './TechDocsCache'; + +type CacheClearRequestParams = { + objects: string[]; +}; + +type CacheClearRequest = Request< + any, + unknown, + CacheClearRequestParams, + unknown +>; + +type CacheMiddlewareOptions = { + cache: TechDocsCache; + logger: Logger; +}; + +type ErrorCallback = (err?: Error) => void; + +export const createCacheMiddleware = ({ + cache, + logger, +}: CacheMiddlewareOptions): Router => { + const cacheMiddleware = router(); + + // And endpoint for handling cache invalidation external to the Backstage + // Backend (e.g. from the TechDocs CLI). + cacheMiddleware.use(json()); + cacheMiddleware.post( + '/cache/invalidate', + async (req: CacheClearRequest, res) => { + if (req.body?.objects?.length) { + logger.debug( + `Clearing ${req.body.objects.length} cache entries: (eg: ${req.body.objects[0]})`, + ); + + try { + const invalidated = await cache.invalidateMultiple(req.body.objects); + logger.debug( + `Successfully invalidated ${invalidated.length} cache entries`, + ); + res.status(204).send(); + } catch (e) { + if (e instanceof CacheInvalidationError) { + const uniqueReasons = [ + ...new Set(e.rejections.map(r => r.reason.message)), + ].join(', '); + logger.warn( + `Problem invalidating ${e.rejections.length} entries: ${uniqueReasons}`, + ); + } + res.status(500).send(); + } + } else { + res.status(400).send(); + } + }, + ); + + // Middleware that, through socket monkey patching, captures responses as + // they're sent over /static/docs/* and caches them. Subsequent requests are + // loaded from cache. Cache key is the object's path (after `/static/docs/`). + cacheMiddleware.use(async (req, res, next) => { + const socket = res.socket; + const isCacheable = req.path.includes('/static/docs/'); + + // Continue early if this is non-cacheable, or there's no socket. + if (!isCacheable || !socket) { + next(); + return; + } + + // Make concrete references to these things. + const reqPath = decodeURI(req.path.match(/\/static\/docs\/(.*)$/)![1]); + const realEnd = socket.end.bind(socket); + const realWrite = socket.write.bind(socket); + let writeToCache = true; + const chunks: Buffer[] = []; + + // Monkey-patch the response's socket to keep track of chunks as they are + // written over the wire. + socket.write = ( + data, + encoding?: BufferEncoding | ErrorCallback, + callback?: ErrorCallback, + ) => { + chunks.push(Buffer.from(data)); + if (typeof encoding === 'function') { + return realWrite(data, encoding); + } + return realWrite(data, encoding, callback); + }; + + // When a socket is closed, if there were no errors and the data written + // over the socket should be cached, cache it as a base64-encoded string! + socket.on('close', hadError => { + if (writeToCache && !hadError) { + cache.set(reqPath, Buffer.concat(chunks)); + } + }); + + // Attempt to retrieve data from the cache. + const cached = await cache.get(reqPath); + + // If there is a cache hit, write it out on the socket, ensure we don't re- + // cache the data, and prevent going back to canonical storage by never + // calling next(). + if (cached) { + writeToCache = false; + realEnd(cached); + return; + } + + // No data retrieved from cache: allow retrieval from canonical storage. + next(); + }); + + return cacheMiddleware; +}; diff --git a/plugins/techdocs-backend/src/cache/index.ts b/plugins/techdocs-backend/src/cache/index.ts new file mode 100644 index 0000000000..751d8e8625 --- /dev/null +++ b/plugins/techdocs-backend/src/cache/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { createCacheMiddleware } from './cacheMiddleware'; +export { TechDocsCache } from './TechDocsCache'; From 0a44eb7d5297bd61a7f224505f4711b60e7a87a2 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 24 Jul 2021 17:00:22 +0200 Subject: [PATCH 05/50] Initial implementation of TechDocs cache in backend plugin Signed-off-by: Eric Peterson --- plugins/techdocs-backend/config.d.ts | 17 ++++++++++++++++ .../src/DocsBuilder/builder.ts | 12 ++++++++++- .../techdocs-backend/src/service/router.ts | 20 ++++++++++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index 9e563137bd..1767120a35 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -226,6 +226,23 @@ export interface Config { }; }; + /** + * @example http://localhost:7007/api/techdocs + * Techdocs cache information + */ + cache?: { + /** + * The cache time-to-live for TechDocs sites (in milliseconds). Set this + * to a non-zero value to cache TechDocs sites and assets as they are + * read from storage. + * + * Note: you must also configure `backend.cache` appropriately as well, + * and to pass a PluginCacheManager instance to TechDocs Backend's + * createRouter method in your backend. + */ + ttl: number; + }; + /** * @example http://localhost:7007/api/techdocs * @visibility frontend diff --git a/plugins/techdocs-backend/src/DocsBuilder/builder.ts b/plugins/techdocs-backend/src/DocsBuilder/builder.ts index 821b3478a0..4843acf2c2 100644 --- a/plugins/techdocs-backend/src/DocsBuilder/builder.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/builder.ts @@ -36,6 +36,7 @@ import path from 'path'; import { Writable } from 'stream'; import { Logger } from 'winston'; import { BuildMetadataStorage } from './BuildMetadataStorage'; +import { TechDocsCache } from '../cache'; type DocsBuilderArguments = { preparers: PreparerBuilder; @@ -46,6 +47,7 @@ type DocsBuilderArguments = { config: Config; scmIntegrations: ScmIntegrationRegistry; logStream?: Writable; + cache?: TechDocsCache; }; export class DocsBuilder { @@ -57,6 +59,7 @@ export class DocsBuilder { private config: Config; private scmIntegrations: ScmIntegrationRegistry; private logStream: Writable | undefined; + private cache?: TechDocsCache; constructor({ preparers, @@ -67,6 +70,7 @@ export class DocsBuilder { config, scmIntegrations, logStream, + cache, }: DocsBuilderArguments) { this.preparer = preparers.get(entity); this.generator = generators.get(entity); @@ -76,6 +80,7 @@ export class DocsBuilder { this.config = config; this.scmIntegrations = scmIntegrations; this.logStream = logStream; + this.cache = cache; } /** @@ -210,11 +215,16 @@ export class DocsBuilder { )}`, ); - await this.publisher.publish({ + const published = await this.publisher.publish({ entity: this.entity, directory: outputDir, }); + // Invalidate the cache for any published objects. + if (this.cache && published?.objects?.length) { + await this.cache.invalidateMultiple(published.objects); + } + try { // Not a blocker hence no need to await this. fs.remove(outputDir); diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index d53a2faed4..07c6887ef7 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { + PluginEndpointDiscovery, + PluginCacheManager, +} from '@backstage/backend-common'; import { CatalogClient } from '@backstage/catalog-client'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; @@ -31,6 +34,7 @@ import { Knex } from 'knex'; import { Logger } from 'winston'; import { ScmIntegrations } from '@backstage/integration'; import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer'; +import { createCacheMiddleware, TechDocsCache } from '../cache'; /** * All of the required dependencies for running TechDocs in the "out-of-the-box" @@ -44,6 +48,7 @@ type OutOfTheBoxDeploymentOptions = { discovery: PluginEndpointDiscovery; database?: Knex; // TODO: Make database required when we're implementing database stuff. config: Config; + cache?: PluginCacheManager; }; /** @@ -88,6 +93,14 @@ export async function createRouter( scmIntegrations, }); + // Set up a cache client if configured. + let cache: TechDocsCache | undefined; + const defaultTtl = config.getOptionalNumber('techdocs.cache.ttl'); + if (isOutOfTheBoxOption(options) && options.cache && defaultTtl) { + const cacheClient = options.cache.getClient({ defaultTtl }); + cache = new TechDocsCache({ cache: cacheClient, logger }); + } + router.get('/metadata/techdocs/:namespace/:kind/:name', async (req, res) => { const { kind, namespace, name } = req.params; const entityName = { kind, namespace, name }; @@ -199,6 +212,11 @@ export async function createRouter( ); }); + // If a cache manager was provided, attach the cache middleware. + if (cache) { + router.use(createCacheMiddleware({ logger, cache })); + } + // Route middleware which serves files from the storage set in the publisher. router.use('/static/docs', publisher.docsRouter()); From 8b438c77176d2df962a85abbca1f98a096d02092 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Nov 2021 10:43:05 +0100 Subject: [PATCH 06/50] Update all Publishers so that they resolve a list of objects on successful publish. Signed-off-by: Eric Peterson --- .../src/stages/publish/awsS3.test.ts | 8 +++++- .../src/stages/publish/awsS3.ts | 9 ++++++- .../stages/publish/azureBlobStorage.test.ts | 8 +++++- .../src/stages/publish/azureBlobStorage.ts | 23 +++++++++++------ .../src/stages/publish/googleStorage.test.ts | 8 +++++- .../src/stages/publish/googleStorage.ts | 25 ++++++++++++------- .../src/stages/publish/local.ts | 25 +++++++++++-------- .../src/stages/publish/openStackSwift.test.ts | 8 +++++- .../src/stages/publish/openStackSwift.ts | 11 ++++++-- .../src/stages/publish/types.ts | 6 +++++ 10 files changed, 97 insertions(+), 34 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index 1f49668d35..abae5fda44 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -171,7 +171,13 @@ describe('AwsS3Publish', () => { bucketRootPath: 'backstage-data/techdocs', legacyUseCaseSensitiveTripletPaths: true, }); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'test-namespace/TestKind/test-component-name/404.html', + `test-namespace/TestKind/test-component-name/index.html`, + `test-namespace/TestKind/test-component-name/assets/main.css`, + ]), + }); }); it('should publish a directory when sse is specified', async () => { diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index abfa8ddd24..ed76edc0cb 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -39,6 +39,7 @@ import { import { PublisherBase, PublishRequest, + PublishResponse, ReadinessResponse, TechDocsMetadata, } from './types'; @@ -214,7 +215,11 @@ export class AwsS3Publish implements PublisherBase { * Upload all the files from the generated `directory` to the S3 bucket. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ - async publish({ entity, directory }: PublishRequest): Promise { + async publish({ + entity, + directory, + }: PublishRequest): Promise { + const objects: string[] = []; const useLegacyPathCasing = this.legacyPathCasing; const bucketRootPath = this.bucketRootPath; const sse = this.sse; @@ -263,6 +268,7 @@ export class AwsS3Publish implements PublisherBase { ...(sse && { ServerSideEncryption: sse }), } as aws.S3.PutObjectRequest; + objects.push(params.Key); return this.storageClient.upload(params).promise(); }, absoluteFilesToUpload, @@ -311,6 +317,7 @@ export class AwsS3Publish implements PublisherBase { const errorMessage = `Unable to delete file(s) from AWS S3. ${error}`; this.logger.error(errorMessage); } + return { objects }; } async fetchTechDocsMetadata( diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index a2503bd7b2..0e261f7f86 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -161,7 +161,13 @@ describe('AzureBlobStoragePublish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'test-namespace/TestKind/test-component-name/404.html', + `test-namespace/TestKind/test-component-name/index.html`, + `test-namespace/TestKind/test-component-name/assets/main.css`, + ]), + }); }); it('should fail to publish a directory', async () => { diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index 7a58d92095..b082079be2 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -39,6 +39,7 @@ import { import { PublisherBase, PublishRequest, + PublishResponse, ReadinessResponse, TechDocsMetadata, } from './types'; @@ -156,7 +157,11 @@ export class AzureBlobStoragePublish implements PublisherBase { * Upload all the files from the generated `directory` to the Azure Blob Storage container. * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html */ - async publish({ entity, directory }: PublishRequest): Promise { + async publish({ + entity, + directory, + }: PublishRequest): Promise { + const objects: string[] = []; const useLegacyPathCasing = this.legacyPathCasing; // First, try to retrieve a list of all individual files currently existing @@ -194,14 +199,14 @@ export class AzureBlobStoragePublish implements PublisherBase { const relativeFilePath = path.normalize( path.relative(directory, absoluteFilePath), ); + const remotePath = getCloudPathForLocalPath( + entity, + relativeFilePath, + useLegacyPathCasing, + ); + objects.push(remotePath); const response = await container - .getBlockBlobClient( - getCloudPathForLocalPath( - entity, - relativeFilePath, - useLegacyPathCasing, - ), - ) + .getBlockBlobClient(remotePath) .uploadFile(absoluteFilePath); if (response._response.status >= 400) { @@ -264,6 +269,8 @@ export class AzureBlobStoragePublish implements PublisherBase { const errorMessage = `Unable to delete file(s) from Azure. ${error}`; this.logger.error(errorMessage); } + + return { objects }; } private download(containerName: string, blobPath: string): Promise { diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index a52fd0f0bd..ea97d81269 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -164,7 +164,13 @@ describe('GoogleGCSPublish', () => { bucketRootPath: 'backstage-data/techdocs', legacyUseCaseSensitiveTripletPaths: true, }); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'test-namespace/TestKind/test-component-name/404.html', + `test-namespace/TestKind/test-component-name/index.html`, + `test-namespace/TestKind/test-component-name/assets/main.css`, + ]), + }); }); it('should fail to publish a directory', async () => { diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index f072412c20..ba70c36e27 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -36,6 +36,7 @@ import { MigrateWriteStream } from './migrations'; import { PublisherBase, PublishRequest, + PublishResponse, ReadinessResponse, TechDocsMetadata, } from './types'; @@ -145,7 +146,11 @@ export class GoogleGCSPublish implements PublisherBase { * Upload all the files from the generated `directory` to the GCS bucket. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ - async publish({ entity, directory }: PublishRequest): Promise { + async publish({ + entity, + directory, + }: PublishRequest): Promise { + const objects: string[] = []; const useLegacyPathCasing = this.legacyPathCasing; const bucket = this.storageClient.bucket(this.bucketName); const bucketRootPath = this.bucketRootPath; @@ -178,14 +183,14 @@ export class GoogleGCSPublish implements PublisherBase { await bulkStorageOperation( async absoluteFilePath => { const relativeFilePath = path.relative(directory, absoluteFilePath); - return await bucket.upload(absoluteFilePath, { - destination: getCloudPathForLocalPath( - entity, - relativeFilePath, - useLegacyPathCasing, - bucketRootPath, - ), - }); + const destination = getCloudPathForLocalPath( + entity, + relativeFilePath, + useLegacyPathCasing, + bucketRootPath, + ); + objects.push(destination); + return await bucket.upload(absoluteFilePath, { destination }); }, absoluteFilesToUpload, { concurrencyLimit: 10 }, @@ -228,6 +233,8 @@ export class GoogleGCSPublish implements PublisherBase { const errorMessage = `Unable to delete file(s) from Google Cloud Storage. ${error}`; this.logger.error(errorMessage); } + + return { objects }; } fetchTechDocsMetadata(entityName: EntityName): Promise { diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 70b4eb3ff2..67a4a22914 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -113,7 +113,7 @@ export class LocalPublish implements PublisherBase { } return new Promise((resolve, reject) => { - fs.copy(directory, publishDir, err => { + fs.copy(directory, publishDir, async err => { if (err) { this.logger.debug( `Failed to copy docs from ${directory} to ${publishDir}`, @@ -121,16 +121,21 @@ export class LocalPublish implements PublisherBase { reject(err); } this.logger.info(`Published site stored at ${publishDir}`); - this.discovery - .getBaseUrl('techdocs') - .then(techdocsApiUrl => { - resolve({ - remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`, - }); - }) - .catch(reason => { - reject(reason); + + try { + const techdocsApiUrl = await this.discovery.getBaseUrl('techdocs'); + const objects = (await getFileTreeRecursively(publishDir)).map( + abs => { + return abs.split(`${staticDocsDir}/`)[1]; + }, + ); + resolve({ + remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`, + objects, }); + } catch (reason) { + reject(reason); + } }); }); } diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts index c2236fb880..b8ad43aac3 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts @@ -170,7 +170,13 @@ describe('OpenStackSwiftPublish', () => { entity, directory: entityRootDir, }), - ).toBeUndefined(); + ).toMatchObject({ + objects: expect.arrayContaining([ + 'test-namespace/TestKind/test-component-name/404.html', + `test-namespace/TestKind/test-component-name/index.html`, + `test-namespace/TestKind/test-component-name/assets/main.css`, + ]), + }); }); it('should fail to publish a directory', async () => { diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.ts index 7b623933e3..62b40f9a76 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.ts @@ -32,6 +32,7 @@ import { import { PublisherBase, PublishRequest, + PublishResponse, ReadinessResponse, TechDocsMetadata, } from './types'; @@ -139,8 +140,13 @@ export class OpenStackSwiftPublish implements PublisherBase { * Upload all the files from the generated `directory` to the OpenStack Swift container. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ - async publish({ entity, directory }: PublishRequest): Promise { + async publish({ + entity, + directory, + }: PublishRequest): Promise { try { + const objects: string[] = []; + // Note: OpenStack Swift manages creation of parent directories if they do not exist. // So collecting path of only the files is good enough. const allFilesToUpload = await getFileTreeRecursively(directory); @@ -161,6 +167,7 @@ export class OpenStackSwiftPublish implements PublisherBase { // The / delimiter is intentional since it represents the cloud storage and not the local file system. const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Swift container file relative path + objects.push(destination); // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(async () => { @@ -178,7 +185,7 @@ export class OpenStackSwiftPublish implements PublisherBase { this.logger.info( `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, ); - return; + return { objects }; } catch (e) { const errorMessage = `Unable to upload file(s) to OpenStack Swift. ${e}`; this.logger.error(errorMessage); diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index 229c853427..f68cb9cc8f 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -35,6 +35,12 @@ export type PublishRequest = { /* `remoteUrl` is the URL which serves files from the local publisher's static directory. */ export type PublishResponse = { remoteUrl?: string; + /** + * The list of objects (specifically their paths) that were published. + * Objects should not have a preceding slash, and should match how one would + * load the object over the `/static/docs/` TechDocs Backend Plugin endpoint. + */ + objects?: string[]; } | void; /** From 5fe0c2ce7879b961bd1fe2209f4603b2c8b23b87 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 3 Jul 2021 20:07:13 +0200 Subject: [PATCH 07/50] Wire up example backend to use in-memory cache. Signed-off-by: Eric Peterson --- app-config.yaml | 3 ++- packages/backend/src/plugins/techdocs.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app-config.yaml b/app-config.yaml index 913a72dbab..881e97aa5f 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -112,7 +112,8 @@ techdocs: # pullImage: true # or false to disable automatic pulling of image (e.g. if custom docker login is required) publisher: type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureBlobStorage' or 'openStackSwift'. Read documentation for using alternatives. - + cache: + ttl: 60000 # 1 minute cache for demonstration purposes. You may wish to set this higher in production. sentry: organization: my-company diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index eb1e0502db..c32bbccbb0 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -29,6 +29,7 @@ export default async function createPlugin({ config, discovery, reader, + cache, }: PluginEnvironment): Promise { // Preparers are responsible for fetching source files for documentation. const preparers = await Preparers.fromConfig(config, { @@ -64,5 +65,6 @@ export default async function createPlugin({ logger, config, discovery, + cache, }); } From 790f02c898df9e053a302097d0992f58cb96d05a Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 6 Jul 2021 10:14:00 +0200 Subject: [PATCH 08/50] Improve types/documentation. Signed-off-by: Eric Peterson --- packages/techdocs-common/src/stages/publish/types.ts | 12 +++++++++--- plugins/techdocs-backend/src/DocsBuilder/builder.ts | 2 +- .../techdocs-backend/src/cache/cacheMiddleware.ts | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index f68cb9cc8f..ab497067b3 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -32,13 +32,19 @@ export type PublishRequest = { directory: string; }; -/* `remoteUrl` is the URL which serves files from the local publisher's static directory. */ +/** + * Response containing metadata about where files were published and what may + * have been published or updated. + */ export type PublishResponse = { + /** + * The URL which serves files from the local publisher's static directory. + */ remoteUrl?: string; /** * The list of objects (specifically their paths) that were published. - * Objects should not have a preceding slash, and should match how one would - * load the object over the `/static/docs/` TechDocs Backend Plugin endpoint. + * Objects do not have a preceding slash, and match how one would load the + * object over the `/static/docs/*` TechDocs Backend Plugin endpoint. */ objects?: string[]; } | void; diff --git a/plugins/techdocs-backend/src/DocsBuilder/builder.ts b/plugins/techdocs-backend/src/DocsBuilder/builder.ts index 4843acf2c2..1d725bd69e 100644 --- a/plugins/techdocs-backend/src/DocsBuilder/builder.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/builder.ts @@ -221,7 +221,7 @@ export class DocsBuilder { }); // Invalidate the cache for any published objects. - if (this.cache && published?.objects?.length) { + if (this.cache && published && published?.objects?.length) { await this.cache.invalidateMultiple(published.objects); } diff --git a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts index 2abbb24cf3..2650b39296 100644 --- a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts +++ b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts @@ -100,7 +100,7 @@ export const createCacheMiddleware = ({ // Monkey-patch the response's socket to keep track of chunks as they are // written over the wire. socket.write = ( - data, + data: string | Uint8Array, encoding?: BufferEncoding | ErrorCallback, callback?: ErrorCallback, ) => { From 0a5278b92afcad9cb009c5e8e5f5760f5d791f2a Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Nov 2021 10:46:53 +0100 Subject: [PATCH 09/50] Update Backstage.io docs with config/recs Signed-off-by: Eric Peterson --- .../architecture-recommended.drawio.svg | 20 +++++++++---------- docs/features/techdocs/architecture.md | 14 ++++++------- docs/features/techdocs/configuration.md | 9 +++++++++ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/docs/assets/techdocs/architecture-recommended.drawio.svg b/docs/assets/techdocs/architecture-recommended.drawio.svg index 12892cfad7..8ddc9dc35a 100644 --- a/docs/assets/techdocs/architecture-recommended.drawio.svg +++ b/docs/assets/techdocs/architecture-recommended.drawio.svg @@ -1,4 +1,4 @@ - + @@ -383,15 +383,15 @@ - - - - - + + + + + -
+
@@ -401,7 +401,7 @@
- + Invalidate Objects (Optional) @@ -471,8 +471,8 @@ - - + + diff --git a/docs/features/techdocs/architecture.md b/docs/features/techdocs/architecture.md index 8d25b4047f..23af83d09b 100644 --- a/docs/features/techdocs/architecture.md +++ b/docs/features/techdocs/architecture.md @@ -40,7 +40,7 @@ storage system (e.g. AWS S3, GCS or Azure Blob Storage). Read more in ## Recommended deployment -This is how we recommend deploying TechDocs in production environment. +This is how we recommend deploying TechDocs in a production environment. TechDocs Architecture diagram @@ -58,12 +58,12 @@ Similar to how it is done in the Basic setup, the TechDocs Reader requests your configured storage solution for the necessary files and returns them to TechDocs Reader. -Note about caching: We have noticed internally that some storage providers can -be quite slow, which is why we are recommending a cache that sits between the -TechDocs Reader and the Storage. - -_Feel free to suggest better ideas to us in #docs-like-code channel in Discord -or via a GitHub issue._ +Depending on your chosen cloud storage provider and its real-world proximity to +your backend server, there may be a comparably high amount of latency when +loading TechDocs sites using this deployment approach. If you encounter this, +you can optionally configure the `techdocs-backend` to cache responses in a +cache store +[supported by Backstage](../../overview/architecture-overview.md#cache). ### Security consideration diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 4148749a62..3234cdc034 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -135,6 +135,15 @@ techdocs: # the old, case-sensitive entity triplet behavior. legacyUseCaseSensitiveTripletPaths: false + # techdocs.cache is optional, and is only recommended when you've configured + # an external techdocs.publisher.type above. Also requires backend.cache to + # be configured with a valid cache store. + cache: + # Represents the number of milliseconds a statically built asset should + # stay cached. Cache invalidation is handled automatically if you publish + # to storage using the techdocs-cli, allowing long TTLs (e.g. 1 month/year) + ttl: 3600000 + # (Optional and Legacy) TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc. # You don't have to specify this anymore. From 12ce46b229ad440eac9c73647b06816d1536bc5a Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 24 Jul 2021 17:04:24 +0200 Subject: [PATCH 10/50] Update TechDocs Backend API Report Signed-off-by: Eric Peterson --- plugins/techdocs-backend/api-report.md | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index f9be757e6a..9736555687 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -10,6 +10,7 @@ import express from 'express'; import { GeneratorBuilder } from '@backstage/techdocs-common'; import { Knex } from 'knex'; import { Logger as Logger_2 } from 'winston'; +import { PluginCacheManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PreparerBuilder } from '@backstage/techdocs-common'; import { PublisherBase } from '@backstage/techdocs-common'; From d48aa5ca1c1669f8764ca45683e4b2e72505aa78 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 6 Jul 2021 11:34:04 +0200 Subject: [PATCH 11/50] Update create-app so that cache manager is passed through to router. Signed-off-by: Eric Peterson --- .../default-app/packages/backend/src/plugins/techdocs.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts index 906d86d4a2..054c64db65 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts @@ -14,6 +14,7 @@ export default async function createPlugin({ config, discovery, reader, + cache, }: PluginEnvironment): Promise { // Preparers are responsible for fetching source files for documentation. const preparers = await Preparers.fromConfig(config, { @@ -49,5 +50,6 @@ export default async function createPlugin({ logger, config, discovery, + cache, }); } From 1bada775a9e15f6407d3bf0afcd779b9fc429dea Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 6 Jul 2021 13:13:46 +0200 Subject: [PATCH 12/50] Changesets for affected packages. Signed-off-by: Eric Peterson --- .changeset/techdocs-satisfied-you-blinked.md | 7 ++++ .changeset/the-renegade-feeling.md | 42 ++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 .changeset/techdocs-satisfied-you-blinked.md create mode 100644 .changeset/the-renegade-feeling.md diff --git a/.changeset/techdocs-satisfied-you-blinked.md b/.changeset/techdocs-satisfied-you-blinked.md new file mode 100644 index 0000000000..4cbc9d4f3e --- /dev/null +++ b/.changeset/techdocs-satisfied-you-blinked.md @@ -0,0 +1,7 @@ +--- +'@backstage/techdocs-common': minor +'@backstage/plugin-techdocs-backend': minor +--- + +Added the ability for the TechDocs Backend to (optionally) leverage a cache +store to improve performance when reading files from a cloud storage provider. diff --git a/.changeset/the-renegade-feeling.md b/.changeset/the-renegade-feeling.md new file mode 100644 index 0000000000..c9a4b1965e --- /dev/null +++ b/.changeset/the-renegade-feeling.md @@ -0,0 +1,42 @@ +--- +'@backstage/create-app': patch +--- + +TechDocs Backend may now (optionally) leverage a cache store to improve +performance when reading content from a cloud storage provider. + +To apply this change to an existing app, pass the cache manager from the plugin +environment to the `createRouter` function in your backend: + +```diff +// packages/backend/src/plugins/techdocs.ts + +export default async function createPlugin({ + logger, + config, + discovery, + reader, ++ cache, +}: PluginEnvironment): Promise { + + // ... + + return await createRouter({ + preparers, + generators, + publisher, + logger, + config, + discovery, ++ cache, + }); +``` + +If your `PluginEnvironment` does not include a cache manager, be sure you've +applied [the cache management change][cm-change] to your backend as well. + +[Additional configuration][td-rec-arch] is required if you wish to enable +caching in TechDocs. + +[cm-change]: https://github.com/backstage/backstage/blob/master/packages/create-app/CHANGELOG.md#patch-changes-6 +[td-rec-arch]: https://backstage.io/docs/features/techdocs/architecture#recommended-deployment From 693db1da5487cc1c5d37ccfa495a4994eb3f30de Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 24 Jul 2021 17:45:48 +0200 Subject: [PATCH 13/50] Review feedback. Signed-off-by: Eric Peterson --- app-config.yaml | 2 +- .../src/stages/publish/local.ts | 47 +++++++++---------- .../src/cache/cacheMiddleware.ts | 2 +- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 881e97aa5f..bde037dfe3 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -113,7 +113,7 @@ techdocs: publisher: type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureBlobStorage' or 'openStackSwift'. Read documentation for using alternatives. cache: - ttl: 60000 # 1 minute cache for demonstration purposes. You may wish to set this higher in production. + ttl: 3600000 # 1 hour cache for demonstration purposes. You may wish to set this higher in production. sentry: organization: my-company diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 67a4a22914..91a9b57a9e 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -98,7 +98,10 @@ export class LocalPublish implements PublisherBase { }; } - publish({ entity, directory }: PublishRequest): Promise { + async publish({ + entity, + directory, + }: PublishRequest): Promise { const entityNamespace = entity.metadata.namespace ?? 'default'; const publishDir = this.staticEntityPathJoin( @@ -112,32 +115,26 @@ export class LocalPublish implements PublisherBase { fs.mkdirSync(publishDir, { recursive: true }); } - return new Promise((resolve, reject) => { - fs.copy(directory, publishDir, async err => { - if (err) { - this.logger.debug( - `Failed to copy docs from ${directory} to ${publishDir}`, - ); - reject(err); - } - this.logger.info(`Published site stored at ${publishDir}`); + try { + await fs.copy(directory, publishDir); + this.logger.info(`Published site stored at ${publishDir}`); + } catch (error) { + this.logger.debug( + `Failed to copy docs from ${directory} to ${publishDir}`, + ); + throw error; + } - try { - const techdocsApiUrl = await this.discovery.getBaseUrl('techdocs'); - const objects = (await getFileTreeRecursively(publishDir)).map( - abs => { - return abs.split(`${staticDocsDir}/`)[1]; - }, - ); - resolve({ - remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`, - objects, - }); - } catch (reason) { - reject(reason); - } - }); + // Generate publish response. + const techdocsApiUrl = await this.discovery.getBaseUrl('techdocs'); + const objects = (await getFileTreeRecursively(publishDir)).map(abs => { + return abs.split(`${staticDocsDir}/`)[1]; }); + + return { + remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`, + objects, + }; } async fetchTechDocsMetadata( diff --git a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts index 2650b39296..0c06052ba4 100644 --- a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts +++ b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts @@ -112,7 +112,7 @@ export const createCacheMiddleware = ({ }; // When a socket is closed, if there were no errors and the data written - // over the socket should be cached, cache it as a base64-encoded string! + // over the socket should be cached, cache it! socket.on('close', hadError => { if (writeToCache && !hadError) { cache.set(reqPath, Buffer.concat(chunks)); From a16dce0433c161dad036aec5a85dfeaad13e8fa6 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 24 Jul 2021 18:46:28 +0200 Subject: [PATCH 14/50] Tests for TechDocsCache class Signed-off-by: Eric Peterson --- .../src/cache/TechDocsCache.test.ts | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 plugins/techdocs-backend/src/cache/TechDocsCache.test.ts diff --git a/plugins/techdocs-backend/src/cache/TechDocsCache.test.ts b/plugins/techdocs-backend/src/cache/TechDocsCache.test.ts new file mode 100644 index 0000000000..e4308b7c3b --- /dev/null +++ b/plugins/techdocs-backend/src/cache/TechDocsCache.test.ts @@ -0,0 +1,147 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CacheClient, getVoidLogger } from '@backstage/backend-common'; +import { CacheInvalidationError, TechDocsCache } from './TechDocsCache'; + +const cached = (str: string): string => { + return Buffer.from(str).toString('base64'); +}; + +describe('TechDocsCache', () => { + let CacheUnderTest: TechDocsCache; + let MockClient: jest.Mocked; + + beforeEach(() => { + MockClient = { + get: jest.fn(), + set: jest.fn(), + delete: jest.fn(), + }; + CacheUnderTest = new TechDocsCache({ + cache: MockClient, + logger: getVoidLogger(), + }); + }); + + describe('get', () => { + it('returns undefined if no response', async () => { + const expectedPath = 'some/index.html'; + MockClient.get.mockResolvedValueOnce(undefined); + + const actual = await CacheUnderTest.get(expectedPath); + expect(MockClient.get).toHaveBeenCalledWith(expectedPath); + expect(actual).toBe(undefined); + }); + + it('returns undefined if cache get throws', async () => { + const expectedPath = 'some/index.html'; + MockClient.get.mockRejectedValueOnce(new Error()); + + const actual = await CacheUnderTest.get(expectedPath); + expect(actual).toBe(undefined); + }); + + it('returns undefined if no response after 1s', async () => { + const expectedPath = 'some/index.html'; + MockClient.get.mockImplementationOnce(() => { + return new Promise(resolve => { + setTimeout(() => resolve(cached('value')), 1500); + }); + }); + const actual = await CacheUnderTest.get(expectedPath); + expect(actual).toBe(undefined); + }); + + it('returns data if cache get returns it', async () => { + const expectedPath = 'some/index.html'; + MockClient.get.mockResolvedValueOnce(cached('expected value')); + + const actual = await CacheUnderTest.get(expectedPath); + expect(actual?.toString()).toBe('expected value'); + }); + }); + + describe('set', () => { + it('sets a base64-encoded string', async () => { + const expectedPath = 'some/index.html'; + MockClient.set.mockResolvedValueOnce(undefined); + + await CacheUnderTest.set(expectedPath, Buffer.from('some data')); + expect(MockClient.set).toHaveBeenCalledWith( + expectedPath, + cached('some data'), + ); + }); + + it('does not throw if client throws', () => { + MockClient.set.mockRejectedValueOnce(new Error()); + expect(() => CacheUnderTest.set('i.html', Buffer.from(''))).not.toThrow(); + }); + }); + + describe('invalidate', () => { + it('calls delete on client', async () => { + const expectedPath = 'some/index.html'; + MockClient.delete.mockResolvedValueOnce(undefined); + + await CacheUnderTest.invalidate(expectedPath); + expect(MockClient.delete).toHaveBeenCalledWith(expectedPath); + }); + }); + + describe('invalidateMultiple', () => { + it('calls delete once per given path', async () => { + const expectedPaths = ['one/index.html', 'two/index.html']; + MockClient.delete.mockResolvedValue(undefined); + + await CacheUnderTest.invalidateMultiple(expectedPaths); + expect(MockClient.delete).toHaveBeenNthCalledWith(1, expectedPaths[0]); + expect(MockClient.delete).toHaveBeenNthCalledWith(2, expectedPaths[1]); + }); + + it('returns an array of as many paths provided', async () => { + const expectedPaths = ['one/index.html', 'two/index.html']; + MockClient.delete.mockResolvedValue(undefined); + + const actual = await CacheUnderTest.invalidateMultiple(expectedPaths); + expect(actual.length).toBe(2); + }); + + it('calls delete on all paths even if the first rejects', async () => { + const expectedPaths = ['one/index.html', 'two/index.html']; + MockClient.delete.mockRejectedValueOnce(new Error()); + MockClient.delete.mockResolvedValueOnce(undefined); + + await expect( + CacheUnderTest.invalidateMultiple(expectedPaths), + ).rejects.toThrowError(CacheInvalidationError); + expect(MockClient.delete).toHaveBeenCalledTimes(2); + }); + + it('rejects with invalidations error response', async () => { + const expectedPaths = ['one/index.html', 'two/index.html']; + MockClient.delete.mockResolvedValueOnce(undefined); + MockClient.delete.mockRejectedValueOnce(new Error()); + + await expect( + CacheUnderTest.invalidateMultiple.bind(CacheUnderTest, expectedPaths), + ).rejects.toThrow( + expect.objectContaining({ rejections: expect.arrayContaining([]) }), + ); + }); + }); +}); From 74ccd66e26ab36612173ca4a1bb3914bfcdfca41 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 24 Jul 2021 22:27:18 +0200 Subject: [PATCH 15/50] Add tests for cache middleware. Do not cache non-200 responses. Signed-off-by: Eric Peterson --- .../src/cache/cacheMiddleware.test.ts | 142 ++++++++++++++++++ .../src/cache/cacheMiddleware.ts | 6 +- 2 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts diff --git a/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts b/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts new file mode 100644 index 0000000000..5c591bdda4 --- /dev/null +++ b/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts @@ -0,0 +1,142 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import express from 'express'; +import request from 'supertest'; +import { createCacheMiddleware, TechDocsCache } from '.'; + +/** + * Mocks cached HTTP response. + */ +const getMockHttpResponseFor = (content: string): Buffer => { + return Buffer.concat([ + Buffer.from(`HTTP/1.1 200 OK +Content-Type: text/plain; charset=utf-8 +Accept-Ranges: bytes +Cache-Control: public, max-age=0 +Last-Modified: Sat, 1 Jul 2021 12:00:00 GMT +Date: Sat, 1 Jul 2021 12:00:00 GMT +Connection: close +Content-Length: ${content.length}\n\n`), + Buffer.from(content), + ]); +}; + +/** + * Wait for the socket to close. Works because, above, we set connection: close + */ +const waitForSocketClose = () => new Promise(resolve => setTimeout(resolve, 0)); + +describe('createCacheMiddleware', () => { + let cache: jest.Mocked; + let app: express.Express; + + beforeEach(async () => { + cache = ({ + get: jest.fn().mockResolvedValue(undefined), + set: jest.fn().mockResolvedValue(undefined), + invalidate: jest.fn().mockResolvedValue(undefined), + invalidateMultiple: jest.fn().mockResolvedValue(undefined), + } as unknown) as jest.Mocked; + const router = await createCacheMiddleware({ + logger: getVoidLogger(), + cache, + }); + app = express().use(router); + app.use((req, res, next) => { + // By default, send cacheable content. + if (req.path !== '/api/static/docs/error.png') { + res.send('default-response'); + } else { + next(new Error()); + } + }); + }); + + describe('invalidate', () => { + it('responds with 400 when no objects are provided', async () => { + const response = await request(app).post('/cache/invalidate'); + expect(response.status).toBe(400); + }); + + it('responds with 500 if invalidation throws', async () => { + cache.invalidateMultiple.mockRejectedValueOnce(new Error()); + const response = await request(app) + .post('/cache/invalidate') + .set('Content-Type', 'application/json') + .send({ + objects: ['one/index.html', 'two/index.html'], + }); + + expect(response.status).toBe(500); + }); + + it('responds with 204 if invalidation succeeds', async () => { + const expectedObjects = ['one/index.html', 'two/index.html']; + cache.invalidateMultiple.mockResolvedValue([]); + await request(app) + .post('/cache/invalidate') + .set('Content-Type', 'application/json') + .send({ + objects: expectedObjects, + }) + .expect(204); + + expect(cache.invalidateMultiple).toHaveBeenCalledWith(expectedObjects); + }); + }); + + describe('middleware', () => { + it('does not apply to non-static/docs paths', async () => { + await request(app) + .get('/api/static/not-docs') + .expect(200, 'default-response'); + + expect(cache.set).not.toHaveBeenCalled(); + }); + + it('responds with cached response', async () => { + cache.get.mockResolvedValueOnce(getMockHttpResponseFor('xyz')); + + await request(app).get('/api/static/docs/foo.html').expect(200, 'xyz'); + + await waitForSocketClose(); + expect(cache.set).not.toHaveBeenCalled(); + }); + + it('sets cache when content is cacheable', async () => { + const expectedPath = 'default/api/xyz/index.html'; + await request(app) + .get(`/api/static/docs/${expectedPath}`) + .expect(200, 'default-response'); + + await waitForSocketClose(); + expect(cache.set).toHaveBeenCalled(); + + const [actualPath, actualBuffer] = (cache.set as jest.Mock).mock.calls[0]; + expect(actualPath).toBe(expectedPath); + expect(actualBuffer.toString()).toContain('default-response'); + }); + + it('does not set cache on error', async () => { + await request(app).get('/api/static/docs/error.png').expect(500); + + await waitForSocketClose(); + expect(cache.set).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts index 0c06052ba4..c25d002181 100644 --- a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts +++ b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts @@ -114,8 +114,10 @@ export const createCacheMiddleware = ({ // When a socket is closed, if there were no errors and the data written // over the socket should be cached, cache it! socket.on('close', hadError => { - if (writeToCache && !hadError) { - cache.set(reqPath, Buffer.concat(chunks)); + const content = Buffer.concat(chunks); + const head = content.toString('utf8', 0, 12); + if (writeToCache && !hadError && head === 'HTTP/1.1 200') { + cache.set(reqPath, content); } }); From d011fd61ea16aacfcbd32716adc093117ea1c03d Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 24 Jul 2021 23:07:33 +0200 Subject: [PATCH 16/50] Ensure DocsSynchronizer has access to cache client. Signed-off-by: Eric Peterson --- .../techdocs-backend/src/DocsBuilder/builder.ts | 3 +++ .../src/service/DocsSynchronizer.test.ts | 8 ++++++++ .../src/service/DocsSynchronizer.ts | 6 ++++++ plugins/techdocs-backend/src/service/router.ts | 16 +++++++++------- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/plugins/techdocs-backend/src/DocsBuilder/builder.ts b/plugins/techdocs-backend/src/DocsBuilder/builder.ts index 1d725bd69e..66eb52c1aa 100644 --- a/plugins/techdocs-backend/src/DocsBuilder/builder.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/builder.ts @@ -222,6 +222,9 @@ export class DocsBuilder { // Invalidate the cache for any published objects. if (this.cache && published && published?.objects?.length) { + this.logger.debug( + `Invalidating ${published.objects.length} cache objects`, + ); await this.cache.invalidateMultiple(published.objects); } diff --git a/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts b/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts index ac60f3da3a..c579f62fb8 100644 --- a/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts +++ b/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts @@ -25,6 +25,7 @@ import { PreparerBuilder, PublisherBase, } from '@backstage/techdocs-common'; +import { TechDocsCache } from '../cache'; import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder'; import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer'; @@ -52,6 +53,12 @@ describe('DocsSynchronizer', () => { getBaseUrl: jest.fn(), getExternalBaseUrl: jest.fn(), }; + const cache: jest.Mocked = ({ + get: jest.fn(), + set: jest.fn(), + invalidate: jest.fn(), + invalidateMultiple: jest.fn(), + } as unknown) as jest.Mocked; let docsSynchronizer: DocsSynchronizer; const mockResponseHandler: jest.Mocked = { @@ -71,6 +78,7 @@ describe('DocsSynchronizer', () => { config: new ConfigReader({}), logger: getVoidLogger(), scmIntegrations: ScmIntegrations.fromConfig(new ConfigReader({})), + cache, }); }); diff --git a/plugins/techdocs-backend/src/service/DocsSynchronizer.ts b/plugins/techdocs-backend/src/service/DocsSynchronizer.ts index 6f977dcb47..d76420d936 100644 --- a/plugins/techdocs-backend/src/service/DocsSynchronizer.ts +++ b/plugins/techdocs-backend/src/service/DocsSynchronizer.ts @@ -25,6 +25,7 @@ import { } from '@backstage/techdocs-common'; import { PassThrough } from 'stream'; import * as winston from 'winston'; +import { TechDocsCache } from '../cache'; import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder'; export type DocsSynchronizerSyncOpts = { @@ -38,22 +39,26 @@ export class DocsSynchronizer { private readonly logger: winston.Logger; private readonly config: Config; private readonly scmIntegrations: ScmIntegrationRegistry; + private readonly cache: TechDocsCache | undefined; constructor({ publisher, logger, config, scmIntegrations, + cache, }: { publisher: PublisherBase; logger: winston.Logger; config: Config; scmIntegrations: ScmIntegrationRegistry; + cache: TechDocsCache | undefined; }) { this.config = config; this.logger = logger; this.publisher = publisher; this.scmIntegrations = scmIntegrations; + this.cache = cache; } async doSync({ @@ -104,6 +109,7 @@ export class DocsSynchronizer { config: this.config, scmIntegrations: this.scmIntegrations, logStream, + cache: this.cache, }); const updated = await docsBuilder.build(); diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 07c6887ef7..96c5aaf172 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -85,13 +85,6 @@ export async function createRouter( const router = Router(); const { publisher, config, logger, discovery } = options; const catalogClient = new CatalogClient({ discoveryApi: discovery }); - const scmIntegrations = ScmIntegrations.fromConfig(config); - const docsSynchronizer = new DocsSynchronizer({ - publisher, - logger, - config, - scmIntegrations, - }); // Set up a cache client if configured. let cache: TechDocsCache | undefined; @@ -101,6 +94,15 @@ export async function createRouter( cache = new TechDocsCache({ cache: cacheClient, logger }); } + const scmIntegrations = ScmIntegrations.fromConfig(config); + const docsSynchronizer = new DocsSynchronizer({ + publisher, + logger, + config, + scmIntegrations, + cache, + }); + router.get('/metadata/techdocs/:namespace/:kind/:name', async (req, res) => { const { kind, namespace, name } = req.params; const entityName = { kind, namespace, name }; From e99606d3e66c16a1db6ba299f7908c6fc19efb38 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Nov 2021 10:49:50 +0100 Subject: [PATCH 17/50] Add all generated files to techdocs_metadata.json Signed-off-by: Eric Peterson --- .../src/stages/generate/helpers.test.ts | 28 +++++++++++++------ .../src/stages/generate/helpers.ts | 24 ++++++++++++++-- .../src/stages/generate/techdocs.ts | 6 ++-- .../src/stages/publish/types.ts | 2 ++ 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/packages/techdocs-common/src/stages/generate/helpers.test.ts b/packages/techdocs-common/src/stages/generate/helpers.test.ts index 9e01c3e2ad..9323ab4f18 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.test.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.test.ts @@ -23,7 +23,7 @@ import os from 'os'; import path, { resolve as resolvePath } from 'path'; import { ParsedLocationAnnotation } from '../../helpers'; import { - addBuildTimestampMetadata, + createOrUpdateMetadata, getGeneratorKey, getMkdocsYml, getRepoUrlFromLocationAnnotation, @@ -369,13 +369,15 @@ describe('helpers', () => { }); describe('addBuildTimestampMetadata', () => { + const mockFiles = { + 'invalid_techdocs_metadata.json': 'dsds', + 'techdocs_metadata.json': '{"site_name": "Tech Docs"}', + }; + beforeEach(() => { mockFs.restore(); mockFs({ - [rootDir]: { - 'invalid_techdocs_metadata.json': 'dsds', - 'techdocs_metadata.json': '{"site_name": "Tech Docs"}', - }, + [rootDir]: mockFiles, }); }); @@ -385,7 +387,7 @@ describe('helpers', () => { it('should create the file if it does not exist', async () => { const filePath = path.join(rootDir, 'wrong_techdocs_metadata.json'); - await addBuildTimestampMetadata(filePath, mockLogger); + await createOrUpdateMetadata(filePath, mockLogger); // Check if the file exists await expect( @@ -397,18 +399,28 @@ describe('helpers', () => { const filePath = path.join(rootDir, 'invalid_techdocs_metadata.json'); await expect( - addBuildTimestampMetadata(filePath, mockLogger), + createOrUpdateMetadata(filePath, mockLogger), ).rejects.toThrowError('Unexpected token d in JSON at position 0'); }); it('should add build timestamp to the metadata json', async () => { const filePath = path.join(rootDir, 'techdocs_metadata.json'); - await addBuildTimestampMetadata(filePath, mockLogger); + await createOrUpdateMetadata(filePath, mockLogger); const json = await fs.readJson(filePath); expect(json.build_timestamp).toBeLessThanOrEqual(Date.now()); }); + + it('should add list of files to the metadata json', async () => { + const filePath = path.join(rootDir, 'techdocs_metadata.json'); + + await createOrUpdateMetadata(filePath, mockLogger); + + const json = await fs.readJson(filePath); + expect(json.files[0]).toEqual(Object.keys(mockFiles)[0]); + expect(json.files[1]).toEqual(Object.keys(mockFiles)[1]); + }); }); describe('storeEtagMetadata', () => { diff --git a/packages/techdocs-common/src/stages/generate/helpers.ts b/packages/techdocs-common/src/stages/generate/helpers.ts index f77b557076..f96d1209b6 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.ts @@ -27,6 +27,7 @@ import { PassThrough, Writable } from 'stream'; import { Logger } from 'winston'; import { ParsedLocationAnnotation } from '../../helpers'; import { SupportedGeneratorKey } from './types'; +import { getFileTreeRecursively } from '../publish/helpers'; // TODO: Implement proper support for more generators. export function getGeneratorKey(entity: Entity): SupportedGeneratorKey { @@ -345,14 +346,20 @@ export const patchIndexPreBuild = async ({ }; /** - * Update the techdocs_metadata.json to add a new build timestamp metadata. Create the .json file if it doesn't exist. + * Create or update the techdocs_metadata.json. Values initialized/updated are: + * - The build_timestamp (now) + * - The list of files generated * * @param {string} techdocsMetadataPath File path to techdocs_metadata.json */ -export const addBuildTimestampMetadata = async ( +export const createOrUpdateMetadata = async ( techdocsMetadataPath: string, logger: Logger, ): Promise => { + const techdocsMetadataDir = techdocsMetadataPath + .split(path.sep) + .slice(0, -1) + .join(path.sep); // check if file exists, create if it does not. try { await fs.access(techdocsMetadataPath, fs.constants.F_OK); @@ -372,6 +379,19 @@ export const addBuildTimestampMetadata = async ( } json.build_timestamp = Date.now(); + + // Get and write generated files to the metadata JSON. Each file string is in + // a form appropriate for invalidating the associated object from cache. + try { + json.files = (await getFileTreeRecursively(techdocsMetadataDir)).map(file => + file.replace(`${techdocsMetadataDir}/`, ''), + ); + } catch (err) { + assertError(err); + json.files = []; + logger.warn(`Unable to add files list to metadata: ${err.message}`); + } + await fs.writeJson(techdocsMetadataPath, json); return; }; diff --git a/packages/techdocs-common/src/stages/generate/techdocs.ts b/packages/techdocs-common/src/stages/generate/techdocs.ts index 728cab5a81..8681736815 100644 --- a/packages/techdocs-common/src/stages/generate/techdocs.ts +++ b/packages/techdocs-common/src/stages/generate/techdocs.ts @@ -23,7 +23,7 @@ import { ScmIntegrations, } from '@backstage/integration'; import { - addBuildTimestampMetadata, + createOrUpdateMetadata, getMkdocsYml, patchIndexPreBuild, patchMkdocsYmlPreBuild, @@ -164,9 +164,9 @@ export class TechdocsGenerator implements GeneratorBase { * Post Generate steps */ - // Add build timestamp to techdocs_metadata.json + // Add build timestamp and files to techdocs_metadata.json // Creates techdocs_metadata.json if file does not exist. - await addBuildTimestampMetadata( + await createOrUpdateMetadata( path.join(outputDir, 'techdocs_metadata.json'), childLogger, ); diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index ab497067b3..1fb301340b 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -65,6 +65,8 @@ export type TechDocsMetadata = { site_name: string; site_description: string; etag: string; + build_timestamp: number; + files?: string[]; }; export type MigrateRequest = { From a3909d2c2fc67d12451cc8f3c800ac97fac5c695 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 7 Aug 2021 20:33:49 +0200 Subject: [PATCH 18/50] Invalidate stale cache entries on read for external builder config when cache is enabled. Signed-off-by: Eric Peterson --- .../src/service/DocsSynchronizer.test.ts | 91 +++++++++++++++++++ .../src/service/DocsSynchronizer.ts | 74 ++++++++++++++- .../techdocs-backend/src/service/router.ts | 11 +++ 3 files changed, 174 insertions(+), 2 deletions(-) diff --git a/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts b/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts index c579f62fb8..4089b08ef4 100644 --- a/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts +++ b/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts @@ -31,6 +31,19 @@ import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer'; jest.mock('../DocsBuilder'); +jest.mock('cross-fetch', () => ({ + __esModule: true, + default: async () => { + return { + json: async () => { + return { + build_timestamp: 123, + }; + }, + }; + }, +})); + const MockedDocsBuilder = DocsBuilder as jest.MockedClass; describe('DocsSynchronizer', () => { @@ -201,4 +214,82 @@ describe('DocsSynchronizer', () => { expect(mockResponseHandler.error).toBeCalledWith(error); }); }); + + describe('doCacheSync', () => { + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + uid: '0', + name: 'test', + namespace: 'default', + }, + }; + + it('should not check metadata too often', async () => { + (shouldCheckForUpdate as jest.Mock).mockReturnValue(false); + + await docsSynchronizer.doCacheSync({ + responseHandler: mockResponseHandler, + discovery, + token: undefined, + entity, + }); + + expect(mockResponseHandler.finish).toBeCalledWith({ updated: false }); + expect(shouldCheckForUpdate).toBeCalledTimes(1); + }); + + it('should do nothing if source/cached metadata matches', async () => { + (shouldCheckForUpdate as jest.Mock).mockReturnValue(true); + (publisher.fetchTechDocsMetadata as jest.Mock).mockResolvedValue({ + build_timestamp: 123, + }); + + await docsSynchronizer.doCacheSync({ + responseHandler: mockResponseHandler, + discovery, + token: undefined, + entity, + }); + + expect(mockResponseHandler.finish).toBeCalledWith({ updated: false }); + }); + + it('should invalidate expected files when source/cached metadata differ', async () => { + (shouldCheckForUpdate as jest.Mock).mockReturnValue(true); + (publisher.fetchTechDocsMetadata as jest.Mock).mockResolvedValue({ + build_timestamp: 456, + files: ['index.html'], + }); + + await docsSynchronizer.doCacheSync({ + responseHandler: mockResponseHandler, + discovery, + token: undefined, + entity, + }); + + expect(mockResponseHandler.finish).toBeCalledWith({ updated: true }); + expect(cache.invalidateMultiple).toHaveBeenCalledWith([ + 'default/Component/test/index.html', + ]); + }); + + it('should gracefully handle errors', async () => { + (shouldCheckForUpdate as jest.Mock).mockReturnValue(true); + (publisher.fetchTechDocsMetadata as jest.Mock).mockRejectedValue( + new Error(), + ); + + await docsSynchronizer.doCacheSync({ + responseHandler: mockResponseHandler, + discovery, + token: undefined, + entity, + }); + + expect(mockResponseHandler.finish).toBeCalledWith({ updated: false }); + }); + }); }); diff --git a/plugins/techdocs-backend/src/service/DocsSynchronizer.ts b/plugins/techdocs-backend/src/service/DocsSynchronizer.ts index d76420d936..c6d198c41d 100644 --- a/plugins/techdocs-backend/src/service/DocsSynchronizer.ts +++ b/plugins/techdocs-backend/src/service/DocsSynchronizer.ts @@ -14,7 +14,8 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, NotFoundError } from '@backstage/errors'; import { ScmIntegrationRegistry } from '@backstage/integration'; @@ -23,10 +24,15 @@ import { PreparerBuilder, PublisherBase, } from '@backstage/techdocs-common'; +import fetch from 'cross-fetch'; import { PassThrough } from 'stream'; import * as winston from 'winston'; import { TechDocsCache } from '../cache'; -import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder'; +import { + BuildMetadataStorage, + DocsBuilder, + shouldCheckForUpdate, +} from '../DocsBuilder'; export type DocsSynchronizerSyncOpts = { log: (message: string) => void; @@ -151,4 +157,68 @@ export class DocsSynchronizer { finish({ updated: true }); } + + async doCacheSync({ + responseHandler: { finish }, + discovery, + token, + entity, + }: { + responseHandler: DocsSynchronizerSyncOpts; + discovery: PluginEndpointDiscovery; + token: string | undefined; + entity: Entity; + }) { + // Check if the last update check was too recent. + if (!shouldCheckForUpdate(entity.metadata.uid!) || !this.cache) { + finish({ updated: false }); + return; + } + + // Fetch techdocs_metadata.json from the publisher and from cache. + const baseUrl = await discovery.getBaseUrl('techdocs'); + const namespace = entity.metadata?.namespace || ENTITY_DEFAULT_NAMESPACE; + const kind = entity.kind; + const name = entity.metadata.name; + const entityTripletPath = `${namespace}/${kind}/${name}`; + try { + const [sourceMetadata, cachedMetadata] = await Promise.all([ + this.publisher.fetchTechDocsMetadata({ namespace, kind, name }), + fetch( + `${baseUrl}/static/docs/${entityTripletPath}/techdocs_metadata.json`, + { + headers: token ? { Authorization: `Bearer ${token}` } : {}, + }, + ).then( + f => + f.json().catch(() => undefined) as ReturnType< + PublisherBase['fetchTechDocsMetadata'] + >, + ), + ]); + + // If build timestamps differ, merge their files[] lists and invalidate all objects. + if (sourceMetadata.build_timestamp !== cachedMetadata.build_timestamp) { + const files = [ + ...new Set([ + ...(sourceMetadata.files || []), + ...(cachedMetadata.files || []), + ]), + ].map(f => `${entityTripletPath}/${f}`); + await this.cache.invalidateMultiple(files); + finish({ updated: true }); + } else { + finish({ updated: false }); + } + } catch (e) { + // In case of error, log and allow the user to go about their business. + this.logger.error( + `Error syncing cache for ${entityTripletPath}: ${e.message}`, + ); + finish({ updated: false }); + } finally { + // Update the last check time for the entity + new BuildMetadataStorage(entity.metadata.uid!).setLastUpdated(); + } + } } diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 96c5aaf172..ba3fc6ea87 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -190,6 +190,17 @@ export async function createRouter( // If set to 'external', it will assume that an external process (e.g. CI/CD pipeline // of the repository) is responsible for building and publishing documentation to the storage provider if (config.getString('techdocs.builder') !== 'local') { + // However, if caching is enabled, take the opportunity to check and + // invalidate stale cache entries. + if (cache) { + await docsSynchronizer.doCacheSync({ + responseHandler, + discovery, + token, + entity, + }); + return; + } responseHandler.finish({ updated: false }); return; } From 8cf4684a4eae472abb9d060958c56a9b5fe02ac3 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 7 Aug 2021 20:34:10 +0200 Subject: [PATCH 19/50] Remove cache invalidation endpoint from middleware. Signed-off-by: Eric Peterson --- .../src/cache/cacheMiddleware.test.ts | 33 ------------- .../src/cache/cacheMiddleware.ts | 49 +------------------ 2 files changed, 1 insertion(+), 81 deletions(-) diff --git a/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts b/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts index 5c591bdda4..4bd680e8c7 100644 --- a/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts +++ b/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts @@ -67,39 +67,6 @@ describe('createCacheMiddleware', () => { }); }); - describe('invalidate', () => { - it('responds with 400 when no objects are provided', async () => { - const response = await request(app).post('/cache/invalidate'); - expect(response.status).toBe(400); - }); - - it('responds with 500 if invalidation throws', async () => { - cache.invalidateMultiple.mockRejectedValueOnce(new Error()); - const response = await request(app) - .post('/cache/invalidate') - .set('Content-Type', 'application/json') - .send({ - objects: ['one/index.html', 'two/index.html'], - }); - - expect(response.status).toBe(500); - }); - - it('responds with 204 if invalidation succeeds', async () => { - const expectedObjects = ['one/index.html', 'two/index.html']; - cache.invalidateMultiple.mockResolvedValue([]); - await request(app) - .post('/cache/invalidate') - .set('Content-Type', 'application/json') - .send({ - objects: expectedObjects, - }) - .expect(204); - - expect(cache.invalidateMultiple).toHaveBeenCalledWith(expectedObjects); - }); - }); - describe('middleware', () => { it('does not apply to non-static/docs paths', async () => { await request(app) diff --git a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts index c25d002181..66471b0189 100644 --- a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts +++ b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts @@ -13,22 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Router, Request, json } from 'express'; +import { Router } from 'express'; import router from 'express-promise-router'; import { Logger } from 'winston'; import { TechDocsCache } from '.'; -import { CacheInvalidationError } from './TechDocsCache'; - -type CacheClearRequestParams = { - objects: string[]; -}; - -type CacheClearRequest = Request< - any, - unknown, - CacheClearRequestParams, - unknown ->; type CacheMiddlewareOptions = { cache: TechDocsCache; @@ -39,44 +27,9 @@ type ErrorCallback = (err?: Error) => void; export const createCacheMiddleware = ({ cache, - logger, }: CacheMiddlewareOptions): Router => { const cacheMiddleware = router(); - // And endpoint for handling cache invalidation external to the Backstage - // Backend (e.g. from the TechDocs CLI). - cacheMiddleware.use(json()); - cacheMiddleware.post( - '/cache/invalidate', - async (req: CacheClearRequest, res) => { - if (req.body?.objects?.length) { - logger.debug( - `Clearing ${req.body.objects.length} cache entries: (eg: ${req.body.objects[0]})`, - ); - - try { - const invalidated = await cache.invalidateMultiple(req.body.objects); - logger.debug( - `Successfully invalidated ${invalidated.length} cache entries`, - ); - res.status(204).send(); - } catch (e) { - if (e instanceof CacheInvalidationError) { - const uniqueReasons = [ - ...new Set(e.rejections.map(r => r.reason.message)), - ].join(', '); - logger.warn( - `Problem invalidating ${e.rejections.length} entries: ${uniqueReasons}`, - ); - } - res.status(500).send(); - } - } else { - res.status(400).send(); - } - }, - ); - // Middleware that, through socket monkey patching, captures responses as // they're sent over /static/docs/* and caches them. Subsequent requests are // loaded from cache. Cache key is the object's path (after `/static/docs/`). From 6e618bef06ca2c4150540b6f75914b00c38ab9a3 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Nov 2021 11:00:33 +0100 Subject: [PATCH 20/50] Fix up types/tests/report. Signed-off-by: Eric Peterson --- packages/techdocs-common/api-report.md | 2 ++ packages/techdocs-common/src/stages/publish/awsS3.test.ts | 1 + .../src/stages/publish/azureBlobStorage.test.ts | 1 + .../src/stages/publish/googleStorage.test.ts | 1 + .../src/stages/publish/openStackSwift.test.ts | 6 ++++-- 5 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/techdocs-common/api-report.md b/packages/techdocs-common/api-report.md index 61371525e5..b2f9e40d0a 100644 --- a/packages/techdocs-common/api-report.md +++ b/packages/techdocs-common/api-report.md @@ -287,6 +287,8 @@ export type TechDocsMetadata = { site_name: string; site_description: string; etag: string; + build_timestamp: number; + files?: string[]; }; // Warning: (ae-missing-release-tag) "transformDirLocation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index abae5fda44..37c2e06283 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -95,6 +95,7 @@ describe('AwsS3Publish', () => { site_name: 'backstage', site_description: 'site_content', etag: 'etag', + build_timestamp: 612741599, }; const directory = getEntityRootDir(entity); diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 0e261f7f86..0ac00988a8 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -89,6 +89,7 @@ describe('AzureBlobStoragePublish', () => { site_name: 'backstage', site_description: 'site_content', etag: 'etag', + build_timestamp: 612741599, }; const directory = getEntityRootDir(entity); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index ea97d81269..61c152c90d 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -88,6 +88,7 @@ describe('GoogleGCSPublish', () => { site_name: 'backstage', site_description: 'site_content', etag: 'etag', + build_timestamp: 612741599, }; const directory = getEntityRootDir(entity); diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts index b8ad43aac3..ef06e26cfa 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts @@ -247,7 +247,7 @@ describe('OpenStackSwiftPublish', () => { mockFs({ [entityRootDir]: { 'techdocs_metadata.json': - '{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}', + '{"site_name": "backstage", "site_description": "site_content", "etag": "etag", "build_timestamp": 612741599}', }, }); @@ -255,6 +255,7 @@ describe('OpenStackSwiftPublish', () => { site_name: 'backstage', site_description: 'site_content', etag: 'etag', + build_timestamp: 612741599, }; expect( await publisher.fetchTechDocsMetadata(entityNameMock), @@ -269,7 +270,7 @@ describe('OpenStackSwiftPublish', () => { mockFs({ [entityRootDir]: { - 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, + 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag', 'build_timestamp': 612741599}`, }, }); @@ -277,6 +278,7 @@ describe('OpenStackSwiftPublish', () => { site_name: 'backstage', site_description: 'site_content', etag: 'etag', + build_timestamp: 612741599, }; expect( await publisher.fetchTechDocsMetadata(entityNameMock), From 12157e8f5b2163931fc9eda45e34f1732aba9314 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Nov 2021 12:36:03 +0100 Subject: [PATCH 21/50] Fix up tests after rebase. Signed-off-by: Eric Peterson --- app-config.yaml | 3 +- .../src/stages/publish/awsS3.test.ts | 30 +++++++++++++++---- .../stages/publish/azureBlobStorage.test.ts | 14 ++++++--- .../src/stages/publish/googleStorage.test.ts | 30 +++++++++++++++---- 4 files changed, 59 insertions(+), 18 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index bde037dfe3..913a72dbab 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -112,8 +112,7 @@ techdocs: # pullImage: true # or false to disable automatic pulling of image (e.g. if custom docker login is required) publisher: type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureBlobStorage' or 'openStackSwift'. Read documentation for using alternatives. - cache: - ttl: 3600000 # 1 hour cache for demonstration purposes. You may wish to set this higher in production. + sentry: organization: my-company diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index 37c2e06283..591131907b 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -150,21 +150,39 @@ describe('AwsS3Publish', () => { describe('publish', () => { it('should publish a directory', async () => { const publisher = createPublisherFromConfig(); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'default/component/backstage/404.html', + `default/component/backstage/index.html`, + `default/component/backstage/assets/main.css`, + ]), + }); }); it('should publish a directory as well when legacy casing is used', async () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'default/Component/backstage/404.html', + `default/Component/backstage/index.html`, + `default/Component/backstage/assets/main.css`, + ]), + }); }); it('should publish a directory when root path is specified', async () => { const publisher = createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', }); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'backstage-data/techdocs/default/component/backstage/404.html', + `backstage-data/techdocs/default/component/backstage/index.html`, + `backstage-data/techdocs/default/component/backstage/assets/main.css`, + ]), + }); }); it('should publish a directory when root path is specified and legacy casing is used', async () => { @@ -174,9 +192,9 @@ describe('AwsS3Publish', () => { }); expect(await publisher.publish({ entity, directory })).toMatchObject({ objects: expect.arrayContaining([ - 'test-namespace/TestKind/test-component-name/404.html', - `test-namespace/TestKind/test-component-name/index.html`, - `test-namespace/TestKind/test-component-name/assets/main.css`, + 'backstage-data/techdocs/default/Component/backstage/404.html', + `backstage-data/techdocs/default/Component/backstage/index.html`, + `backstage-data/techdocs/default/Component/backstage/assets/main.css`, ]), }); }); diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 0ac00988a8..e6fd45eed7 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -155,7 +155,13 @@ describe('AzureBlobStoragePublish', () => { describe('publish', () => { it('should publish a directory', async () => { const publisher = createPublisherFromConfig(); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'default/component/backstage/404.html', + `default/component/backstage/index.html`, + `default/component/backstage/assets/main.css`, + ]), + }); }); it('should publish a directory as well when legacy casing is used', async () => { @@ -164,9 +170,9 @@ describe('AzureBlobStoragePublish', () => { }); expect(await publisher.publish({ entity, directory })).toMatchObject({ objects: expect.arrayContaining([ - 'test-namespace/TestKind/test-component-name/404.html', - `test-namespace/TestKind/test-component-name/index.html`, - `test-namespace/TestKind/test-component-name/assets/main.css`, + 'default/Component/backstage/404.html', + `default/Component/backstage/index.html`, + `default/Component/backstage/assets/main.css`, ]), }); }); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index 61c152c90d..ed35f2e445 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -143,21 +143,39 @@ describe('GoogleGCSPublish', () => { describe('publish', () => { it('should publish a directory', async () => { const publisher = createPublisherFromConfig(); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'default/component/backstage/404.html', + `default/component/backstage/index.html`, + `default/component/backstage/assets/main.css`, + ]), + }); }); it('should publish a directory as well when legacy casing is used', async () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'default/Component/backstage/404.html', + `default/Component/backstage/index.html`, + `default/Component/backstage/assets/main.css`, + ]), + }); }); it('should publish a directory when root path is specified', async () => { const publisher = createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', }); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'backstage-data/techdocs/default/component/backstage/404.html', + `backstage-data/techdocs/default/component/backstage/index.html`, + `backstage-data/techdocs/default/component/backstage/assets/main.css`, + ]), + }); }); it('should publish a directory when root path is specified and legacy casing is used', async () => { @@ -167,9 +185,9 @@ describe('GoogleGCSPublish', () => { }); expect(await publisher.publish({ entity, directory })).toMatchObject({ objects: expect.arrayContaining([ - 'test-namespace/TestKind/test-component-name/404.html', - `test-namespace/TestKind/test-component-name/index.html`, - `test-namespace/TestKind/test-component-name/assets/main.css`, + 'backstage-data/techdocs/default/Component/backstage/404.html', + `backstage-data/techdocs/default/Component/backstage/index.html`, + `backstage-data/techdocs/default/Component/backstage/assets/main.css`, ]), }); }); From da0867f9526f7361848dd4b9ab1d57a187732da1 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Nov 2021 12:58:17 +0100 Subject: [PATCH 22/50] Account for legacy path casing in cache invalidation. Signed-off-by: Eric Peterson --- .../src/cache/cacheMiddleware.test.ts | 4 +-- .../src/service/DocsSynchronizer.test.ts | 34 +++++++++++++++++-- .../src/service/DocsSynchronizer.ts | 10 +++++- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts b/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts index 4bd680e8c7..91e4b84222 100644 --- a/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts +++ b/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts @@ -46,12 +46,12 @@ describe('createCacheMiddleware', () => { let app: express.Express; beforeEach(async () => { - cache = ({ + cache = { get: jest.fn().mockResolvedValue(undefined), set: jest.fn().mockResolvedValue(undefined), invalidate: jest.fn().mockResolvedValue(undefined), invalidateMultiple: jest.fn().mockResolvedValue(undefined), - } as unknown) as jest.Mocked; + } as unknown as jest.Mocked; const router = await createCacheMiddleware({ logger: getVoidLogger(), cache, diff --git a/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts b/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts index 4089b08ef4..1eb8c8f56f 100644 --- a/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts +++ b/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts @@ -66,12 +66,12 @@ describe('DocsSynchronizer', () => { getBaseUrl: jest.fn(), getExternalBaseUrl: jest.fn(), }; - const cache: jest.Mocked = ({ + const cache: jest.Mocked = { get: jest.fn(), set: jest.fn(), invalidate: jest.fn(), invalidateMultiple: jest.fn(), - } as unknown) as jest.Mocked; + } as unknown as jest.Mocked; let docsSynchronizer: DocsSynchronizer; const mockResponseHandler: jest.Mocked = { @@ -270,6 +270,36 @@ describe('DocsSynchronizer', () => { entity, }); + expect(mockResponseHandler.finish).toBeCalledWith({ updated: true }); + expect(cache.invalidateMultiple).toHaveBeenCalledWith([ + 'default/component/test/index.html', + ]); + }); + + it('should invalidate expected files when source/cached metadata differ with legacy casing', async () => { + (shouldCheckForUpdate as jest.Mock).mockReturnValue(true); + (publisher.fetchTechDocsMetadata as jest.Mock).mockResolvedValue({ + build_timestamp: 456, + files: ['index.html'], + }); + + const docsSynchronizerWithLegacy = new DocsSynchronizer({ + publisher, + config: new ConfigReader({ + techdocs: { legacyUseCaseSensitiveTripletPaths: true }, + }), + logger: getVoidLogger(), + scmIntegrations: ScmIntegrations.fromConfig(new ConfigReader({})), + cache, + }); + + await docsSynchronizerWithLegacy.doCacheSync({ + responseHandler: mockResponseHandler, + discovery, + token: undefined, + entity, + }); + expect(mockResponseHandler.finish).toBeCalledWith({ updated: true }); expect(cache.invalidateMultiple).toHaveBeenCalledWith([ 'default/Component/test/index.html', diff --git a/plugins/techdocs-backend/src/service/DocsSynchronizer.ts b/plugins/techdocs-backend/src/service/DocsSynchronizer.ts index c6d198c41d..c7d5bf109a 100644 --- a/plugins/techdocs-backend/src/service/DocsSynchronizer.ts +++ b/plugins/techdocs-backend/src/service/DocsSynchronizer.ts @@ -180,7 +180,14 @@ export class DocsSynchronizer { const namespace = entity.metadata?.namespace || ENTITY_DEFAULT_NAMESPACE; const kind = entity.kind; const name = entity.metadata.name; - const entityTripletPath = `${namespace}/${kind}/${name}`; + const legacyPathCasing = + this.config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + const tripletPath = `${namespace}/${kind}/${name}`; + const entityTripletPath = `${ + legacyPathCasing ? tripletPath : tripletPath.toLocaleLowerCase() + }`; try { const [sourceMetadata, cachedMetadata] = await Promise.all([ this.publisher.fetchTechDocsMetadata({ namespace, kind, name }), @@ -211,6 +218,7 @@ export class DocsSynchronizer { finish({ updated: false }); } } catch (e) { + assertError(e); // In case of error, log and allow the user to go about their business. this.logger.error( `Error syncing cache for ${entityTripletPath}: ${e.message}`, From ef04b09f75058cce47e0b35eebef533e82772b2d Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Nov 2021 15:01:37 +0100 Subject: [PATCH 23/50] Make cache readTimeout configurable. Signed-off-by: Eric Peterson --- docs/features/techdocs/configuration.md | 6 +++++ plugins/techdocs-backend/config.d.ts | 9 +++++++ .../src/cache/TechDocsCache.test.ts | 27 +++++++++++++++++-- .../src/cache/TechDocsCache.ts | 26 ++++++++++++++++-- .../techdocs-backend/src/service/router.ts | 2 +- 5 files changed, 65 insertions(+), 5 deletions(-) diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 3234cdc034..983a8dd3e7 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -144,6 +144,12 @@ techdocs: # to storage using the techdocs-cli, allowing long TTLs (e.g. 1 month/year) ttl: 3600000 + # (Optional) The time (in milliseconds) that the TechDocs backend will wait + # for a cache service to respond before continuing on as though the cached + # object was not found (e.g. when the cache sercice is unavailable). The + # default value is 1000 + readTimeout: 500 + # (Optional and Legacy) TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc. # You don't have to specify this anymore. diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index 1767120a35..8d531e0536 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -241,6 +241,15 @@ export interface Config { * createRouter method in your backend. */ ttl: number; + + /** + * The time (in milliseconds) that the TechDocs backend will wait for + * a cache service to respond before continuing on as though the cached + * object was not found (e.g. when the cache sercice is unavailable). + * + * Defaults to 1000 milliseconds. + */ + readTimeout?: number; }; /** diff --git a/plugins/techdocs-backend/src/cache/TechDocsCache.test.ts b/plugins/techdocs-backend/src/cache/TechDocsCache.test.ts index e4308b7c3b..17056fc442 100644 --- a/plugins/techdocs-backend/src/cache/TechDocsCache.test.ts +++ b/plugins/techdocs-backend/src/cache/TechDocsCache.test.ts @@ -15,6 +15,7 @@ */ import { CacheClient, getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import { CacheInvalidationError, TechDocsCache } from './TechDocsCache'; const cached = (str: string): string => { @@ -31,7 +32,7 @@ describe('TechDocsCache', () => { set: jest.fn(), delete: jest.fn(), }; - CacheUnderTest = new TechDocsCache({ + CacheUnderTest = TechDocsCache.fromConfig(new ConfigReader({}), { cache: MockClient, logger: getVoidLogger(), }); @@ -55,7 +56,7 @@ describe('TechDocsCache', () => { expect(actual).toBe(undefined); }); - it('returns undefined if no response after 1s', async () => { + it('returns undefined if no response after 1s by default', async () => { const expectedPath = 'some/index.html'; MockClient.get.mockImplementationOnce(() => { return new Promise(resolve => { @@ -66,6 +67,28 @@ describe('TechDocsCache', () => { expect(actual).toBe(undefined); }); + it('returns undefined if no response after configured readTimeout', async () => { + const expectedPath = 'some/index.html'; + MockClient.get.mockImplementationOnce(() => { + return new Promise(resolve => { + setTimeout(() => resolve(cached('value')), 20); + }); + }); + + CacheUnderTest = TechDocsCache.fromConfig( + new ConfigReader({ + techdocs: { cache: { readTimeout: 10 } }, + }), + { + cache: MockClient, + logger: getVoidLogger(), + }, + ); + + const actual = await CacheUnderTest.get(expectedPath); + expect(actual).toBe(undefined); + }); + it('returns data if cache get returns it', async () => { const expectedPath = 'some/index.html'; MockClient.get.mockResolvedValueOnce(cached('expected value')); diff --git a/plugins/techdocs-backend/src/cache/TechDocsCache.ts b/plugins/techdocs-backend/src/cache/TechDocsCache.ts index 83497608cf..b12142dda5 100644 --- a/plugins/techdocs-backend/src/cache/TechDocsCache.ts +++ b/plugins/techdocs-backend/src/cache/TechDocsCache.ts @@ -14,6 +14,8 @@ * limitations under the License. */ import { CacheClient } from '@backstage/backend-common'; +import { assertError } from '@backstage/errors'; +import { Config } from '@backstage/config'; import { Logger } from 'winston'; export class CacheInvalidationError extends Error { @@ -28,10 +30,29 @@ export class CacheInvalidationError extends Error { export class TechDocsCache { protected readonly cache: CacheClient; protected readonly logger: Logger; + protected readonly readTimeout: number; - constructor({ cache, logger }: { cache: CacheClient; logger: Logger }) { + private constructor({ + cache, + logger, + readTimeout, + }: { + cache: CacheClient; + logger: Logger; + readTimeout: number; + }) { this.cache = cache; this.logger = logger; + this.readTimeout = readTimeout; + } + + static fromConfig( + config: Config, + { cache, logger }: { cache: CacheClient; logger: Logger }, + ) { + const readTimeout = + config.getOptionalNumber('techdocs.cache.readTimeout') || 1000; + return new TechDocsCache({ cache, logger, readTimeout }); } async get(path: string): Promise { @@ -40,7 +61,7 @@ export class TechDocsCache { // temporarily unreachable. const response = (await Promise.race([ this.cache.get(path), - new Promise(cancelAfter => setTimeout(cancelAfter, 1000)), + new Promise(cancelAfter => setTimeout(cancelAfter, this.readTimeout)), ])) as string | undefined; if (response !== undefined) { @@ -51,6 +72,7 @@ export class TechDocsCache { this.logger.debug(`Cache miss: ${path}`); return response; } catch (e) { + assertError(e); this.logger.warn(`Error getting cache entry ${path}: ${e.message}`); this.logger.debug(e.stack); return undefined; diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index ba3fc6ea87..8f48342683 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -91,7 +91,7 @@ export async function createRouter( const defaultTtl = config.getOptionalNumber('techdocs.cache.ttl'); if (isOutOfTheBoxOption(options) && options.cache && defaultTtl) { const cacheClient = options.cache.getClient({ defaultTtl }); - cache = new TechDocsCache({ cache: cacheClient, logger }); + cache = TechDocsCache.fromConfig(config, { cache: cacheClient, logger }); } const scmIntegrations = ScmIntegrations.fromConfig(config); From e4adc768a5a4c7ea2c5d4ab987dad3d400cb4f5a Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Nov 2021 15:07:39 +0100 Subject: [PATCH 24/50] Update architecture diagram. Signed-off-by: Eric Peterson --- .../architecture-recommended.drawio.svg | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/docs/assets/techdocs/architecture-recommended.drawio.svg b/docs/assets/techdocs/architecture-recommended.drawio.svg index 8ddc9dc35a..1768b5dc86 100644 --- a/docs/assets/techdocs/architecture-recommended.drawio.svg +++ b/docs/assets/techdocs/architecture-recommended.drawio.svg @@ -1,4 +1,4 @@ - + @@ -383,29 +383,6 @@ - - - - - - - - -
-
-
- - Invalidate Objects (Optional) - -
-
-
- - - Invalidate Objects (Optional) - - - From e0930f578e0c40a898b516bd1e5c63f78d13dce3 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Nov 2021 17:18:06 +0100 Subject: [PATCH 25/50] en-US lowercase locale Signed-off-by: Eric Peterson --- plugins/techdocs-backend/src/service/DocsSynchronizer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/techdocs-backend/src/service/DocsSynchronizer.ts b/plugins/techdocs-backend/src/service/DocsSynchronizer.ts index c7d5bf109a..4d407ab54a 100644 --- a/plugins/techdocs-backend/src/service/DocsSynchronizer.ts +++ b/plugins/techdocs-backend/src/service/DocsSynchronizer.ts @@ -186,7 +186,7 @@ export class DocsSynchronizer { ) || false; const tripletPath = `${namespace}/${kind}/${name}`; const entityTripletPath = `${ - legacyPathCasing ? tripletPath : tripletPath.toLocaleLowerCase() + legacyPathCasing ? tripletPath : tripletPath.toLocaleLowerCase('en-US') }`; try { const [sourceMetadata, cachedMetadata] = await Promise.all([ From 6e9e82b99de658f610553df6913da60fc1154468 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sun, 28 Nov 2021 13:33:11 -0700 Subject: [PATCH 26/50] Review feedback. Signed-off-by: Eric Peterson --- .../techdocs-common/src/stages/publish/local.ts | 4 +++- .../src/cache/TechDocsCache.test.ts | 4 +--- .../techdocs-backend/src/cache/TechDocsCache.ts | 16 ++++++---------- .../src/cache/cacheMiddleware.ts | 4 ++-- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 91a9b57a9e..fc64776e4c 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -132,7 +132,9 @@ export class LocalPublish implements PublisherBase { }); return { - remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`, + remoteUrl: `${techdocsApiUrl}/static/docs/${encodeURIComponent( + entity.metadata.name, + )}`, objects, }; } diff --git a/plugins/techdocs-backend/src/cache/TechDocsCache.test.ts b/plugins/techdocs-backend/src/cache/TechDocsCache.test.ts index 17056fc442..17613e1b58 100644 --- a/plugins/techdocs-backend/src/cache/TechDocsCache.test.ts +++ b/plugins/techdocs-backend/src/cache/TechDocsCache.test.ts @@ -162,9 +162,7 @@ describe('TechDocsCache', () => { await expect( CacheUnderTest.invalidateMultiple.bind(CacheUnderTest, expectedPaths), - ).rejects.toThrow( - expect.objectContaining({ rejections: expect.arrayContaining([]) }), - ); + ).rejects.toThrow(CacheInvalidationError); }); }); }); diff --git a/plugins/techdocs-backend/src/cache/TechDocsCache.ts b/plugins/techdocs-backend/src/cache/TechDocsCache.ts index b12142dda5..147aed0a18 100644 --- a/plugins/techdocs-backend/src/cache/TechDocsCache.ts +++ b/plugins/techdocs-backend/src/cache/TechDocsCache.ts @@ -14,18 +14,11 @@ * limitations under the License. */ import { CacheClient } from '@backstage/backend-common'; -import { assertError } from '@backstage/errors'; +import { assertError, CustomErrorBase } from '@backstage/errors'; import { Config } from '@backstage/config'; import { Logger } from 'winston'; -export class CacheInvalidationError extends Error { - public readonly rejections: PromiseRejectedResult[]; - - constructor(rejections: PromiseRejectedResult[]) { - super(); - this.rejections = rejections; - } -} +export class CacheInvalidationError extends CustomErrorBase {} export class TechDocsCache { protected readonly cache: CacheClient; @@ -101,7 +94,10 @@ export class TechDocsCache { ) as PromiseRejectedResult[]; if (rejected.length) { - throw new CacheInvalidationError(rejected); + throw new CacheInvalidationError( + 'TechDocs cache invalidation error', + rejected, + ); } return settled; diff --git a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts index 66471b0189..9e545c235c 100644 --- a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts +++ b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts @@ -16,7 +16,7 @@ import { Router } from 'express'; import router from 'express-promise-router'; import { Logger } from 'winston'; -import { TechDocsCache } from '.'; +import { TechDocsCache } from './TechDocsCache'; type CacheMiddlewareOptions = { cache: TechDocsCache; @@ -69,7 +69,7 @@ export const createCacheMiddleware = ({ socket.on('close', hadError => { const content = Buffer.concat(chunks); const head = content.toString('utf8', 0, 12); - if (writeToCache && !hadError && head === 'HTTP/1.1 200') { + if (writeToCache && !hadError && head.match(/HTTP\/\d\.\d 200/)) { cache.set(reqPath, content); } }); From dcabc18f7edf7aca99f63c1e76c82f18d53f5e54 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sun, 28 Nov 2021 13:44:21 -0700 Subject: [PATCH 27/50] Fix tests after rebase. Signed-off-by: Eric Peterson --- packages/techdocs-common/src/stages/publish/awsS3.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index 591131907b..3e4c4c705b 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -203,7 +203,13 @@ describe('AwsS3Publish', () => { const publisher = createPublisherFromConfig({ sse: 'aws:kms', }); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'default/component/backstage/404.html', + 'default/component/backstage/index.html', + 'default/component/backstage/assets/main.css', + ]), + }); }); it('should fail to publish a directory', async () => { From 2c009db3b640deb31c1bd3dfcce72fba0d817792 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 2 Dec 2021 12:20:44 -0500 Subject: [PATCH 28/50] Bump Spotify's prettier Signed-off-by: Adam Harvey --- packages/create-app/templates/default-app/package.json.hbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-app/templates/default-app/package.json.hbs b/packages/create-app/templates/default-app/package.json.hbs index bdae71d9e3..b0747c4454 100644 --- a/packages/create-app/templates/default-app/package.json.hbs +++ b/packages/create-app/templates/default-app/package.json.hbs @@ -31,7 +31,7 @@ }, "devDependencies": { "@backstage/cli": "^{{version '@backstage/cli'}}", - "@spotify/prettier-config": "^11.0.0", + "@spotify/prettier-config": "^12.0.0", "concurrently": "^6.0.0", "lerna": "^4.0.0", "prettier": "^2.3.2" From 2c503737550f0b14951f11e428160fbbdf2dd19d Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 2 Dec 2021 12:21:06 -0500 Subject: [PATCH 29/50] Move test-utils to devDeps Signed-off-by: Adam Harvey --- .../templates/default-app/packages/app/package.json.hbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-app/templates/default-app/packages/app/package.json.hbs b/packages/create-app/templates/default-app/packages/app/package.json.hbs index 9e15608a01..fd4de85b60 100644 --- a/packages/create-app/templates/default-app/packages/app/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/app/package.json.hbs @@ -22,7 +22,6 @@ "@backstage/plugin-tech-radar": "^{{version '@backstage/plugin-tech-radar'}}", "@backstage/plugin-techdocs": "^{{version '@backstage/plugin-techdocs'}}", "@backstage/plugin-user-settings": "^{{version '@backstage/plugin-user-settings'}}", - "@backstage/test-utils": "^{{version '@backstage/test-utils'}}", "@backstage/theme": "^{{version '@backstage/theme'}}", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", @@ -34,6 +33,7 @@ "react-use": "^15.3.3" }, "devDependencies": { + "@backstage/test-utils": "^{{version '@backstage/test-utils'}}", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", From cfd019b72114da5a536bfc41e08806d5cff900f8 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 2 Dec 2021 12:29:16 -0500 Subject: [PATCH 30/50] Update Backstage docs link Signed-off-by: Adam Harvey --- packages/create-app/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-app/README.md b/packages/create-app/README.md index 9f86cdb448..8ed2fb93d5 100644 --- a/packages/create-app/README.md +++ b/packages/create-app/README.md @@ -22,4 +22,4 @@ yarn backstage-create-app ## Documentation - [Backstage Readme](https://github.com/backstage/backstage/blob/master/README.md) -- [Backstage Documentation](https://github.com/backstage/backstage/blob/master/docs/README.md) +- [Backstage Documentation](hhttps://backstage.io/docs/) From 2d3dd1e76fce11642314989f6e50bba758efa88e Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 2 Dec 2021 12:29:39 -0500 Subject: [PATCH 31/50] Fix doco typo Signed-off-by: Adam Harvey --- packages/create-app/src/createApp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-app/src/createApp.ts b/packages/create-app/src/createApp.ts index d301792380..ac42ed0ff3 100644 --- a/packages/create-app/src/createApp.ts +++ b/packages/create-app/src/createApp.ts @@ -65,7 +65,7 @@ export default async (cmd: Command, version: string): Promise => { const templateDir = paths.resolveOwn('templates/default-app'); const tempDir = resolvePath(os.tmpdir(), answers.name); - // Use `--path` argument as applicaiton directory when specified, otherwise + // Use `--path` argument as application directory when specified, otherwise // create a directory using `answers.name` const appDir = cmd.path ? resolvePath(paths.targetDir, cmd.path) From bdf357956cd2dfd251f5c714ec93921bf2201462 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 2 Dec 2021 12:33:38 -0500 Subject: [PATCH 32/50] Align scripts order to monorepo Signed-off-by: Adam Harvey --- .../templates/default-app/packages/app/package.json.hbs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/create-app/templates/default-app/packages/app/package.json.hbs b/packages/create-app/templates/default-app/packages/app/package.json.hbs index fd4de85b60..68848d2de9 100644 --- a/packages/create-app/templates/default-app/packages/app/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/app/package.json.hbs @@ -48,11 +48,11 @@ "scripts": { "start": "backstage-cli app:serve", "build": "backstage-cli app:build", - "test": "backstage-cli test", - "lint": "backstage-cli lint", "clean": "backstage-cli clean", + "test": "backstage-cli test", "test:e2e": "cross-env PORT=3001 start-server-and-test start http://localhost:3001 cy:dev", "test:e2e:ci": "cross-env PORT=3001 start-server-and-test start http://localhost:3001 cy:run", + "lint": "backstage-cli lint", "cy:dev": "cypress open", "cy:run": "cypress run" }, From 4862fbc64f5503a5f5944cb69a64d575c20339bc Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 2 Dec 2021 12:42:19 -0500 Subject: [PATCH 33/50] Add changeset Signed-off-by: Adam Harvey --- .changeset/thirty-readers-attack.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/thirty-readers-attack.md diff --git a/.changeset/thirty-readers-attack.md b/.changeset/thirty-readers-attack.md new file mode 100644 index 0000000000..bb05fd45f2 --- /dev/null +++ b/.changeset/thirty-readers-attack.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': patch +--- + +Alignments of template with Backstage example app in monorepo From 7d24709264d4a673ea77730ad7519138cb139754 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 2 Dec 2021 14:30:44 -0500 Subject: [PATCH 34/50] Fix URL typo Signed-off-by: Adam Harvey --- packages/create-app/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-app/README.md b/packages/create-app/README.md index 8ed2fb93d5..d5b5039d4e 100644 --- a/packages/create-app/README.md +++ b/packages/create-app/README.md @@ -22,4 +22,4 @@ yarn backstage-create-app ## Documentation - [Backstage Readme](https://github.com/backstage/backstage/blob/master/README.md) -- [Backstage Documentation](hhttps://backstage.io/docs/) +- [Backstage Documentation](https://backstage.io/docs/) From c401170b360d58d9295d4d7d6b940c5db423455f Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 2 Dec 2021 14:47:36 -0500 Subject: [PATCH 35/50] Update changeset wording Signed-off-by: Adam Harvey --- .changeset/thirty-readers-attack.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/thirty-readers-attack.md b/.changeset/thirty-readers-attack.md index bb05fd45f2..cd05f4ebf4 100644 --- a/.changeset/thirty-readers-attack.md +++ b/.changeset/thirty-readers-attack.md @@ -2,4 +2,4 @@ '@backstage/create-app': patch --- -Alignments of template with Backstage example app in monorepo +Bump @spotify/prettier-config From e5976071eae1c2164d788832fb883762148cc54f Mon Sep 17 00:00:00 2001 From: Phil Kuang Date: Thu, 2 Dec 2021 17:10:00 -0500 Subject: [PATCH 36/50] fix(sidebar): use ellipsis styling for overflowed text Signed-off-by: Phil Kuang --- .changeset/olive-pens-poke.md | 5 +++++ packages/core-components/src/layout/Sidebar/Items.tsx | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changeset/olive-pens-poke.md diff --git a/.changeset/olive-pens-poke.md b/.changeset/olive-pens-poke.md new file mode 100644 index 0000000000..8a009af971 --- /dev/null +++ b/.changeset/olive-pens-poke.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-components': patch +--- + +Use ellipsis style for overflowed text in sidebar menu diff --git a/packages/core-components/src/layout/Sidebar/Items.tsx b/packages/core-components/src/layout/Sidebar/Items.tsx index a41c3e4bdd..903053eda4 100644 --- a/packages/core-components/src/layout/Sidebar/Items.tsx +++ b/packages/core-components/src/layout/Sidebar/Items.tsx @@ -235,7 +235,7 @@ const SidebarItemWithSubmenu = ({ {itemIcon}
{text && ( - + {text} )} @@ -402,7 +402,7 @@ export const SidebarItem = forwardRef((props, ref) => { {itemIcon}
{text && ( - + {text} )} From 343ef34599c642f8a26951da7c7839099f3f25c1 Mon Sep 17 00:00:00 2001 From: Niklas Granander Date: Fri, 3 Dec 2021 16:10:55 +0100 Subject: [PATCH 37/50] Handle missing Xcode data from old versions of XCMetrics Signed-off-by: Niklas Granander --- .changeset/three-coins-kiss.md | 5 +++++ plugins/xcmetrics/src/api/types.ts | 2 +- .../components/BuildDetails/BuildDetails.test.tsx | 14 ++++++++++++++ .../src/components/BuildDetails/BuildDetails.tsx | 2 +- 4 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 .changeset/three-coins-kiss.md diff --git a/.changeset/three-coins-kiss.md b/.changeset/three-coins-kiss.md new file mode 100644 index 0000000000..3a231be259 --- /dev/null +++ b/.changeset/three-coins-kiss.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-xcmetrics': patch +--- + +Handle a case where Xcode data from XCMetrics version < 0.0.8 could be missing diff --git a/plugins/xcmetrics/src/api/types.ts b/plugins/xcmetrics/src/api/types.ts index 44f27872ca..6c7d133259 100644 --- a/plugins/xcmetrics/src/api/types.ts +++ b/plugins/xcmetrics/src/api/types.ts @@ -163,7 +163,7 @@ export type Xcode = { export type BuildResponse = { build: Build; targets: Target[]; - xcode: Xcode; + xcode?: Xcode; // Can be undefined if XCMetrics version < v0.0.8 }; export type BuildFilters = { diff --git a/plugins/xcmetrics/src/components/BuildDetails/BuildDetails.test.tsx b/plugins/xcmetrics/src/components/BuildDetails/BuildDetails.test.tsx index 7083d890d6..c26c101143 100644 --- a/plugins/xcmetrics/src/components/BuildDetails/BuildDetails.test.tsx +++ b/plugins/xcmetrics/src/components/BuildDetails/BuildDetails.test.tsx @@ -51,6 +51,20 @@ describe('BuildDetails', () => { ).toBeInTheDocument(); expect(rendered.getByText(client.mockBuild.schema)).toBeInTheDocument(); }); + + it('should render if xcode data is not present', async () => { + const rendered = await renderInTestApp( + + + , + ); + + expect( + rendered.getByText('Xcode').parentNode?.childNodes[1].textContent, + ).toEqual('Unknown'); + }); }); describe('BuildDetails with request', () => { diff --git a/plugins/xcmetrics/src/components/BuildDetails/BuildDetails.tsx b/plugins/xcmetrics/src/components/BuildDetails/BuildDetails.tsx index 1d77bf7a7a..8732bd92d8 100644 --- a/plugins/xcmetrics/src/components/BuildDetails/BuildDetails.tsx +++ b/plugins/xcmetrics/src/components/BuildDetails/BuildDetails.tsx @@ -78,7 +78,7 @@ export const BuildDetails = ({ {formatStatus(build.buildStatus)} ), - xcode: `${xcode.version} (${xcode.buildNumber})`, + xcode: xcode ? `${xcode.version} (${xcode.buildNumber})` : 'Unknown', CI: build.isCi, }; From cfd01a226956fde10c3a2d5b7f6b4fa3cdfcbe8a Mon Sep 17 00:00:00 2001 From: Niklas Granander Date: Fri, 3 Dec 2021 16:58:41 +0100 Subject: [PATCH 38/50] Fix spelling in changeset Signed-off-by: Niklas Granander --- .changeset/three-coins-kiss.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/three-coins-kiss.md b/.changeset/three-coins-kiss.md index 3a231be259..a4b0c83288 100644 --- a/.changeset/three-coins-kiss.md +++ b/.changeset/three-coins-kiss.md @@ -2,4 +2,4 @@ '@backstage/plugin-xcmetrics': patch --- -Handle a case where Xcode data from XCMetrics version < 0.0.8 could be missing +Handle a case where XCode data from backend (before 0.0.8) could be missing From 3ebc93120233a56a6df5d8b8e27853d0291cd5e4 Mon Sep 17 00:00:00 2001 From: Milton Jacomini Neto Date: Fri, 3 Dec 2021 17:37:52 -0300 Subject: [PATCH 39/50] Add Mosaico to ADOPTERS List --- ADOPTERS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ADOPTERS.md b/ADOPTERS.md index 529c22d144..8f83dbc355 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -73,3 +73,5 @@ | [QBE](https://www.qbe.com/) | [Daniel Steel](https://github.com/danielsteelqbe), [Pete Jespers](https://github.com/petejespersqbe) | Developer portal allowing our global teams to explore and create applications, documentation and cloud infrastructure easily and quickly 🚀 | | [LogMeIn](https://www.logmein.com) | [Lorenzo Orsatti](https://github.com/lorsatti) | Improve onboarding experience of new developers. Discover faster and painlessly developer documentation, API definitions and team information. Provide useful dev metrics in a central place. Provide easy-to-use templates for new services. | | [Telstra](https://www.telstra.com.au) | [@kiranpatel11](https://github.com/kiranpatel11), [JasonC](https://github.com/JasonC17) | Primary usage: software catalog and templates
Emerging usage : TechDocs, Explore Ecosystem, TechRadar, etc | + +| [Mosaico](https://www.mosaico.com.br/) | [Wédney Yuri](https://github.com/wedneyyuri),[@tino.milton](https://github.com/miltonjacomini) | A centralized service catalog of our documentation for our service engineers. From 113a7442d8de8b5d916e1619cb87ffb2bce7cf6c Mon Sep 17 00:00:00 2001 From: Nataliya Issayeva Date: Fri, 3 Dec 2021 16:34:27 -0500 Subject: [PATCH 40/50] Fix scaffolder repo owner bug Signed-off-by: Nataliya Issayeva --- .../src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx index 12931c297f..c4b1e79262 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx @@ -27,7 +27,7 @@ import FormHelperText from '@material-ui/core/FormHelperText'; import { useApi } from '@backstage/core-plugin-api'; import { Progress } from '@backstage/core-components'; -function splitFormData(url: string | undefined, allowedOwners: string[]) { +function splitFormData(url: string | undefined, allowedOwners?: string[]) { let host = undefined; let owner = undefined; let repo = undefined; @@ -39,10 +39,7 @@ function splitFormData(url: string | undefined, allowedOwners: string[]) { if (url) { const parsed = new URL(`https://${url}`); host = parsed.host; - owner = - parsed.searchParams.get('owner') || allowedOwners - ? allowedOwners[0] - : undefined; + owner = parsed.searchParams.get('owner') || allowedOwners?.[0]; repo = parsed.searchParams.get('repo') || undefined; // This is azure dev ops specific. not used for any other provider. organization = parsed.searchParams.get('organization') || undefined; From 2edcf7738f01a6672ed1605da2eaab57246c02cc Mon Sep 17 00:00:00 2001 From: Nataliya Issayeva Date: Fri, 3 Dec 2021 16:48:53 -0500 Subject: [PATCH 41/50] Add changeset Signed-off-by: Nataliya Issayeva --- .changeset/light-pigs-protect.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/light-pigs-protect.md diff --git a/.changeset/light-pigs-protect.md b/.changeset/light-pigs-protect.md new file mode 100644 index 0000000000..358547af8a --- /dev/null +++ b/.changeset/light-pigs-protect.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +Fix bug with setting owner in RepoUrlPicker causing validation failure From dd19340ebc1ae91b5881aa14463678d79a67ab00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sun, 5 Dec 2021 12:37:12 +0100 Subject: [PATCH 42/50] make prettier happy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- ADOPTERS.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ADOPTERS.md b/ADOPTERS.md index 8f83dbc355..55c6a81db5 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -73,5 +73,4 @@ | [QBE](https://www.qbe.com/) | [Daniel Steel](https://github.com/danielsteelqbe), [Pete Jespers](https://github.com/petejespersqbe) | Developer portal allowing our global teams to explore and create applications, documentation and cloud infrastructure easily and quickly 🚀 | | [LogMeIn](https://www.logmein.com) | [Lorenzo Orsatti](https://github.com/lorsatti) | Improve onboarding experience of new developers. Discover faster and painlessly developer documentation, API definitions and team information. Provide useful dev metrics in a central place. Provide easy-to-use templates for new services. | | [Telstra](https://www.telstra.com.au) | [@kiranpatel11](https://github.com/kiranpatel11), [JasonC](https://github.com/JasonC17) | Primary usage: software catalog and templates
Emerging usage : TechDocs, Explore Ecosystem, TechRadar, etc | - -| [Mosaico](https://www.mosaico.com.br/) | [Wédney Yuri](https://github.com/wedneyyuri),[@tino.milton](https://github.com/miltonjacomini) | A centralized service catalog of our documentation for our service engineers. +| [Mosaico](https://www.mosaico.com.br/) | [Wédney Yuri](https://github.com/wedneyyuri),[@tino.milton](https://github.com/miltonjacomini) | A centralized service catalog of our documentation for our service engineers. | From ef600b3c8cfb84cb02fdfed6bb767b42257f96da Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sun, 5 Dec 2021 16:50:09 -0600 Subject: [PATCH 43/50] Review feedback. Signed-off-by: Eric Peterson --- packages/techdocs-common/src/stages/publish/local.ts | 10 ++++++---- plugins/techdocs-backend/package.json | 1 + plugins/techdocs-backend/src/cache/TechDocsCache.ts | 4 ++-- .../techdocs-backend/src/cache/cacheMiddleware.test.ts | 10 +++++----- plugins/techdocs-backend/src/cache/cacheMiddleware.ts | 6 +++--- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index fc64776e4c..9c146d2935 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -127,15 +127,17 @@ export class LocalPublish implements PublisherBase { // Generate publish response. const techdocsApiUrl = await this.discovery.getBaseUrl('techdocs'); - const objects = (await getFileTreeRecursively(publishDir)).map(abs => { - return abs.split(`${staticDocsDir}/`)[1]; - }); + const publishedFilePaths = (await getFileTreeRecursively(publishDir)).map( + abs => { + return abs.split(`${staticDocsDir}/`)[1]; + }, + ); return { remoteUrl: `${techdocsApiUrl}/static/docs/${encodeURIComponent( entity.metadata.name, )}`, - objects, + objects: publishedFilePaths, }; } diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index ffad4bc4c1..6f3495737a 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -40,6 +40,7 @@ "@backstage/search-common": "^0.2.1", "@backstage/techdocs-common": "^0.10.8", "@types/express": "^4.17.6", + "cross-fetch": "^3.0.6", "dockerode": "^3.3.1", "express": "^4.17.1", "express-promise-router": "^4.1.0", diff --git a/plugins/techdocs-backend/src/cache/TechDocsCache.ts b/plugins/techdocs-backend/src/cache/TechDocsCache.ts index 147aed0a18..807d6ca87b 100644 --- a/plugins/techdocs-backend/src/cache/TechDocsCache.ts +++ b/plugins/techdocs-backend/src/cache/TechDocsCache.ts @@ -43,8 +43,8 @@ export class TechDocsCache { config: Config, { cache, logger }: { cache: CacheClient; logger: Logger }, ) { - const readTimeout = - config.getOptionalNumber('techdocs.cache.readTimeout') || 1000; + const timeout = config.getOptionalNumber('techdocs.cache.readTimeout'); + const readTimeout = timeout === undefined ? 1000 : timeout; return new TechDocsCache({ cache, logger, readTimeout }); } diff --git a/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts b/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts index 91e4b84222..312674a5bf 100644 --- a/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts +++ b/plugins/techdocs-backend/src/cache/cacheMiddleware.test.ts @@ -59,7 +59,7 @@ describe('createCacheMiddleware', () => { app = express().use(router); app.use((req, res, next) => { // By default, send cacheable content. - if (req.path !== '/api/static/docs/error.png') { + if (req.path !== '/static/docs/error.png') { res.send('default-response'); } else { next(new Error()); @@ -70,7 +70,7 @@ describe('createCacheMiddleware', () => { describe('middleware', () => { it('does not apply to non-static/docs paths', async () => { await request(app) - .get('/api/static/not-docs') + .get('/static/not-docs') .expect(200, 'default-response'); expect(cache.set).not.toHaveBeenCalled(); @@ -79,7 +79,7 @@ describe('createCacheMiddleware', () => { it('responds with cached response', async () => { cache.get.mockResolvedValueOnce(getMockHttpResponseFor('xyz')); - await request(app).get('/api/static/docs/foo.html').expect(200, 'xyz'); + await request(app).get('/static/docs/foo.html').expect(200, 'xyz'); await waitForSocketClose(); expect(cache.set).not.toHaveBeenCalled(); @@ -88,7 +88,7 @@ describe('createCacheMiddleware', () => { it('sets cache when content is cacheable', async () => { const expectedPath = 'default/api/xyz/index.html'; await request(app) - .get(`/api/static/docs/${expectedPath}`) + .get(`/static/docs/${expectedPath}`) .expect(200, 'default-response'); await waitForSocketClose(); @@ -100,7 +100,7 @@ describe('createCacheMiddleware', () => { }); it('does not set cache on error', async () => { - await request(app).get('/api/static/docs/error.png').expect(500); + await request(app).get('/static/docs/error.png').expect(500); await waitForSocketClose(); expect(cache.set).not.toHaveBeenCalled(); diff --git a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts index 9e545c235c..cb59681304 100644 --- a/plugins/techdocs-backend/src/cache/cacheMiddleware.ts +++ b/plugins/techdocs-backend/src/cache/cacheMiddleware.ts @@ -35,7 +35,7 @@ export const createCacheMiddleware = ({ // loaded from cache. Cache key is the object's path (after `/static/docs/`). cacheMiddleware.use(async (req, res, next) => { const socket = res.socket; - const isCacheable = req.path.includes('/static/docs/'); + const isCacheable = req.path.startsWith('/static/docs/'); // Continue early if this is non-cacheable, or there's no socket. if (!isCacheable || !socket) { @@ -66,11 +66,11 @@ export const createCacheMiddleware = ({ // When a socket is closed, if there were no errors and the data written // over the socket should be cached, cache it! - socket.on('close', hadError => { + socket.on('close', async hadError => { const content = Buffer.concat(chunks); const head = content.toString('utf8', 0, 12); if (writeToCache && !hadError && head.match(/HTTP\/\d\.\d 200/)) { - cache.set(reqPath, content); + await cache.set(reqPath, content); } }); From e3e6e9a86d2d654d1378e52adbf2f3ec03d00c2a Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sun, 5 Dec 2021 17:05:07 -0600 Subject: [PATCH 44/50] One last docs update to clarify new architecture. Signed-off-by: Eric Peterson --- docs/features/techdocs/configuration.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 983a8dd3e7..6317ae365b 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -140,8 +140,9 @@ techdocs: # be configured with a valid cache store. cache: # Represents the number of milliseconds a statically built asset should - # stay cached. Cache invalidation is handled automatically if you publish - # to storage using the techdocs-cli, allowing long TTLs (e.g. 1 month/year) + # stay cached. Cache invalidation is handled automatically by the frontend, + # which compares the build times in cached metadata vs. canonical storage, + # allowing long TTLs (e.g. 1 month/year) ttl: 3600000 # (Optional) The time (in milliseconds) that the TechDocs backend will wait From b1da6c74d26c65803260f1e8d11a7373a4ae26e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Dec 2021 04:08:22 +0000 Subject: [PATCH 45/50] chore(deps-dev): bump prettier from 2.5.0 to 2.5.1 in /microsite Bumps [prettier](https://github.com/prettier/prettier) from 2.5.0 to 2.5.1. - [Release notes](https://github.com/prettier/prettier/releases) - [Changelog](https://github.com/prettier/prettier/blob/main/CHANGELOG.md) - [Commits](https://github.com/prettier/prettier/compare/2.5.0...2.5.1) --- updated-dependencies: - dependency-name: prettier dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- microsite/package.json | 2 +- microsite/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/microsite/package.json b/microsite/package.json index a24de8935d..0a90cb2ec5 100644 --- a/microsite/package.json +++ b/microsite/package.json @@ -19,7 +19,7 @@ "@spotify/prettier-config": "^12.0.0", "docusaurus": "^2.0.0-alpha.70", "js-yaml": "^4.1.0", - "prettier": "^2.5.0", + "prettier": "^2.5.1", "yarn-lock-check": "^1.0.5" }, "prettier": "@spotify/prettier-config" diff --git a/microsite/yarn.lock b/microsite/yarn.lock index 9946f4914d..b4d97d549d 100644 --- a/microsite/yarn.lock +++ b/microsite/yarn.lock @@ -5190,10 +5190,10 @@ prepend-http@^2.0.0: resolved "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897" integrity sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc= -prettier@^2.5.0: - version "2.5.0" - resolved "https://registry.npmjs.org/prettier/-/prettier-2.5.0.tgz#a6370e2d4594e093270419d9cc47f7670488f893" - integrity sha512-FM/zAKgWTxj40rH03VxzIPdXmj39SwSjwG0heUcNFwI+EMZJnY93yAiKXM3dObIKAM5TA88werc8T/EwhB45eg== +prettier@^2.5.1: + version "2.5.1" + resolved "https://registry.npmjs.org/prettier/-/prettier-2.5.1.tgz#fff75fa9d519c54cf0fce328c1017d94546bc56a" + integrity sha512-vBZcPRUR5MZJwoyi3ZoyQlc1rXeEck8KgeC9AwwOn+exuxLxq5toTRDTSaVrXHxelDMHy9zlicw8u66yxoSUFg== prismjs@^1.22.0: version "1.25.0" From e13f7e5605baf841d5bc4d1eda8eca3c218ae0ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Dec 2021 04:13:29 +0000 Subject: [PATCH 46/50] chore(deps): bump express-prom-bundle from 6.3.6 to 6.4.1 Bumps [express-prom-bundle](https://github.com/jochen-schweizer/express-prom-bundle) from 6.3.6 to 6.4.1. - [Release notes](https://github.com/jochen-schweizer/express-prom-bundle/releases) - [Commits](https://github.com/jochen-schweizer/express-prom-bundle/compare/6.3.6...6.4.1) --- updated-dependencies: - dependency-name: express-prom-bundle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index c28d72146b..e19cbcd3bd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14398,9 +14398,9 @@ expect@^26.6.2: jest-regex-util "^26.0.0" express-prom-bundle@^6.3.6: - version "6.3.6" - resolved "https://registry.npmjs.org/express-prom-bundle/-/express-prom-bundle-6.3.6.tgz#c8da1c1024edfcc54953c365991aca57ffd0cfda" - integrity sha512-IRsTRCEKCVCHEriQlZ1FuutjEFc89KASsveXh+1HcGEnuZKiAC4LugxrsGEIdySqYvqOYSr2SWHJ6L8/BK2SHA== + version "6.4.1" + resolved "https://registry.npmjs.org/express-prom-bundle/-/express-prom-bundle-6.4.1.tgz#a688050b9e090f6969825c33143106d3e0e5a70e" + integrity sha512-Sg0svLQe/SS5z1tHDTVfZVjNumobiDlXM0jmemt5Dm9K6BX8z9yCwEr93zbko6fNMR4zKav77iPfxUWi6gAjNA== dependencies: on-finished "^2.3.0" url-value-parser "^2.0.0" From ef3a6977cfd8d7d84735e00e160cd5f1998200b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Dec 2021 04:16:45 +0000 Subject: [PATCH 47/50] chore(deps-dev): bump @types/compression from 1.7.0 to 1.7.2 Bumps [@types/compression](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/compression) from 1.7.0 to 1.7.2. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/compression) --- updated-dependencies: - dependency-name: "@types/compression" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index c28d72146b..7eb934c8d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6923,9 +6923,9 @@ commander "*" "@types/compression@^1.7.0": - version "1.7.0" - resolved "https://registry.npmjs.org/@types/compression/-/compression-1.7.0.tgz#8dc2a56604873cf0dd4e746d9ae4d31ae77b2390" - integrity sha512-3LzWUM+3k3XdWOUk/RO+uSjv7YWOatYq2QADJntK1pjkk4DfVP0KrIEPDnXRJxAAGKe0VpIPRmlINLDuCedZWw== + version "1.7.2" + resolved "https://registry.npmjs.org/@types/compression/-/compression-1.7.2.tgz#7cc1cdb01b4730eea284615a68fc70a2cdfd5e71" + integrity sha512-lwEL4M/uAGWngWFLSG87ZDr2kLrbuR8p7X+QZB1OQlT+qkHsCPDVFnHPyXf4Vyl4yDDorNY+mAhosxkCvppatg== dependencies: "@types/express" "*" From f22299958f46a33b15511ca1861e18361f3aff23 Mon Sep 17 00:00:00 2001 From: Nick Laqua <52642976+nick-laqua-dragon@users.noreply.github.com> Date: Mon, 6 Dec 2021 17:38:17 +0800 Subject: [PATCH 48/50] Update ADOPTERS.md Adding Mox Bank as a public adopter --- ADOPTERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ADOPTERS.md b/ADOPTERS.md index 55c6a81db5..b611bf66e8 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -74,3 +74,4 @@ | [LogMeIn](https://www.logmein.com) | [Lorenzo Orsatti](https://github.com/lorsatti) | Improve onboarding experience of new developers. Discover faster and painlessly developer documentation, API definitions and team information. Provide useful dev metrics in a central place. Provide easy-to-use templates for new services. | | [Telstra](https://www.telstra.com.au) | [@kiranpatel11](https://github.com/kiranpatel11), [JasonC](https://github.com/JasonC17) | Primary usage: software catalog and templates
Emerging usage : TechDocs, Explore Ecosystem, TechRadar, etc | | [Mosaico](https://www.mosaico.com.br/) | [Wédney Yuri](https://github.com/wedneyyuri),[@tino.milton](https://github.com/miltonjacomini) | A centralized service catalog of our documentation for our service engineers. | +| [Mox Bank](https://www.mox.com/) | [Nick Laqua](https://github.com/nick-laqua-dragon), [Gauthier Roebroeck](https://github.com/gauthier-roebroeck-mox) | "Single pane of glass" developer portal for providing a best-in-class developer experience to our product teams and making Mox the best tech environment in Hongkong 🥰🚀 | From 594e73aa1c222820f7f96f0162076aea38adec25 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 6 Dec 2021 14:29:07 +0100 Subject: [PATCH 49/50] use path.sep instead of / to accound for differnt platforms (#8380) Signed-off-by: Emma Indal --- packages/techdocs-common/src/stages/generate/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/techdocs-common/src/stages/generate/helpers.ts b/packages/techdocs-common/src/stages/generate/helpers.ts index f96d1209b6..794200fb7d 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.ts @@ -384,7 +384,7 @@ export const createOrUpdateMetadata = async ( // a form appropriate for invalidating the associated object from cache. try { json.files = (await getFileTreeRecursively(techdocsMetadataDir)).map(file => - file.replace(`${techdocsMetadataDir}/`, ''), + file.replace(`${techdocsMetadataDir}${path.sep}`, ''), ); } catch (err) { assertError(err); From 36bb4fb2e95dcf76c856e5777482bdcc90118d5c Mon Sep 17 00:00:00 2001 From: Tim Hansen Date: Mon, 6 Dec 2021 13:13:03 -0700 Subject: [PATCH 50/50] More scaffolder config cleanup Signed-off-by: Tim Hansen --- .changeset/tiny-trains-notice.md | 5 ++++ .../software-templates/configuration.md | 28 ++++++++++++++----- .../templates/default-app/app-config.yaml.hbs | 4 +-- 3 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 .changeset/tiny-trains-notice.md diff --git a/.changeset/tiny-trains-notice.md b/.changeset/tiny-trains-notice.md new file mode 100644 index 0000000000..e6d260633e --- /dev/null +++ b/.changeset/tiny-trains-notice.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': patch +--- + +Removed the `scaffolder.github.visibility` configuration that is no longer used from the default app template. diff --git a/docs/features/software-templates/configuration.md b/docs/features/software-templates/configuration.md index b57bb96818..d22b82bc16 100644 --- a/docs/features/software-templates/configuration.md +++ b/docs/features/software-templates/configuration.md @@ -18,17 +18,31 @@ The next step is to add [add templates](http://backstage.io/docs/features/software-templates/adding-templates) to your Backstage app. -### GitHub +### Publishing defaults -For GitHub, you can configure who can see the new repositories that are created -by specifying `visibility` option. Valid options are `public`, `private` and -`internal`. The `internal` option is for GitHub Enterprise clients, which means -public within the enterprise. +Software templates can define _publish_ actions, such as `publish:github`, to +create new repositories or submit pull / merge requests to existing +repositories. You can configure the author and commit message through the +`scaffolder` configuration in `app-config.yaml`: ```yaml scaffolder: - github: - visibility: public # or 'internal' or 'private' + defaultAuthor: + name: M.C. Hammer # Defaults to `Scaffolder` + email: hammer@donthurtem.com # Defaults to `scaffolder@backstage.io` + defaultCommitMessage: "U can't touch this" # Defaults to 'Initial commit' +``` + +To configure who can see the new repositories created from software templates, +add the `repoVisibility` key within a software template: + +```yaml +- id: publish + name: Publish + action: publish:github + input: + repoUrl: '{{ parameters.repoUrl }}' + repoVisibility: public # or 'internal' or 'private' ``` ### Disabling Docker in Docker situation (Optional) diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index be144d91f7..c804b4b561 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -77,9 +77,7 @@ auth: providers: {} scaffolder: - github: - token: ${GITHUB_TOKEN} - visibility: public # or 'internal' or 'private' + # see https://backstage.io/docs/features/software-templates/configuration for software template options catalog: rules: