From 8ae4208c93f5f41ac8b3004d25dc7c551c9f9b15 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 12 Oct 2021 17:32:29 +0200 Subject: [PATCH 1/3] workflows/ci: run tests in a band as they are already parallelized by lerna Signed-off-by: Patrik Oldsberg --- .github/workflows/ci.yml | 4 ++-- .github/workflows/master-win.yml | 2 +- .github/workflows/master.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b68a96efbe..7222edef08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,7 @@ jobs: - name: test changed packages if: ${{ steps.yarn-lock.outcome == 'success' }} - run: yarn lerna -- run test --since origin/master -- --coverage + run: yarn lerna -- run test --since origin/master -- --coverage --runInBand env: BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }} BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }} @@ -149,7 +149,7 @@ jobs: - name: test all packages (and upload coverage) if: ${{ steps.yarn-lock.outcome == 'failure' }} run: | - yarn lerna -- run test -- --coverage + yarn lerna -- run test -- --coverage --runInBand bash <(curl -s https://codecov.io/bash) -N $(git rev-parse FETCH_HEAD) env: BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }} diff --git a/.github/workflows/master-win.yml b/.github/workflows/master-win.yml index 873f12086a..c6a9222483 100644 --- a/.github/workflows/master-win.yml +++ b/.github/workflows/master-win.yml @@ -58,7 +58,7 @@ jobs: run: lerna run --scope @backstage/core-* build - name: test - run: yarn lerna -- run test + run: yarn lerna -- run test --runInBand env: BACKSTAGE_TEST_DISABLE_DOCKER: 1 diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 7bd73e2c61..8106dd1198 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -116,7 +116,7 @@ jobs: - name: test (and upload coverage) run: | - yarn lerna -- run test -- --coverage + yarn lerna -- run test -- --coverage --runInBand bash <(curl -s https://codecov.io/bash) # Upload code coverage for some specific flags. Also see .codecov.yml bash <(curl -s https://codecov.io/bash) -f packages/core-app-api/coverage/* -F core-app-api From b486adb8c6707e010044178ba3f19557ac4ee6a7 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 9 Oct 2021 14:24:01 +0200 Subject: [PATCH 2/3] cli: rework jest ESM handling by transforming by default Signed-off-by: Patrik Oldsberg --- .changeset/brave-waves-explain.md | 15 +++++ .changeset/wise-camels-run.md | 13 ++++ package.json | 5 -- packages/cli/config/jest.js | 56 ++++++++++++---- packages/cli/config/jestSucraseTransform.js | 66 +++++++++++++++++++ .../templates/default-app/package.json.hbs | 5 -- 6 files changed, 136 insertions(+), 24 deletions(-) create mode 100644 .changeset/brave-waves-explain.md create mode 100644 .changeset/wise-camels-run.md create mode 100644 packages/cli/config/jestSucraseTransform.js diff --git a/.changeset/brave-waves-explain.md b/.changeset/brave-waves-explain.md new file mode 100644 index 0000000000..5e76f192e0 --- /dev/null +++ b/.changeset/brave-waves-explain.md @@ -0,0 +1,15 @@ +--- +'@backstage/create-app': patch +--- + +Removed the included `jest` configuration from the root `package.json` as the `transformModules` option no longer exists. + +To apply this change to an existing app, make the follow change to the root `package.json`: + +```diff +- "jest": { +- "transformModules": [ +- "@asyncapi/react-component" +- ] +- } +``` diff --git a/.changeset/wise-camels-run.md b/.changeset/wise-camels-run.md new file mode 100644 index 0000000000..1e81e2f9ac --- /dev/null +++ b/.changeset/wise-camels-run.md @@ -0,0 +1,13 @@ +--- +'@backstage/cli': minor +--- + +The Jest configuration that's included with the Backstage CLI has received several changes. + +As a part of migrating to more widespread usage of ESM modules, the default configuration now transforms all source files everywhere, including those within `node_modules`. Due to this change the existing `transformModules` option has been removed and will be ignored. There is also a list of known packages that do not require transforms in the CLI, which will evolve over time. If needed there will also be an option to add packages to this list in the future, but it is not included yet to avoid clutter. + +To counteract the slowdown of the additional transforms that have been introduced, the default configuration has also been reworked to enable caching across different packages. Previously each package in a Backstage monorepo would have its own isolated Jest cache, but it is now shared between packages that have a similar enough Jest configuration. + +Another change that will speed up test execution is that the transformer for `.esm.js` files has been switched. It used to be an ESM transformer based on Babel, but it is also done by sucrase now since it is significantly faster. + +The changes above are not strictly breaking as all tests should still work. It may however cause excessive slowdowns in projects that have configured custom transforms in the `jest` field within `package.json` files. In this case it is either best to consider removing the custom transforms, or overriding the `transformIgnorePatterns` to instead use Jest's default `'/node_modules/'` pattern. diff --git a/package.json b/package.json index 9c0f41ca53..4aa6905085 100644 --- a/package.json +++ b/package.json @@ -85,10 +85,5 @@ "*.md": [ "node ./scripts/check-docs-quality" ] - }, - "jest": { - "transformModules": [ - "@asyncapi/react-component" - ] } } diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index c687616a7d..e889d23c3a 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -16,7 +16,25 @@ const fs = require('fs-extra'); const path = require('path'); +const crypto = require('crypto'); const glob = require('util').promisify(require('glob')); +const { version } = require('../package.json'); + +const transformIgnorePattern = [ + '@material-ui', + '@rjsf', + 'ajv', + 'core-js', + 'jest-.*', + 'jsdom', + 'knex', + 'react', + 'react-dom', + 'highlight\\.js', + 'prismjs', + 'react-use', + 'typescript', +].join('|'); async function getProjectConfig(targetPath, displayName) { const configJsPath = path.resolve(targetPath, 'jest.config.js'); @@ -58,10 +76,7 @@ async function getProjectConfig(targetPath, displayName) { currentPath = newPath; } - // We add an additional Jest config parameter only known by the Backstage CLI - // called `transformModules`. It's a list of modules that we want to apply - // our configured jest transformations for. - // This is useful when packages are published in untranspiled ESM or TS form. + // This is an old deprecated option that is no longer used. const transformModules = pkgJsonConfigs .flatMap(conf => { const modules = conf.transformModules || []; @@ -70,10 +85,14 @@ async function getProjectConfig(targetPath, displayName) { }) .map(name => `${name}/`) .join('|'); - const transformModulePattern = transformModules && `(?!${transformModules})`; + if (transformModules.length > 0) { + console.warn( + 'The Backstage CLI jest transformModules option is no longer used and will be ignored. All modules are now always transformed.', + ); + } const options = { - displayName, + ...(displayName && { displayName }), rootDir: path.resolve(targetPath, 'src'), coverageDirectory: path.resolve(targetPath, 'coverage'), collectCoverageFrom: ['**/*.{js,jsx,ts,tsx}', '!**/*.d.ts'], @@ -82,8 +101,7 @@ async function getProjectConfig(targetPath, displayName) { }, transform: { - '\\.esm\\.js$': require.resolve('./jestEsmTransform.js'), // See jestEsmTransform.js - '\\.(js|jsx|ts|tsx)$': require.resolve('@sucrase/jest-plugin'), + '\\.(js|jsx|ts|tsx)$': require.resolve('./jestSucraseTransform.js'), '\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg|eot|woff|woff2|ttf)$': require.resolve('./jestFileTransform.js'), '\\.(yaml)$': require.resolve('jest-transform-yaml'), @@ -92,11 +110,7 @@ async function getProjectConfig(targetPath, displayName) { // A bit more opinionated testMatch: ['**/?(*.)test.{js,jsx,mjs,ts,tsx}'], - // Default behaviour is to not apply transforms for node_modules, but we still want - // to apply the esm-transformer to .esm.js files, since that's what we use in backstage packages. - transformIgnorePatterns: [ - `/node_modules/${transformModulePattern}.*\\.(?:(?/setupTests.ts']; } - return Object.assign(options, ...pkgJsonConfigs); + const config = Object.assign(options, ...pkgJsonConfigs); + + // The config name is a cache key that lets us share the jest cache across projects. + // If no explicit name was configured, generated one based on the configuration. + if (!config.name) { + const configHash = crypto + .createHash('md5') + .update(version) + .update(Buffer.alloc(1)) + .update(JSON.stringify(config.transform)) + .digest('hex'); + config.name = `backstage_cli_${configHash}`; + } + + return config; } // This loads the root jest config, which in turn will either refer to a single diff --git a/packages/cli/config/jestSucraseTransform.js b/packages/cli/config/jestSucraseTransform.js new file mode 100644 index 0000000000..01acfee38d --- /dev/null +++ b/packages/cli/config/jestSucraseTransform.js @@ -0,0 +1,66 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { createHash } = require('crypto'); +const { transform } = require('sucrase'); +const sucrasePkg = require('sucrase/package.json'); +const sucrasePluginPkg = require('@sucrase/jest-plugin/package.json'); + +const ESM_REGEX = /\b(?:import|export)\b/; + +function process(source, filePath) { + let transforms; + + if (filePath.endsWith('.esm.js')) { + transforms = ['imports']; + } else if (filePath.endsWith('.js')) { + // This is a very rough filter to avoid transforming things that we quickly + // can be sure are definitely not ESM modules. + if (ESM_REGEX.test(source)) { + transforms = ['imports', 'jsx']; // JSX within .js is currently allowed + } + } else if (filePath.endsWith('.jsx')) { + transforms = ['jsx', 'imports']; + } else if (filePath.endsWith('.ts')) { + transforms = ['typescript', 'imports']; + } else if (filePath.endsWith('.tsx')) { + transforms = ['typescript', 'jsx', 'imports']; + } + + // Only apply the jest transform to the test files themselves + if (transforms && filePath.includes('.test.')) { + transforms.push('jest'); + } + + if (transforms) { + return transform(source, { transforms, filePath }).code; + } + + return source; +} + +// TODO: contribute something like this to @sucrase/jest-plugin +function getCacheKey(sourceText) { + return createHash('md5') + .update(sourceText) + .update(Buffer.alloc(1)) + .update(sucrasePkg.version) + .update(Buffer.alloc(1)) + .update(sucrasePluginPkg.version) + .digest('hex'); +} + +module.exports = { process, getCacheKey }; diff --git a/packages/create-app/templates/default-app/package.json.hbs b/packages/create-app/templates/default-app/package.json.hbs index 7e2702e213..5ed066a30b 100644 --- a/packages/create-app/templates/default-app/package.json.hbs +++ b/packages/create-app/templates/default-app/package.json.hbs @@ -49,10 +49,5 @@ "*.{json,md}": [ "prettier --write" ] - }, - "jest": { - "transformModules": [ - "@asyncapi/react-component" - ] } } From 4050e33f039c984be6d0a6edccb8f5d2fbcbadbd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 12 Oct 2021 18:01:56 +0200 Subject: [PATCH 3/3] cli: remove old jest ESM transform Signed-off-by: Patrik Oldsberg --- .changeset/wise-camels-run.md | 2 ++ packages/cli/config/jestEsmTransform.js | 36 ------------------------- packages/cli/package.json | 3 --- yarn.lock | 4 +-- 4 files changed, 4 insertions(+), 41 deletions(-) delete mode 100644 packages/cli/config/jestEsmTransform.js diff --git a/.changeset/wise-camels-run.md b/.changeset/wise-camels-run.md index 1e81e2f9ac..4438032b77 100644 --- a/.changeset/wise-camels-run.md +++ b/.changeset/wise-camels-run.md @@ -11,3 +11,5 @@ To counteract the slowdown of the additional transforms that have been introduce Another change that will speed up test execution is that the transformer for `.esm.js` files has been switched. It used to be an ESM transformer based on Babel, but it is also done by sucrase now since it is significantly faster. The changes above are not strictly breaking as all tests should still work. It may however cause excessive slowdowns in projects that have configured custom transforms in the `jest` field within `package.json` files. In this case it is either best to consider removing the custom transforms, or overriding the `transformIgnorePatterns` to instead use Jest's default `'/node_modules/'` pattern. + +This change also removes the `@backstage/cli/config/jestEsmTransform.js` transform, which can be replaced by using the `@backstage/cli/config/sucraseEsmTransform.js` transform instead. diff --git a/packages/cli/config/jestEsmTransform.js b/packages/cli/config/jestEsmTransform.js deleted file mode 100644 index 742822274d..0000000000 --- a/packages/cli/config/jestEsmTransform.js +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const babel = require('@babel/core'); - -// We build .esm.js files with plugin:build, so to be able to load these in tests they need to be transformed -// TODO: jest is working on module support, it's possible that we can remove this in the future -module.exports = { - process(src) { - const result = babel.transform(src, { - babelrc: false, - compact: false, - plugins: [ - // This transforms the regular ESM syntax, import and export statements - require.resolve('@babel/plugin-transform-modules-commonjs'), - // This transforms dynamic `import()`, which is not supported yet in the Node.js VM API - require.resolve('babel-plugin-dynamic-import-node'), - ], - }); - - return result.code; - }, -}; diff --git a/packages/cli/package.json b/packages/cli/package.json index 53c1b06e76..426c4fd1ac 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -28,8 +28,6 @@ "backstage-cli": "bin/backstage-cli" }, "dependencies": { - "@babel/core": "^7.4.4", - "@babel/plugin-transform-modules-commonjs": "^7.4.4", "@backstage/cli-common": "^0.1.4", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.6.10", @@ -54,7 +52,6 @@ "@typescript-eslint/eslint-plugin": "^v4.30.0", "@typescript-eslint/parser": "^v4.28.3", "@yarnpkg/lockfile": "^1.1.0", - "babel-plugin-dynamic-import-node": "^2.3.3", "bfj": "^7.0.2", "buffer": "^6.0.3", "chalk": "^4.0.0", diff --git a/yarn.lock b/yarn.lock index 1a0caad7e9..521628b019 100644 --- a/yarn.lock +++ b/yarn.lock @@ -375,7 +375,7 @@ semver "^5.4.1" source-map "^0.5.0" -"@babel/core@^7.0.0", "@babel/core@^7.1.0", "@babel/core@^7.12.10", "@babel/core@^7.12.3", "@babel/core@^7.13.16", "@babel/core@^7.4.4", "@babel/core@^7.7.5": +"@babel/core@^7.0.0", "@babel/core@^7.1.0", "@babel/core@^7.12.10", "@babel/core@^7.12.3", "@babel/core@^7.13.16", "@babel/core@^7.7.5": version "7.14.8" resolved "https://registry.npmjs.org/@babel/core/-/core-7.14.8.tgz#20cdf7c84b5d86d83fac8710a8bc605a7ba3f010" integrity sha512-/AtaeEhT6ErpDhInbXmjHcUQXH0L0TEgscfcxk1qbOvLuKCa5aZT0SOOtDKFY96/CLROwbLSKyFor6idgNaU4Q== @@ -1536,7 +1536,7 @@ "@babel/helper-plugin-utils" "^7.14.5" babel-plugin-dynamic-import-node "^2.3.3" -"@babel/plugin-transform-modules-commonjs@^7.0.0", "@babel/plugin-transform-modules-commonjs@^7.13.8", "@babel/plugin-transform-modules-commonjs@^7.14.0", "@babel/plugin-transform-modules-commonjs@^7.14.5", "@babel/plugin-transform-modules-commonjs@^7.4.4": +"@babel/plugin-transform-modules-commonjs@^7.0.0", "@babel/plugin-transform-modules-commonjs@^7.13.8", "@babel/plugin-transform-modules-commonjs@^7.14.0", "@babel/plugin-transform-modules-commonjs@^7.14.5": version "7.14.5" resolved "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.14.5.tgz#7aaee0ea98283de94da98b28f8c35701429dad97" integrity sha512-en8GfBtgnydoao2PS+87mKyw62k02k7kJ9ltbKe0fXTHrQmG6QZZflYuGI1VVG7sVpx4E1n7KBpNlPb8m78J+A==