Merge pull request #1238 from spotify/shmidt-i/backend-hmr-2

Backend HMR

packages/backend-common/package.json

@@ -2,7 +2,7 @@
   "name": "@backstage/backend-common",
   "description": "Common functionality library for Backstage backends",
   "version": "0.1.1-alpha.8",
-  "main": "dist",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "private": false,
   "publishConfig": {
@@ -32,6 +32,7 @@
     "express": "^4.17.1",
     "helmet": "^3.22.0",
     "morgan": "^1.10.0",
+    "stoppable": "^1.1.0",
     "winston": "^3.2.1"
   },
   "devDependencies": {
@@ -41,7 +42,9 @@
     "@types/express": "^4.17.6",
     "@types/http-errors": "^1.6.3",
     "@types/morgan": "^1.9.0",
+    "@types/stoppable": "^1.1.0",
     "@types/supertest": "^2.0.8",
+    "@types/webpack-env": "^1.15.2",
     "@types/yaml": "^1.9.7",
     "get-port": "^5.1.1",
     "http-errors": "^1.7.3",

packages/backend-common/src/hot.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This function allows devs to cleanup
+ * ongoing effects when module gets hot-reloaded
+ * Useful for cleaning intervals, timers, requests etc
+ * @example
+ * ```ts
+ * const intervalId = setInterval(doStuff, 1000);
+ * useHotCleanup(module, () => clearInterval(intervalId));
+ * ```
+ * @param _module Reference to the current module where you invoke the fn
+ * @param cancelEffect Fn that cleans up the ongoing effects
+ */
+export function useHotCleanup(_module: NodeModule, cancelEffect: () => void) {
+  if (_module.hot) {
+    _module.hot.addDisposeHandler(() => {
+      cancelEffect();
+    });
+  }
+}
+
+/**
+ * This function allows devs to preserve
+ * some value between hot-reloads.
+ * Useful for stateful parts of the backend
+ * @example
+ * ```ts
+ * const db = useHotMemoize(module, () => createDB(dbParams));
+ * ```
+ * @param _module Reference to the current module where you invoke the fn
+ * @param valueFactory Fn that returns the value you want to memoize
+ * @warning Don't use inside conditionals or loops,
+ * same rules as for hooks apply (https://reactjs.org/docs/hooks-rules.html)
+ */
+export function useHotMemoize<T>(
+  _module: NodeModule,
+  valueFactory: () => T,
+): T {
+  const CURRENT_HOT_MEMOIZE_INDEX_KEY = 'backstage.io/hmr-memoize-key';
+
+  if (!_module.hot) {
+    // Just return value straight away
+    return valueFactory();
+  }
+
+  if (_module.hot && typeof _module.hot.data === 'undefined') {
+    // First run, init the module data
+    _module.hot.data = {
+      [CURRENT_HOT_MEMOIZE_INDEX_KEY]: 0,
+    };
+  }
+
+  // Let's store data per module based on the order of the code invocation
+  const index = _module.hot.data?.[CURRENT_HOT_MEMOIZE_INDEX_KEY];
+  // Increasing the counter after each call
+  _module.hot.data[CURRENT_HOT_MEMOIZE_INDEX_KEY] += 1;
+
+  const prevValue = _module.hot.data?.[index];
+  const createDisposeHandler = (value: any) => (data: {
+    [key: number]: any;
+    [indexKey: string]: number;
+  }) => {
+    // Preserving the value through the HMR process
+    data[index] = value;
+    // Decreasing the counter after each handler
+    data[CURRENT_HOT_MEMOIZE_INDEX_KEY] = index - 1;
+  };
+
+  if (prevValue) {
+    _module.hot!.addDisposeHandler(createDisposeHandler(prevValue));
+    return prevValue;
+  }
+
+  const newValue = valueFactory();
+  _module.hot.addDisposeHandler(createDisposeHandler(newValue));
+  return newValue;
+}

packages/backend-common/src/index.ts

@@ -18,3 +18,4 @@ export * from './errors';
 export * from './logging';
 export * from './middleware';
 export * from './service';
+export * from './hot';

packages/backend-common/src/service/ServiceBuilderImpl.ts

@@ -19,6 +19,7 @@ import cors from 'cors';
 import express, { Router } from 'express';
 import helmet from 'helmet';
 import { Server } from 'http';
+import stoppable from 'stoppable';
 import { Logger } from 'winston';
 import { getRootLogger } from '../logging';
 import {
@@ -27,6 +28,7 @@ import {
   requestLoggingHandler,
 } from '../middleware';
 import { ServiceBuilder } from './types';
+import { useHotCleanup } from '../hot';
 
 const DEFAULT_PORT = 7000;
 
@@ -35,9 +37,14 @@ export class ServiceBuilderImpl implements ServiceBuilder {
   private logger: Logger | undefined;
   private corsOptions: cors.CorsOptions | undefined;
   private routers: [string, Router][];
-
-  constructor() {
+  /**
+   * Reference to the module where builder is created
+   * Needed for the HMR
+   */
+  private module: NodeModule;
+  constructor(module: NodeModule) {
     this.routers = [];
+    this.module = module;
   }
 
   setPort(port: number): ServiceBuilder {
@@ -82,9 +89,19 @@ export class ServiceBuilderImpl implements ServiceBuilder {
         logger.error(`Failed to start up on port ${port}, ${e}`);
         reject(e);
       });
-      const server = app.listen(port, () => {
-        logger.info(`Listening on port ${port}`);
-      });
+      const server = stoppable(
+        app.listen(port, () => {
+          logger.info(`Listening on port ${port}`);
+        }),
+        0,
+      );
+
+      useHotCleanup(this.module, () =>
+        server.stop((e: any) => {
+          if (e) console.error(e);
+        }),
+      );
+
       resolve(server);
     });
   }

packages/backend-common/src/service/createServiceBuilder.ts

@@ -19,6 +19,6 @@ import { ServiceBuilderImpl } from './ServiceBuilderImpl';
 /**
  * Creates a new service builder.
  */
-export function createServiceBuilder() {
-  return new ServiceBuilderImpl();
+export function createServiceBuilder(_module: NodeModule) {
+  return new ServiceBuilderImpl(_module);
 }

packages/backend-common/tsconfig.json

@@ -9,6 +9,6 @@
     "target": "ES2019",
     "module": "commonjs",
     "esModuleInterop": true,
-    "types": ["node", "jest"]
+    "types": ["node", "jest", "webpack-env"]
   }
 }

packages/backend/package.json

@@ -10,7 +10,7 @@
   },
   "scripts": {
     "build": "tsc",
-    "start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess \\\"cross-env NODE_ENV=development nodemon -r esm\\\"",
+    "start": "backstage-cli backend:dev",
     "lint": "backstage-cli lint",
     "test": "backstage-cli test",
     "clean": "backstage-cli clean",

packages/backend/src/index.ts

@@ -22,7 +22,11 @@
  * Happy hacking!
  */
 
-import { createServiceBuilder, getRootLogger } from '@backstage/backend-common';
+import {
+  createServiceBuilder,
+  getRootLogger,
+  useHotMemoize,
+} from '@backstage/backend-common';
 import knex from 'knex';
 import auth from './plugins/auth';
 import catalog from './plugins/catalog';
@@ -45,19 +49,24 @@ function createEnv(plugin: string): PluginEnvironment {
 }
 
 async function main() {
-  const service = createServiceBuilder()
+  const catalogEnv = useHotMemoize(module, () => createEnv('catalog'));
+  const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder'));
+  const authEnv = useHotMemoize(module, () => createEnv('auth'));
+  const identityEnv = useHotMemoize(module, () => createEnv('identity'));
+
+  const service = createServiceBuilder(module)
     .enableCors({
       origin: 'http://localhost:3000',
       credentials: true,
     })
-    .addRouter('/catalog', await catalog(createEnv('catalog')))
-    .addRouter('/scaffolder', await scaffolder(createEnv('scaffolder')))
+    .addRouter('/catalog', await catalog(catalogEnv))
+    .addRouter('/scaffolder', await scaffolder(scaffolderEnv))
     .addRouter(
       '/sentry',
       await sentry(getRootLogger().child({ type: 'plugin', plugin: 'sentry' })),
     )
-    .addRouter('/auth', await auth(createEnv('auth')))
-    .addRouter('/identity', await identity(createEnv('identity')));
+    .addRouter('/auth', await auth(authEnv))
+    .addRouter('/identity', await identity(identityEnv));
 
   await service.start().catch(err => {
     console.log(err);
@@ -65,6 +74,7 @@ async function main() {
   });
 }
 
+module.hot?.accept();
 main().catch(error => {
   console.error(`Backend failed to start up, ${error}`);
   process.exit(1);

packages/backend/src/plugins/catalog.ts

@@ -24,6 +24,7 @@ import {
   runPeriodically,
 } from '@backstage/plugin-catalog-backend';
 import { PluginEnvironment } from '../types';
+import { useHotCleanup } from '@backstage/backend-common';
 
 export default async function createPlugin({
   logger,
@@ -41,7 +42,10 @@ export default async function createPlugin({
     logger,
   );
 
-  runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000);
+  useHotCleanup(
+    module,
+    runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000),
+  );
 
   return await createRouter({
     entitiesCatalog,

packages/backend/tsconfig.json

@@ -10,6 +10,6 @@
     "module": "commonjs",
     "esModuleInterop": true,
     "lib": ["es2019", "dom"],
-    "types": ["node", "jest"]
+    "types": ["node", "jest", "webpack-env"]
   }
 }

packages/cli/config/tsconfig.json

@@ -32,6 +32,6 @@
     "strictPropertyInitialization": true,
     "stripInternal": true,
     "target": "ES2019",
-    "types": ["node", "jest"]
+    "types": ["node", "jest", "webpack-env"]
   }
 }

packages/cli/package.json

@@ -39,6 +39,9 @@
     "@rollup/plugin-node-resolve": "^7.1.1",
     "@spotify/eslint-config": "^7.0.1",
     "@sucrase/webpack-loader": "^2.0.0",
+    "@types/start-server-webpack-plugin": "^2.2.0",
+    "@types/webpack-node-externals": "^1.7.1",
+    "@types/webpack-env": "^1.15.2",
     "bfj": "^7.0.2",
     "chalk": "^4.0.0",
     "chokidar": "^3.3.1",
@@ -72,6 +75,7 @@
     "rollup-plugin-peer-deps-external": "^2.2.2",
     "rollup-plugin-postcss": "^3.1.1",
     "rollup-plugin-typescript2": "^0.26.0",
+    "start-server-webpack-plugin": "^2.2.5",
     "style-loader": "^1.2.1",
     "sucrase": "^3.14.1",
     "tar": "^6.0.1",
@@ -81,6 +85,7 @@
     "url-loader": "^4.1.0",
     "webpack": "^4.41.6",
     "webpack-dev-server": "^3.10.3",
+    "webpack-node-externals": "^1.7.2",
     "yaml": "^1.10.0",
     "yml-loader": "^2.1.0",
     "yn": "^4.0.0"

packages/cli/src/commands/backend/dev.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { loadConfig } from '@backstage/config-loader';
+import { Command } from 'commander';
+import { serveBackend } from '../../lib/bundler/backend';
+
+export default async (cmd: Command) => {
+  const appConfigs = await loadConfig();
+  const waitForExit = await serveBackend({
+    entry: 'src/index',
+    checksEnabled: cmd.check,
+    config: ConfigReader.fromConfigs(appConfigs),
+    appConfigs,
+  });
+
+  await waitForExit();
+};

packages/cli/src/index.ts

@@ -41,6 +41,12 @@ const main = (argv: string[]) => {
     .option('--check', 'Enable type checking and linting')
     .action(lazyAction(() => import('./commands/app/serve'), 'default'));
 
+  program
+    .command('backend:dev')
+    .description('Start local development server with HMR for the backend')
+    .option('--check', 'Enable type checking and linting')
+    .action(lazyAction(() => import('./commands/backend/dev'), 'default'));
+
   program
     .command('app:diff')
     .option('--check', 'Fail if changes are required')

packages/cli/src/lib/bundler/backend.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import webpack from 'webpack';
+import { createBackendConfig } from './config';
+import { resolveBundlingPaths } from './paths';
+import { ServeOptions } from './types';
+
+export async function serveBackend(options: ServeOptions) {
+  const paths = resolveBundlingPaths(options);
+  const config = createBackendConfig(paths, {
+    ...options,
+    isDev: true,
+  });
+
+  const compiler = webpack(config);
+
+  const watcher = compiler.watch(
+    {
+      poll: true,
+    },
+    (err: Error) => {
+      if (err) {
+        console.error(err);
+      } else console.log('Build succeeded');
+    },
+  );
+
+  const waitForExit = async () => {
+    for (const signal of ['SIGINT', 'SIGTERM'] as const) {
+      process.on(signal, () => {
+        watcher.close(() => console.log('Stopped watcher'));
+        // exit instead of resolve. The process is shutting down and resolving a promise here logs an error
+        process.exit();
+      });
+    }
+
+    // Block indefinitely and wait for the interrupt signal
+    return new Promise(() => {});
+  };
+
+  return waitForExit;
+}

packages/cli/src/lib/bundler/config.ts

@@ -14,15 +14,17 @@
  * limitations under the License.
  */
 
-import webpack from 'webpack';
 import ForkTsCheckerWebpackPlugin from 'fork-ts-checker-webpack-plugin';
-import ModuleScopePlugin from 'react-dev-utils/ModuleScopePlugin';
 import HtmlWebpackPlugin from 'html-webpack-plugin';
+import ModuleScopePlugin from 'react-dev-utils/ModuleScopePlugin';
+import StartServerPlugin from 'start-server-webpack-plugin';
+import webpack from 'webpack';
+import nodeExternals from 'webpack-node-externals';
+import { optimization } from './optimization';
 import { Config } from '@backstage/config';
 import { BundlingPaths } from './paths';
 import { transforms } from './transforms';
-import { optimization } from './optimization';
-import { BundlingOptions } from './types';
+import { BundlingOptions, BackendBundlingOptions } from './types';
 // import checkRequiredFiles from 'react-dev-utils/checkRequiredFiles';
 // import ModuleNotFoundPlugin from 'react-dev-utils/ModuleNotFoundPlugin';
 // import errorOverlayMiddleware from 'react-dev-utils/errorOverlayMiddleware';
@@ -93,6 +95,17 @@ export function createConfig(
   return {
     mode: isDev ? 'development' : 'production',
     profile: false,
+    node: {
+      module: 'empty',
+      dgram: 'empty',
+      dns: 'mock',
+      fs: 'empty',
+      http2: 'empty',
+      net: 'empty',
+      tls: 'empty',
+      child_process: 'empty',
+    },
+    optimization: optimization(options),
     bail: false,
     performance: {
       hints: false, // we check the gzip size instead
@@ -124,17 +137,94 @@ export function createConfig(
         ? '[name].chunk.js'
         : '[name].[chunkhash:8].chunk.js',
     },
-    optimization: optimization(options),
     plugins,
-    node: {
-      module: 'empty',
-      dgram: 'empty',
-      dns: 'mock',
-      fs: 'empty',
-      http2: 'empty',
-      net: 'empty',
-      tls: 'empty',
-      child_process: 'empty',
-    },
+  };
+}
+
+export function createBackendConfig(
+  paths: BundlingPaths,
+  options: BackendBundlingOptions,
+): webpack.Configuration {
+  const { checksEnabled, isDev } = options;
+
+  const { loaders } = transforms(options);
+
+  return {
+    mode: isDev ? 'development' : 'production',
+    profile: false,
+    ...(isDev
+      ? {
+          watch: true,
+          watchOptions: {
+            ignored: [/node_modules\/(?!\@backstage)/],
+          },
+        }
+      : {}),
+    externals: [
+      nodeExternals({
+        modulesDir: paths.rootNodeModules,
+        whitelist: ['webpack/hot/poll?100', /\@backstage\/.*/],
+      }),
+      nodeExternals({
+        modulesDir: paths.targetNodeModules,
+        whitelist: ['webpack/hot/poll?100', /\@backstage\/.*/],
+      }),
+    ],
+    target: 'node' as const,
+    node: {
+      __dirname: true,
+      __filename: true,
+      global: true,
+    },
+    bail: false,
+    performance: {
+      hints: false, // we check the gzip size instead
+    },
+    devtool: isDev ? 'cheap-module-eval-source-map' : 'source-map',
+    context: paths.targetPath,
+    entry: ['webpack/hot/poll?100', paths.targetEntry],
+    resolve: {
+      extensions: ['.ts', '.tsx', '.mjs', '.js', '.jsx'],
+      mainFields: ['main:src', 'browser', 'module', 'main'],
+      modules: [paths.targetNodeModules, paths.rootNodeModules],
+      plugins: [
+        new ModuleScopePlugin(
+          [paths.targetSrc, paths.targetDev],
+          [paths.targetPackageJson],
+        ),
+      ],
+      alias: {
+        'react-dom': '@hot-loader/react-dom',
+      },
+    },
+    module: {
+      rules: loaders,
+    },
+    output: {
+      path: paths.targetDist,
+      filename: isDev ? '[name].js' : '[name].[hash:8].js',
+      chunkFilename: isDev
+        ? '[name].chunk.js'
+        : '[name].[chunkhash:8].chunk.js',
+    },
+    plugins: [
+      new StartServerPlugin('main.js'),
+      new webpack.HotModuleReplacementPlugin(),
+      ...(checksEnabled
+        ? [
+            new ForkTsCheckerWebpackPlugin({
+              tsconfig: paths.targetTsConfig,
+              eslint: true,
+              eslintOptions: {
+                parserOptions: {
+                  project: paths.targetTsConfig,
+                  tsconfigRootDir: paths.targetPath,
+                },
+              },
+              reportFiles: ['**', '!**/__tests__/**', '!**/?(*.)(spec|test).*'],
+            }),
+          ]
+        : []),
+    ],
   };
 }

packages/cli/src/lib/bundler/index.ts

@@ -14,5 +14,6 @@
  * limitations under the License.
  */
 
+export { serveBackend } from './backend';
 export { buildBundle } from './bundle';
 export { serveBundle } from './server';

packages/cli/src/lib/bundler/paths.ts

@@ -60,6 +60,8 @@ export function resolveBundlingPaths(options: BundlingPathsOptions) {
     targetTsConfig: paths.resolveTargetRoot('tsconfig.json'),
     targetNodeModules: paths.resolveTarget('node_modules'),
     targetPackageJson: paths.resolveTarget('package.json'),
+    rootNodeModules: paths.resolveTargetRoot('node_modules'),
+    root: paths.targetRoot,
   };
 }
 

packages/cli/src/lib/bundler/server.ts

@@ -30,7 +30,11 @@ export async function serveBundle(options: ServeOptions) {
   const paths = resolveBundlingPaths(options);
   const pkgPath = paths.targetPackageJson;
   const pkg = await fs.readJson(pkgPath);
-  const config = createConfig(paths, { ...options, isDev: true, baseUrl: url });
+  const config = createConfig(paths, {
+    ...options,
+    isDev: true,
+    baseUrl: url,
+  });
   const compiler = webpack(config);
 
   const server = new WebpackDevServer(compiler, {

packages/cli/src/lib/bundler/transforms.ts

@@ -16,14 +16,16 @@
 
 import webpack, { Module, Plugin } from 'webpack';
 import MiniCssExtractPlugin from 'mini-css-extract-plugin';
-import { BundlingOptions } from './types';
+import { BundlingOptions, BackendBundlingOptions } from './types';
 
 type Transforms = {
   loaders: Module['rules'];
   plugins: Plugin[];
 };
 
-export const transforms = (options: BundlingOptions): Transforms => {
+export const transforms = (
+  options: BundlingOptions | BackendBundlingOptions,
+): Transforms => {
   const { isDev } = options;
 
   const loaders = [

packages/cli/src/lib/bundler/types.ts

@@ -25,6 +25,8 @@ export type BundlingOptions = {
   baseUrl: URL;
 };
 
+export type BackendBundlingOptions = Omit<BundlingOptions, 'baseUrl'>;
+
 export type ServeOptions = BundlingPathsOptions & {
   checksEnabled: boolean;
   config: Config;