Merge pull request #1458 from spotify/mob/pack

cli: add backend:build-image command and use in backend

packages/backend/Dockerfile

@@ -0,0 +1,9 @@
+FROM node:12
+
+WORKDIR /usr/src/app
+
+COPY . .
+
+RUN yarn install --frozen-lockfile --production
+
+CMD ["node", "packages/backend"]

packages/backend/package.json

@@ -10,6 +10,7 @@
   },
   "scripts": {
     "build": "backstage-cli backend:build",
+    "build-image": "backstage-cli backend:build-image example-backend",
     "start": "backstage-cli backend:dev",
     "lint": "backstage-cli lint",
     "test": "backstage-cli test",

packages/cli/src/commands/backend/build.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { buildPackage, Output } from '../../lib/packager';
+import { buildPackage, Output } from '../../lib/builder';
 
 export default async () => {
   await buildPackage({

packages/cli/src/commands/backend/buildImage.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import { createDistWorkspace } from '../../lib/packager';
+import { paths } from '../../lib/paths';
+import { run } from '../../lib/run';
+
+export default async (imageTag: string) => {
+  const tempDistWorkspace = await createDistWorkspace(['example-backend'], {
+    files: [
+      'package.json',
+      'yarn.lock',
+      'app-config.yaml',
+      { src: paths.resolveTarget('Dockerfile'), dest: 'Dockerfile' },
+    ],
+  });
+
+  console.log(`Dist workspace ready at ${tempDistWorkspace}`);
+
+  await run('docker', ['build', '.', '-t', imageTag], {
+    cwd: tempDistWorkspace,
+  });
+
+  await fs.remove(tempDistWorkspace);
+};

packages/cli/src/commands/build.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { buildPackage, Output } from '../lib/packager';
+import { buildPackage, Output } from '../lib/builder';
 import { Command } from 'commander';
 
 export default async (cmd: Command) => {

packages/cli/src/commands/pack.ts

@@ -19,10 +19,15 @@ import { paths } from '../lib/paths';
 
 const SKIPPED_KEYS = ['access', 'registry', 'tag'];
 
-export const pre = async () => {
-  const pkgPath = paths.resolveTarget('package.json');
+const PKG_PATH = 'package.json';
+const PKG_BACKUP_PATH = 'package.json-prepack';
 
-  const pkg = await fs.readJson(pkgPath);
+export const pre = async () => {
+  const pkgPath = paths.resolveTarget(PKG_PATH);
+
+  const pkgContent = await fs.readFile(pkgPath, 'utf8');
+  const pkg = JSON.parse(pkgContent);
+  await fs.writeFile(PKG_BACKUP_PATH, pkgContent);
 
   for (const key of Object.keys(pkg.publishConfig ?? {})) {
     if (!SKIPPED_KEYS.includes(key)) {
@@ -33,5 +38,6 @@ export const pre = async () => {
 };
 
 export const post = async () => {
-  // postpack is a noop for now, since it's not called anyway
+  // postpack isn't called by yarn right now, so it needs to be called manually
+  await fs.move(PKG_BACKUP_PATH, PKG_PATH, { overwrite: true });
 };

packages/cli/src/commands/plugin/build.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { buildPackage, Output } from '../../lib/packager';
+import { buildPackage, Output } from '../../lib/builder';
 
 export default async () => {
   await buildPackage({

packages/cli/src/index.ts

@@ -46,6 +46,15 @@ const main = (argv: string[]) => {
     .description('Build a backend plugin')
     .action(lazyAction(() => import('./commands/backend/build'), 'default'));
 
+  program
+    .command('backend:build-image <image-tag>')
+    .description(
+      'Builds a docker image from the package, with all local deps included',
+    )
+    .action(
+      lazyAction(() => import('./commands/backend/buildImage'), 'default'),
+    );
+
   program
     .command('backend:dev')
     .description('Start local development server with HMR for the backend')

packages/cli/src/lib/builder/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { buildPackage } from './packager';
+export { Output } from './types';
+export type { BuildOptions } from './types';

packages/cli/src/lib/packager/index.ts

@@ -14,6 +14,133 @@
  * limitations under the License.
  */
 
-export { buildPackage } from './packager';
-export { Output } from './types';
-export type { BuildOptions } from './types';
+import fs from 'fs-extra';
+import { resolve as resolvePath, relative as relativePath } from 'path';
+import { paths } from '../paths';
+import { run } from '../run';
+import tar from 'tar';
+import { tmpdir } from 'os';
+
+type LernaPackage = {
+  name: string;
+  private: boolean;
+  location: string;
+  scripts: Record<string, string>;
+};
+
+type FileEntry =
+  | string
+  | {
+      src: string;
+      dest: string;
+    };
+
+type Options = {
+  /**
+   * Target directory for the dist workspace, defaults to a temporary directory
+   */
+  targetDir?: string;
+
+  /**
+   * Files to copy into the target workspace.
+   *
+   * Defaults to ['yarn.lock', 'package.json'].
+   */
+  files?: FileEntry[];
+};
+
+/**
+ * Uses `yarn pack` to package local packages and unpacks them into a dist workspace.
+ * The target workspace will end up containing dist version of each package and
+ * will be suitable for packaging e.g. into a docker image.
+ *
+ * This creates a structure that is functionally similar to if the packages where
+ * installed from NPM, but uses yarn workspaces to link to them at runtime.
+ */
+export async function createDistWorkspace(
+  packageNames: string[],
+  options: Options,
+) {
+  const targetDir =
+    options.targetDir ??
+    (await fs.mkdtemp(resolvePath(tmpdir(), 'dist-workspace')));
+
+  const targets = await findTargetPackages(packageNames);
+
+  await moveToDistWorkspace(targetDir, targets);
+
+  const files: FileEntry[] = options.files ?? ['yarn.lock', 'package.json'];
+
+  for (const file of files) {
+    const src = typeof file === 'string' ? file : file.src;
+    const dest = typeof file === 'string' ? file : file.dest;
+    await fs.copy(paths.resolveTargetRoot(src), resolvePath(targetDir, dest));
+  }
+  return targetDir;
+}
+
+async function moveToDistWorkspace(
+  workspaceDir: string,
+  localPackages: LernaPackage[],
+): Promise<void> {
+  await Promise.all(
+    localPackages.map(async (target, index) => {
+      console.log(`Repacking ${target.name} into dist workspace`);
+      const archive = `temp-package-${index}.tgz`;
+      const archivePath = resolvePath(workspaceDir, archive);
+
+      await run('yarn', ['pack', '--filename', archivePath], {
+        cwd: target.location,
+      });
+      // TODO(Rugvip): yarn pack doesn't call postpack, once the bug is fixed this can be removed
+      if (target.scripts.postpack) {
+        await run('yarn', ['postpack'], { cwd: target.location });
+      }
+
+      const outputDir = relativePath(paths.targetRoot, target.location);
+      const absoluteOutputPath = resolvePath(workspaceDir, outputDir);
+      await fs.ensureDir(absoluteOutputPath);
+
+      await tar.extract({
+        file: archivePath,
+        cwd: absoluteOutputPath,
+        strip: 1,
+      });
+      await fs.remove(archivePath);
+    }),
+  );
+}
+
+async function findTargetPackages(pkgNames: string[]): Promise<LernaPackage[]> {
+  const LernaProject = require('@lerna/project');
+  const PackageGraph = require('@lerna/package-graph');
+
+  const project = new LernaProject(paths.targetDir);
+  const packages = await project.getPackages();
+  const graph = new PackageGraph(packages);
+
+  const targets = new Map<string, any>();
+  const searchNames = pkgNames.slice();
+
+  while (searchNames.length) {
+    const name = searchNames.pop()!;
+
+    if (targets.has(name)) {
+      continue;
+    }
+
+    const node = graph.get(name);
+    if (!node) {
+      throw new Error(`Package '${name}' not found`);
+    }
+
+    const pkgDeps = Object.keys(node.pkg.dependencies);
+    const localDeps: string[] = Array.from(node.localDependencies.keys());
+    const filteredDeps = localDeps.filter(dep => pkgDeps.includes(dep));
+
+    searchNames.push(...filteredDeps);
+    targets.set(name, node.pkg);
+  }
+
+  return Array.from(targets.values());
+}

plugins/auth-backend/package.json

@@ -59,6 +59,7 @@
     "jest-fetch-mock": "^3.0.3"
   },
   "files": [
-    "dist"
+    "dist",
+    "migrations"
   ]
 }