From e73f3fbab7c221e4b99c20d9b08b7a3de340878c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 25 Jun 2020 10:58:33 +0200 Subject: [PATCH 1/7] cli: implement postpack --- packages/cli/src/commands/pack.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/commands/pack.ts b/packages/cli/src/commands/pack.ts index 51c7e36086..1f81b89767 100644 --- a/packages/cli/src/commands/pack.ts +++ b/packages/cli/src/commands/pack.ts @@ -19,10 +19,15 @@ import { paths } from '../lib/paths'; const SKIPPED_KEYS = ['access', 'registry', 'tag']; -export const pre = async () => { - const pkgPath = paths.resolveTarget('package.json'); +const PKG_PATH = 'package.json'; +const PKG_BACKUP_PATH = 'package.json-prepack'; - const pkg = await fs.readJson(pkgPath); +export const pre = async () => { + const pkgPath = paths.resolveTarget(PKG_PATH); + + const pkgContent = await fs.readFile(pkgPath, 'utf8'); + const pkg = JSON.parse(pkgContent); + await fs.writeFile(PKG_BACKUP_PATH, pkgContent); for (const key of Object.keys(pkg.publishConfig ?? {})) { if (!SKIPPED_KEYS.includes(key)) { @@ -33,5 +38,6 @@ export const pre = async () => { }; export const post = async () => { - // postpack is a noop for now, since it's not called anyway + // postpack isn't called by yarn right now, so it needs to be called manually + await fs.move(PKG_BACKUP_PATH, PKG_PATH, { overwrite: true }); }; From c9a1b99af21f8caf61084c84eb547f66a9ebd87a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 25 Jun 2020 11:39:28 +0200 Subject: [PATCH 2/7] cli: rename packager to builder --- packages/cli/src/commands/backend/build.ts | 2 +- packages/cli/src/commands/build.ts | 2 +- packages/cli/src/commands/plugin/build.ts | 2 +- packages/cli/src/lib/{packager => builder}/config.ts | 0 packages/cli/src/lib/{packager => builder}/index.ts | 0 packages/cli/src/lib/{packager => builder}/packager.ts | 0 packages/cli/src/lib/{packager => builder}/types.ts | 0 7 files changed, 3 insertions(+), 3 deletions(-) rename packages/cli/src/lib/{packager => builder}/config.ts (100%) rename packages/cli/src/lib/{packager => builder}/index.ts (100%) rename packages/cli/src/lib/{packager => builder}/packager.ts (100%) rename packages/cli/src/lib/{packager => builder}/types.ts (100%) diff --git a/packages/cli/src/commands/backend/build.ts b/packages/cli/src/commands/backend/build.ts index a9cdc51fa9..ceca5b0286 100644 --- a/packages/cli/src/commands/backend/build.ts +++ b/packages/cli/src/commands/backend/build.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { buildPackage, Output } from '../../lib/packager'; +import { buildPackage, Output } from '../../lib/builder'; export default async () => { await buildPackage({ diff --git a/packages/cli/src/commands/build.ts b/packages/cli/src/commands/build.ts index 19e4b17352..bd5bbc5e9f 100644 --- a/packages/cli/src/commands/build.ts +++ b/packages/cli/src/commands/build.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { buildPackage, Output } from '../lib/packager'; +import { buildPackage, Output } from '../lib/builder'; import { Command } from 'commander'; export default async (cmd: Command) => { diff --git a/packages/cli/src/commands/plugin/build.ts b/packages/cli/src/commands/plugin/build.ts index 7e4e5cd36a..d62ffbeebd 100644 --- a/packages/cli/src/commands/plugin/build.ts +++ b/packages/cli/src/commands/plugin/build.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { buildPackage, Output } from '../../lib/packager'; +import { buildPackage, Output } from '../../lib/builder'; export default async () => { await buildPackage({ diff --git a/packages/cli/src/lib/packager/config.ts b/packages/cli/src/lib/builder/config.ts similarity index 100% rename from packages/cli/src/lib/packager/config.ts rename to packages/cli/src/lib/builder/config.ts diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/builder/index.ts similarity index 100% rename from packages/cli/src/lib/packager/index.ts rename to packages/cli/src/lib/builder/index.ts diff --git a/packages/cli/src/lib/packager/packager.ts b/packages/cli/src/lib/builder/packager.ts similarity index 100% rename from packages/cli/src/lib/packager/packager.ts rename to packages/cli/src/lib/builder/packager.ts diff --git a/packages/cli/src/lib/packager/types.ts b/packages/cli/src/lib/builder/types.ts similarity index 100% rename from packages/cli/src/lib/packager/types.ts rename to packages/cli/src/lib/builder/types.ts From d286e3b93fa597283e75ab2c86d5764e51772b56 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 25 Jun 2020 17:15:23 +0200 Subject: [PATCH 3/7] cli: add packager lib for packaging packages into a dist workspace Co-authored-by: Ivan Shmidt Co-authored-by: Raghunandan --- packages/cli/src/lib/packager/index.ts | 142 +++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 packages/cli/src/lib/packager/index.ts diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/packager/index.ts new file mode 100644 index 0000000000..c702fd3df4 --- /dev/null +++ b/packages/cli/src/lib/packager/index.ts @@ -0,0 +1,142 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, relative as relativePath } from 'path'; +import { paths } from '../paths'; +import { run } from '../run'; +import tar from 'tar'; +import { tmpdir } from 'os'; + +type LernaPackage = { + name: string; + private: boolean; + location: string; +}; + +type FileEntry = + | string + | { + src: string; + dest: string; + }; + +type Options = { + /** + * Target directory for the dist workspace, defaults to a temporary directory + */ + targetDir?: string; + + /** + * Files to copy into the target workspace. + * + * Defaults to ['yarn.lock', 'package.json']. + */ + files?: FileEntry[]; +}; + +/** + * Uses `yarn pack` to package local packages and unpacks them into a dist workspace. + * The target workspace will end up containing dist version of each package and + * will be suitable for packaging e.g. into a docker image. + * + * This creates a structure that is functionally similar to if the packages where + * installed from NPM, but uses yarn workspaces to link to them at runtime. + */ +export async function createDistWorkspace( + packageNames: string[], + options: Options, +) { + const targetDir = + options.targetDir ?? + (await fs.mkdtemp(resolvePath(tmpdir(), 'dist-workspace'))); + + const targets = await findTargetPackages(packageNames); + + await moveToDistWorkspace(targetDir, targets); + + const files: FileEntry[] = options.files ?? ['yarn.lock', 'package.json']; + + for (const file of files) { + const src = typeof file === 'string' ? file : file.src; + const dest = typeof file === 'string' ? file : file.dest; + await fs.copy(paths.resolveTargetRoot(src), resolvePath(targetDir, dest)); + } +} + +async function moveToDistWorkspace( + workspaceDir: string, + localPackages: LernaPackage[], +): Promise { + await Promise.all( + localPackages.map(async (target, index) => { + console.log(`Repacking ${target.name} into dist workspace`); + const archive = `temp-package-${index}.tgz`; + const archivePath = resolvePath(workspaceDir, archive); + + await run('yarn', ['pack', '--filename', archivePath], { + cwd: target.location, + }); + // TODO(Rugvip): yarn pack doesn't call postpack, once the bug is fixed this can be removed + await run('yarn', ['postpack'], { cwd: target.location }); + + const outputDir = relativePath(paths.targetRoot, target.location); + const absoluteOutputPath = resolvePath(workspaceDir, outputDir); + await fs.ensureDir(absoluteOutputPath); + + await tar.extract({ + file: archivePath, + cwd: absoluteOutputPath, + strip: 1, + }); + await fs.remove(archivePath); + }), + ); +} + +async function findTargetPackages(pkgNames: string[]): Promise { + const LernaProject = require('@lerna/project'); + const PackageGraph = require('@lerna/package-graph'); + + const project = new LernaProject(paths.targetDir); + const packages = await project.getPackages(); + const graph = new PackageGraph(packages); + + const targets = new Map(); + const searchNames = pkgNames.slice(); + + while (searchNames.length) { + const name = searchNames.pop()!; + + if (targets.has(name)) { + continue; + } + + const node = graph.get(name); + if (!node) { + throw new Error(`Package '${name}' not found`); + } + + const pkgDeps = Object.keys(node.pkg.dependencies); + const localDeps: string[] = Array.from(node.localDependencies.keys()); + const filteredDeps = localDeps.filter(dep => pkgDeps.includes(dep)); + + searchNames.push(...filteredDeps); + targets.set(name, node.pkg); + } + + return Array.from(targets.values()); +} From 84b55c10ca9363a7249b265766a85f8876da59b0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 25 Jun 2020 18:03:21 +0200 Subject: [PATCH 4/7] cli: added backend:build-image command Co-authored-by: Ivan Shmidt --- .../cli/src/commands/backend/buildImage.ts | 39 +++++++++++++++++++ packages/cli/src/index.ts | 9 +++++ packages/cli/src/lib/packager/index.ts | 1 + 3 files changed, 49 insertions(+) create mode 100644 packages/cli/src/commands/backend/buildImage.ts diff --git a/packages/cli/src/commands/backend/buildImage.ts b/packages/cli/src/commands/backend/buildImage.ts new file mode 100644 index 0000000000..a847f1ef54 --- /dev/null +++ b/packages/cli/src/commands/backend/buildImage.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { createDistWorkspace } from '../../lib/packager'; +import { paths } from '../../lib/paths'; +import { run } from '../../lib/run'; + +export default async (imageTag: string) => { + const tempDistWorkspace = await createDistWorkspace(['example-backend'], { + files: [ + 'package.json', + 'yarn.lock', + 'app-config.yaml', + { src: paths.resolveTarget('Dockerfile'), dest: 'Dockerfile' }, + ], + }); + + console.log(`Dist workspace ready at ${tempDistWorkspace}`); + + await run('docker', ['build', '.', '-t', imageTag], { + cwd: tempDistWorkspace, + }); + + await fs.remove(tempDistWorkspace); +}; diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index e67a52c795..bed32bcf8b 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -46,6 +46,15 @@ const main = (argv: string[]) => { .description('Build a backend plugin') .action(lazyAction(() => import('./commands/backend/build'), 'default')); + program + .command('backend:build-image ') + .description( + 'Builds a docker image from the package, with all local deps included', + ) + .action( + lazyAction(() => import('./commands/backend/buildImage'), 'default'), + ); + program .command('backend:dev') .description('Start local development server with HMR for the backend') diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/packager/index.ts index c702fd3df4..2f5233f93f 100644 --- a/packages/cli/src/lib/packager/index.ts +++ b/packages/cli/src/lib/packager/index.ts @@ -75,6 +75,7 @@ export async function createDistWorkspace( const dest = typeof file === 'string' ? file : file.dest; await fs.copy(paths.resolveTargetRoot(src), resolvePath(targetDir, dest)); } + return targetDir; } async function moveToDistWorkspace( From 66e2bfe073710bd3ac34c0fa0704c2f8523cd2bf Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 25 Jun 2020 18:05:11 +0200 Subject: [PATCH 5/7] backend: update packaging and add build-image command + Dockerfile Co-authored-by: Ivan Shmidt --- packages/backend/Dockerfile | 9 +++++++++ packages/backend/package.json | 10 +++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 packages/backend/Dockerfile diff --git a/packages/backend/Dockerfile b/packages/backend/Dockerfile new file mode 100644 index 0000000000..3e8ba36cec --- /dev/null +++ b/packages/backend/Dockerfile @@ -0,0 +1,9 @@ +FROM node:12 + +WORKDIR /usr/src/app + +COPY . . + +RUN yarn install --frozen-lockfile --production + +CMD ["node", "packages/backend"] diff --git a/packages/backend/package.json b/packages/backend/package.json index 3be1f5ee3e..53b1f431c0 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -1,19 +1,27 @@ { "name": "example-backend", "version": "0.1.1-alpha.12", - "main": "dist/index.cjs.js", + "main": "src/index.ts", "types": "src/index.ts", "private": true, "license": "Apache-2.0", "engines": { "node": ">=12" }, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, "scripts": { "build": "backstage-cli backend:build", + "build-image": "backstage-cli backend:build-image example-backend", "start": "backstage-cli backend:dev", "lint": "backstage-cli lint", "test": "backstage-cli test", "clean": "backstage-cli clean", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", "migrate:create": "knex migrate:make -x ts" }, "dependencies": { From d49e05e9e7c74540bec38639a8e4cfef1ee15167 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 25 Jun 2020 18:05:45 +0200 Subject: [PATCH 6/7] plugins/auth-backend: include migrations in published package Co-authored-by: Ivan Shmidt --- plugins/auth-backend/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 2de098c7eb..290dd39489 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -57,6 +57,7 @@ "jest-fetch-mock": "^3.0.3" }, "files": [ - "dist" + "dist", + "migrations" ] } From c06c7d7e6bf3c9ea197e9e45aa5b99fc2190551c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 25 Jun 2020 19:56:18 +0200 Subject: [PATCH 7/7] packages/backend,cli: revert publish config and remove postpack script requirement from image-build --- packages/backend/package.json | 9 +-------- packages/cli/src/lib/packager/index.ts | 5 ++++- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 53b1f431c0..64ca2c3113 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -1,18 +1,13 @@ { "name": "example-backend", "version": "0.1.1-alpha.12", - "main": "src/index.ts", + "main": "dist/index.cjs.js", "types": "src/index.ts", "private": true, "license": "Apache-2.0", "engines": { "node": ">=12" }, - "publishConfig": { - "access": "public", - "main": "dist/index.cjs.js", - "types": "dist/index.d.ts" - }, "scripts": { "build": "backstage-cli backend:build", "build-image": "backstage-cli backend:build-image example-backend", @@ -20,8 +15,6 @@ "lint": "backstage-cli lint", "test": "backstage-cli test", "clean": "backstage-cli clean", - "prepack": "backstage-cli prepack", - "postpack": "backstage-cli postpack", "migrate:create": "knex migrate:make -x ts" }, "dependencies": { diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/packager/index.ts index 2f5233f93f..5e309ab953 100644 --- a/packages/cli/src/lib/packager/index.ts +++ b/packages/cli/src/lib/packager/index.ts @@ -25,6 +25,7 @@ type LernaPackage = { name: string; private: boolean; location: string; + scripts: Record; }; type FileEntry = @@ -92,7 +93,9 @@ async function moveToDistWorkspace( cwd: target.location, }); // TODO(Rugvip): yarn pack doesn't call postpack, once the bug is fixed this can be removed - await run('yarn', ['postpack'], { cwd: target.location }); + if (target.scripts.postpack) { + await run('yarn', ['postpack'], { cwd: target.location }); + } const outputDir = relativePath(paths.targetRoot, target.location); const absoluteOutputPath = resolvePath(workspaceDir, outputDir);