From 2893daf3b4b2aa5cb92e73610105d509c9381499 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Thu, 23 Jul 2020 20:51:57 +0200 Subject: [PATCH 1/5] feat(cli): create-app creates backend as well with PG support --- .../cli/src/commands/create-app/createApp.ts | 11 +- .../cli/templates/default-app/app-config.yaml | 16 +++ .../default-app/packages/backend/.eslintrc.js | 3 + .../default-app/packages/backend/Dockerfile | 9 ++ .../default-app/packages/backend/README.md | 70 ++++++++++++ .../packages/backend/package.json.hbs | 51 +++++++++ .../packages/backend/src/index.test.ts | 24 ++++ .../packages/backend/src/index.ts.hbs | 105 ++++++++++++++++++ .../packages/backend/src/plugins/auth.ts | 26 +++++ .../packages/backend/src/plugins/catalog.ts | 56 ++++++++++ .../packages/backend/src/plugins/identity.ts | 22 ++++ .../packages/backend/src/plugins/proxy.ts | 25 +++++ .../backend/src/plugins/scaffolder.ts | 56 ++++++++++ .../packages/backend/src/plugins/techdocs.ts | 22 ++++ .../default-app/packages/backend/src/types.ts | 25 +++++ .../cli/templates/default-app/tsconfig.json | 12 +- 16 files changed, 529 insertions(+), 4 deletions(-) create mode 100644 packages/cli/templates/default-app/packages/backend/.eslintrc.js create mode 100644 packages/cli/templates/default-app/packages/backend/Dockerfile create mode 100644 packages/cli/templates/default-app/packages/backend/README.md create mode 100644 packages/cli/templates/default-app/packages/backend/package.json.hbs create mode 100644 packages/cli/templates/default-app/packages/backend/src/index.test.ts create mode 100644 packages/cli/templates/default-app/packages/backend/src/index.ts.hbs create mode 100644 packages/cli/templates/default-app/packages/backend/src/plugins/auth.ts create mode 100644 packages/cli/templates/default-app/packages/backend/src/plugins/catalog.ts create mode 100644 packages/cli/templates/default-app/packages/backend/src/plugins/identity.ts create mode 100644 packages/cli/templates/default-app/packages/backend/src/plugins/proxy.ts create mode 100644 packages/cli/templates/default-app/packages/backend/src/plugins/scaffolder.ts create mode 100644 packages/cli/templates/default-app/packages/backend/src/plugins/techdocs.ts create mode 100644 packages/cli/templates/default-app/packages/backend/src/types.ts diff --git a/packages/cli/src/commands/create-app/createApp.ts b/packages/cli/src/commands/create-app/createApp.ts index 4f4134957d..d0fc5e4450 100644 --- a/packages/cli/src/commands/create-app/createApp.ts +++ b/packages/cli/src/commands/create-app/createApp.ts @@ -18,7 +18,7 @@ import fs from 'fs-extra'; import { promisify } from 'util'; import chalk from 'chalk'; import { Command } from 'commander'; -import inquirer, { Answers, Question } from 'inquirer'; +import inquirer, { Answers, Question, ChoiceBase, ui } from 'inquirer'; import { exec as execCb } from 'child_process'; import { resolve as resolvePath } from 'path'; import os from 'os'; @@ -104,8 +104,17 @@ export default async (cmd: Command): Promise => { return true; }, }, + { + type: 'list', + name: 'dbType', + message: chalk.blue('Select database for the backend [required]'), + // @ts-ignore + choices: ['PostgreSQL', 'SQLite'], + }, ]; const answers: Answers = await inquirer.prompt(questions); + answers['dbTypePG'] = answers.dbType === 'PostgreSQL'; + answers['dbTypeSqlite'] = answers.dbType === 'SQLite'; const templateDir = paths.resolveOwn('templates/default-app'); const tempDir = resolvePath(os.tmpdir(), answers.name); diff --git a/packages/cli/templates/default-app/app-config.yaml b/packages/cli/templates/default-app/app-config.yaml index 58f7b795ff..0d51cf88ef 100644 --- a/packages/cli/templates/default-app/app-config.yaml +++ b/packages/cli/templates/default-app/app-config.yaml @@ -4,3 +4,19 @@ app: organization: name: Acme Corporation + +backend: + baseUrl: http://localhost:7000 + listen: 0.0.0.0:7000 + cors: + origin: http://localhost:3000 + methods: [GET, POST, PUT, DELETE] + credentials: true + +proxy: + '/test': + target: 'https://example.com' + changeOrigin: true + +techdocs: + storageUrl: https://techdocs-mock-sites.storage.googleapis.com diff --git a/packages/cli/templates/default-app/packages/backend/.eslintrc.js b/packages/cli/templates/default-app/packages/backend/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/packages/cli/templates/default-app/packages/backend/Dockerfile b/packages/cli/templates/default-app/packages/backend/Dockerfile new file mode 100644 index 0000000000..3e8ba36cec --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/Dockerfile @@ -0,0 +1,9 @@ +FROM node:12 + +WORKDIR /usr/src/app + +COPY . . + +RUN yarn install --frozen-lockfile --production + +CMD ["node", "packages/backend"] diff --git a/packages/cli/templates/default-app/packages/backend/README.md b/packages/cli/templates/default-app/packages/backend/README.md new file mode 100644 index 0000000000..90c70912f0 --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/README.md @@ -0,0 +1,70 @@ +# example-backend + +This package is an EXAMPLE of a Backstage backend. + +The main purpose of this package is to provide a test bed for Backstage plugins +that have a backend part. Feel free to experiment locally or within your fork +by adding dependencies and routes to this backend, to try things out. + +Our goal is to eventually amend the create-app flow of the CLI, such that a +production ready version of a backend skeleton is made alongside the frontend +app. Until then, feel free to experiment here! + +## Development + +To run the example backend, first go to the project root and run + +```bash +yarn install +yarn tsc +yarn build +``` + +You should only need to do this once. + +After that, go to the `packages/backend` directory and run + +```bash +AUTH_GOOGLE_CLIENT_ID=x AUTH_GOOGLE_CLIENT_SECRET=x \ +AUTH_GITHUB_CLIENT_ID=x AUTH_GITHUB_CLIENT_SECRET=x \ +AUTH_OAUTH2_CLIENT_ID=x AUTH_OAUTH2_CLIENT_SECRET=x \ +AUTH_OAUTH2_AUTH_URL=x AUTH_OAUTH2_TOKEN_URL=x \ +ROLLBAR_ACCOUNT_TOKEN=x \ +SENTRY_TOKEN=x \ +LOG_LEVEL=debug \ +yarn start +``` + +Substitute `x` for actual values, or leave them as +dummy values just to try out the backend without using the auth or sentry features. + +The backend starts up on port 7000 per default. + +## Populating The Catalog + +If you want to use the catalog functionality, you need to add so called locations +to the backend. These are places where the backend can find some entity descriptor +data to consume and serve. + +To get started, you can issue the following after starting the backend, from inside +the `plugins/catalog-backend` directory: + +```bash +yarn mock-data +``` + +You should then start seeing data on `localhost:7000/catalog/entities`. + +The catalog currently runs in-memory only, so feel free to try it out, but it will +need to be re-populated on next startup. + +## Authentication + +We chose [Passport](http://www.passportjs.org/) as authentication platform due to its comprehensive set of supported authentication [strategies](http://www.passportjs.org/packages/). + +Read more about the [auth-backend](https://github.com/spotify/backstage/blob/master/plugins/auth-backend/README.md) and [how to add a new provider](https://github.com/spotify/backstage/blob/master/docs/auth/add-auth-provider.md) + +## Documentation + +- [Backstage Readme](https://github.com/spotify/backstage/blob/master/README.md) +- [Backstage Documentation](https://github.com/spotify/backstage/blob/master/docs/README.md) diff --git a/packages/cli/templates/default-app/packages/backend/package.json.hbs b/packages/cli/templates/default-app/packages/backend/package.json.hbs new file mode 100644 index 0000000000..93c076a967 --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/package.json.hbs @@ -0,0 +1,51 @@ +{ + "name": "backend", + "version": "0.0.0", + "main": "dist/index.cjs.js", + "types": "src/index.ts", + "private": true, + "engines": { + "node": "12" + }, + "scripts": { + "build": "backstage-cli backend:build", + "build-image": "backstage-cli backend:build-image example-backend", + "start": "backstage-cli backend:dev", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "clean": "backstage-cli clean", + "migrate:create": "knex migrate:make -x ts" + }, + "dependencies": { + "@backstage/backend-common": "^{{version}}", + "@backstage/catalog-model": "^{{version}}", + "@backstage/config": "^0.1.1-alpha.13", + "@backstage/config-loader": "^0.1.1-alpha.13", + "@backstage/plugin-auth-backend": "^{{version}}", + "@backstage/plugin-catalog-backend": "^{{version}}", + "@backstage/plugin-identity-backend": "^{{version}}", + "@backstage/plugin-proxy-backend": "^{{version}}", + "@backstage/plugin-rollbar-backend": "^{{version}}", + "@backstage/plugin-scaffolder-backend": "^{{version}}", + "@backstage/plugin-sentry-backend": "^{{version}}", + "@backstage/plugin-techdocs-backend": "^{{version}}", + "@octokit/rest": "^18.0.0", + "dockerode": "^3.2.0", + "express": "^4.17.1", + "knex": "^0.21.1", + {{#if dbTypePG}} + "pg": "^8.3.0", + {{/if}} + {{#if dbTypeSqlite}} + "sqlite3": "^4.2.0", + {{/if}} + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/cli": "^0.1.1-alpha.15", + "@types/dockerode": "^2.5.32", + "@types/express": "^4.17.6", + "@types/express-serve-static-core": "^4.17.5", + "@types/helmet": "^0.0.47" + } +} diff --git a/packages/cli/templates/default-app/packages/backend/src/index.test.ts b/packages/cli/templates/default-app/packages/backend/src/index.test.ts new file mode 100644 index 0000000000..d18873c1f0 --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/src/index.test.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginEnvironment } from './types'; + +describe('test', () => { + it('unbreaks the test runner', () => { + const unbreaker = {} as PluginEnvironment; + expect(unbreaker).toBeTruthy(); + }); +}); diff --git a/packages/cli/templates/default-app/packages/backend/src/index.ts.hbs b/packages/cli/templates/default-app/packages/backend/src/index.ts.hbs new file mode 100644 index 0000000000..d1cf3d1ada --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/src/index.ts.hbs @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Hi! + * + * Note that this is an EXAMPLE Backstage backend. Please check the README. + * + * Happy hacking! + */ + +import { + createServiceBuilder, + getRootLogger, + useHotMemoize, +} from '@backstage/backend-common'; +import { ConfigReader, AppConfig } from '@backstage/config'; +import { loadConfig } from '@backstage/config-loader'; +import knex from 'knex'; +import auth from './plugins/auth'; +import catalog from './plugins/catalog'; +import identity from './plugins/identity'; +import scaffolder from './plugins/scaffolder'; +import proxy from './plugins/proxy'; +import techdocs from './plugins/techdocs'; +import { PluginEnvironment } from './types'; + +function makeCreateEnv(loadedConfigs: AppConfig[]) { + const config = ConfigReader.fromConfigs(loadedConfigs); + + return (plugin: string): PluginEnvironment => { + const logger = getRootLogger().child({ type: 'plugin', plugin }); + + {{#if dbTypePG}} + const knexConfig = { + client: 'pg', + useNullAsDefault: true, + connection: { + host: process.env.PG_HOST, + user: process.env.PG_USER, + password: process.env.PG_PASSWORD, + database: `backstage_plugin_${plugin}`, + }, + }; + {{/if}} + {{#if dbTypeSqlite}} + const knexConfig = { + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }; + {{/if}} + const database = knex(knexConfig); + database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { + resource.run('PRAGMA foreign_keys = ON', () => {}); + }); + return { logger, database, config }; + }; +} + +async function main() { + const configs = await loadConfig(); + const configReader = ConfigReader.fromConfigs(configs); + const createEnv = makeCreateEnv(configs); + + const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); + const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); + const authEnv = useHotMemoize(module, () => createEnv('auth')); + const identityEnv = useHotMemoize(module, () => createEnv('identity')); + const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); + const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); + + const service = createServiceBuilder(module) + .loadConfig(configReader) + .addRouter('/catalog', await catalog(catalogEnv)) + .addRouter('/scaffolder', await scaffolder(scaffolderEnv)) + .addRouter('/auth', await auth(authEnv)) + .addRouter('/identity', await identity(identityEnv)) + .addRouter('/techdocs', await techdocs(techdocsEnv)) + .addRouter('/proxy', await proxy(proxyEnv)); + + await service.start().catch(err => { + console.log(err); + process.exit(1); + }); +} + +module.hot?.accept(); +main().catch(error => { + console.error(`Backend failed to start up, ${error}`); + process.exit(1); +}); diff --git a/packages/cli/templates/default-app/packages/backend/src/plugins/auth.ts b/packages/cli/templates/default-app/packages/backend/src/plugins/auth.ts new file mode 100644 index 0000000000..b24dd6ca6b --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/src/plugins/auth.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createRouter } from '@backstage/plugin-auth-backend'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ + logger, + database, + config, +}: PluginEnvironment) { + return await createRouter({ logger, config, database }); +} diff --git a/packages/cli/templates/default-app/packages/backend/src/plugins/catalog.ts b/packages/cli/templates/default-app/packages/backend/src/plugins/catalog.ts new file mode 100644 index 0000000000..41e4226001 --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/src/plugins/catalog.ts @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + createRouter, + DatabaseEntitiesCatalog, + DatabaseLocationsCatalog, + DatabaseManager, + HigherOrderOperations, + LocationReaders, + runPeriodically, +} from '@backstage/plugin-catalog-backend'; +import { PluginEnvironment } from '../types'; +import { useHotCleanup } from '@backstage/backend-common'; + +export default async function createPlugin({ + logger, + database, +}: PluginEnvironment) { + const locationReader = new LocationReaders(logger); + + const db = await DatabaseManager.createDatabase(database, { logger }); + const entitiesCatalog = new DatabaseEntitiesCatalog(db); + const locationsCatalog = new DatabaseLocationsCatalog(db); + const higherOrderOperation = new HigherOrderOperations( + entitiesCatalog, + locationsCatalog, + locationReader, + logger, + ); + + useHotCleanup( + module, + runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000), + ); + + return await createRouter({ + entitiesCatalog, + locationsCatalog, + higherOrderOperation, + logger, + }); +} diff --git a/packages/cli/templates/default-app/packages/backend/src/plugins/identity.ts b/packages/cli/templates/default-app/packages/backend/src/plugins/identity.ts new file mode 100644 index 0000000000..63a326965c --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/src/plugins/identity.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createRouter } from '@backstage/plugin-identity-backend'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ logger }: PluginEnvironment) { + return await createRouter({ logger }); +} diff --git a/packages/cli/templates/default-app/packages/backend/src/plugins/proxy.ts b/packages/cli/templates/default-app/packages/backend/src/plugins/proxy.ts new file mode 100644 index 0000000000..4964de130e --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/src/plugins/proxy.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// @ts-ignore +import { createRouter } from '@backstage/plugin-proxy-backend'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ + logger, + config, +}: PluginEnvironment) { + return await createRouter({ logger, config }); +} diff --git a/packages/cli/templates/default-app/packages/backend/src/plugins/scaffolder.ts b/packages/cli/templates/default-app/packages/backend/src/plugins/scaffolder.ts new file mode 100644 index 0000000000..a9e5f185ec --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/src/plugins/scaffolder.ts @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + CookieCutter, + createRouter, + FilePreparer, + GithubPreparer, + Preparers, + GithubPublisher, + CreateReactAppTemplater, + Templaters, +} from '@backstage/plugin-scaffolder-backend'; +import { Octokit } from '@octokit/rest'; +import type { PluginEnvironment } from '../types'; +import Docker from 'dockerode'; + +export default async function createPlugin({ logger }: PluginEnvironment) { + const cookiecutterTemplater = new CookieCutter(); + const craTemplater = new CreateReactAppTemplater(); + const templaters = new Templaters(); + templaters.register('cookiecutter', cookiecutterTemplater); + templaters.register('cra', craTemplater); + + const filePreparer = new FilePreparer(); + const githubPreparer = new GithubPreparer(); + const preparers = new Preparers(); + + preparers.register('file', filePreparer); + preparers.register('github', githubPreparer); + + const githubClient = new Octokit({ auth: process.env.GITHUB_ACCESS_TOKEN }); + const publisher = new GithubPublisher({ client: githubClient }); + + const dockerClient = new Docker(); + return await createRouter({ + preparers, + templaters, + publisher, + logger, + dockerClient, + }); +} diff --git a/packages/cli/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/cli/templates/default-app/packages/backend/src/plugins/techdocs.ts new file mode 100644 index 0000000000..0d03bfabab --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/src/plugins/techdocs.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createRouter } from '@backstage/plugin-techdocs-backend'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ logger }: PluginEnvironment) { + return await createRouter({ logger }); +} diff --git a/packages/cli/templates/default-app/packages/backend/src/types.ts b/packages/cli/templates/default-app/packages/backend/src/types.ts new file mode 100644 index 0000000000..f7df3d05c6 --- /dev/null +++ b/packages/cli/templates/default-app/packages/backend/src/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Knex from 'knex'; +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; + +export type PluginEnvironment = { + logger: Logger; + database: Knex; + config: Config; +}; diff --git a/packages/cli/templates/default-app/tsconfig.json b/packages/cli/templates/default-app/tsconfig.json index 52cdd1e825..9dbc3bbe27 100644 --- a/packages/cli/templates/default-app/tsconfig.json +++ b/packages/cli/templates/default-app/tsconfig.json @@ -1,8 +1,14 @@ { "extends": "@backstage/cli/config/tsconfig.json", - "include": ["packages/*/src", "plugins/*/src", "plugins/*/dev"], - "exclude": ["**/node_modules"], + "include": [ + "packages/*/src", + "plugins/*/src", + "plugins/*/dev", + "plugins/*/migrations" + ], + "exclude": ["node_modules"], "compilerOptions": { - "outDir": "dist" + "outDir": "dist", + "skipLibCheck": true } } From 189e1bf547693b2160ad5b1191660392a2677618 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Thu, 23 Jul 2020 22:11:25 +0200 Subject: [PATCH 2/5] feat(backend): add support for PG --- packages/backend/package.json | 1 + packages/backend/src/index.ts | 34 +++++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 86d348d2a1..b040c12554 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -34,6 +34,7 @@ "dockerode": "^3.2.0", "express": "^4.17.1", "knex": "^0.21.1", + "pg": "^8.3.0", "sqlite3": "^4.2.0", "winston": "^3.2.1" }, diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index d060791f98..8b9edec327 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -45,11 +45,35 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { return (plugin: string): PluginEnvironment => { const logger = getRootLogger().child({ type: 'plugin', plugin }); - const database = knex({ - client: 'sqlite3', - connection: ':memory:', - useNullAsDefault: true, - }); + // Supported DBs are sqlite and postgres + const isPg = [ + 'POSTGRES_USER', + 'POSTGRES_HOST', + 'POSTGRES_PASSWORD', + ].every(key => Object.keys(process.env).includes(key)); + + let knexConfig; + + if (isPg) { + knexConfig = { + client: 'pg', + useNullAsDefault: true, + connection: { + host: process.env.PG_HOST, + user: process.env.PG_USER, + password: process.env.PG_PASSWORD, + database: `backstage_plugin_${plugin}`, + }, + }; + } else { + knexConfig = { + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }; + } + + const database = knex(knexConfig); database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { resource.run('PRAGMA foreign_keys = ON', () => {}); }); From 5f025442dc05eb4d5f285f3bdb2b98b3d57ac75c Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Thu, 23 Jul 2020 22:12:53 +0200 Subject: [PATCH 3/5] feat(cli): add e2e test for backend --- .github/workflows/cli.yml | 17 ++ packages/cli/e2e-test/cli-e2e-test.js | 98 ++++++- packages/cli/e2e-test/helpers.js | 6 + packages/cli/package.json | 2 + .../cli/src/commands/create-app/createApp.ts | 6 +- .../cli/templates/default-app/.eslintrc.js | 2 +- .../packages/backend/src/index.ts.hbs | 11 +- yarn.lock | 270 +++++++++++++++++- 8 files changed, 399 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 7704aa36cd..13b84a555f 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -13,6 +13,18 @@ jobs: build: runs-on: ${{ matrix.os }} + services: + postgres: + image: postgres:10.8 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432/tcp + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + strategy: matrix: os: [ubuntu-latest] @@ -44,6 +56,11 @@ jobs: - run: yarn tsc - run: yarn build - name: verify app and plugin creation + env: + POSTGRES_HOST: localhost + POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres run: | sudo sysctl fs.inotify.max_user_watches=524288 node ${{ github.workspace }}/packages/cli/e2e-test/cli-e2e-test.js diff --git a/packages/cli/e2e-test/cli-e2e-test.js b/packages/cli/e2e-test/cli-e2e-test.js index ef5d6ab243..37c1904b6d 100644 --- a/packages/cli/e2e-test/cli-e2e-test.js +++ b/packages/cli/e2e-test/cli-e2e-test.js @@ -16,6 +16,7 @@ const os = require('os'); const fs = require('fs-extra'); +const fetch = require('node-fetch'); const killTree = require('tree-kill'); const { resolve: resolvePath, join: joinPath } = require('path'); const Browser = require('zombie'); @@ -28,6 +29,7 @@ const { waitForExit, print, } = require('./helpers'); +const pgtools = require('pgtools'); async function main() { const rootDir = await fs.mkdtemp(resolvePath(os.tmpdir(), 'backstage-e2e-')); @@ -36,8 +38,9 @@ async function main() { print('Building dist workspace'); const workspaceDir = await buildDistWorkspace('workspace', rootDir); + const isPostgres = Boolean(process.env.POSTGRES_USER); print('Creating a Backstage App'); - const appDir = await createApp('test-app', workspaceDir, rootDir); + const appDir = await createApp('test-app', isPostgres, workspaceDir, rootDir); print('Creating a Backstage Plugin'); const pluginName = await createPlugin('test-plugin', appDir); @@ -45,6 +48,9 @@ async function main() { print('Starting the app'); await testAppServe(pluginName, appDir); + print('Testing the backend startup'); + await testBackendStart(appDir, isPostgres); + print('All tests successful, removing test dir'); await fs.remove(rootDir); } @@ -97,7 +103,7 @@ async function pinYarnVersion(dir) { /** * Creates a new app inside rootDir called test-app, using packages from the workspaceDir */ -async function createApp(appName, workspaceDir, rootDir) { +async function createApp(appName, isPostgres, workspaceDir, rootDir) { const child = spawnPiped( [ 'node', @@ -119,6 +125,14 @@ async function createApp(appName, workspaceDir, rootDir) { await waitFor(() => stdout.includes('Enter a name for the app')); child.stdin.write(`${appName}\n`); + await waitFor(() => stdout.includes('Select database for the backend')); + + if (!isPostgres) { + // Simulate down arrow press + child.stdin.write(`\u001B\u005B\u0042`); + } + child.stdin.write(`\n`); + print('Waiting for app create script to be done'); await waitForExit(child); @@ -250,5 +264,85 @@ async function testAppServe(pluginName, appDir) { } } +/** Creates PG databases (drops if exists before) */ +async function createDB(database) { + const config = { + host: process.env.POSTGRES_HOST, + port: process.env.POSTGRES_PORT, + user: process.env.POSTGRES_USER, + password: process.env.POSTGRES_PASSWORD, + }; + + try { + await pgtools.dropdb({ config }, database); + } catch (_) { + /* do nothing*/ + } + return pgtools.createdb(config, database); +} + +/** + * Start serving the newly created backend and make sure that all db migrations works correctly + */ +async function testBackendStart(appDir, isPostgres) { + if (isPostgres) { + print('Creating DBs'); + await Promise.all( + [ + 'catalog', + 'scaffolder', + 'auth', + 'identity', + 'proxy', + 'techdocs', + ].map(name => createDB(`backstage_plugin_${name}`)), + ); + print('Created DBs'); + } + + const child = spawnPiped(['yarn', 'workspace', 'backend', 'start'], { + cwd: appDir, + }); + + let stdout = ''; + let stderr = ''; + child.stdout.on('data', data => { + stdout = stdout + data.toString('utf8'); + }); + child.stderr.on('data', data => { + stderr = stderr + data.toString('utf8'); + }); + let successful = false; + + try { + await waitFor(() => stdout.includes('Listening on ') || stderr !== ''); + if (stderr !== '') { + // Skipping the whole block + throw new Error(); + } + + print('Try to fetch entities from the backend'); + // Try fetch entities, should be ok + await fetch('http://localhost:7000/catalog/entities').then(res => + res.json(), + ); + print('Entities fetched successfully'); + successful = true; + } finally { + print('Stopping the child process'); + // Kill entire process group, otherwise we'll end up with hanging serve processes + killTree(child.pid); + } + + try { + await waitForExit(child); + } catch (error) { + if (!successful) { + throw error; + } + print('Backend startup test finished successfully'); + } +} + process.on('unhandledRejection', handleError); main(process.argv.slice(2)).catch(handleError); diff --git a/packages/cli/e2e-test/helpers.js b/packages/cli/e2e-test/helpers.js index da2ec343ab..84807119c3 100644 --- a/packages/cli/e2e-test/helpers.js +++ b/packages/cli/e2e-test/helpers.js @@ -76,6 +76,12 @@ function handleError(err) { } } +/** + * Waits for fn() to be true + * Checks every 100ms + * .cancel() is available + * @returns {Promise} Promise of resolution + */ function waitFor(fn) { return new Promise(resolve => { const handle = setInterval(() => { diff --git a/packages/cli/package.json b/packages/cli/package.json index ed7ae9725b..f99f2a7741 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -67,7 +67,9 @@ "jest-css-modules": "^2.1.0", "jest-esm-transformer": "^1.0.0", "mini-css-extract-plugin": "^0.9.0", + "node-fetch": "^2.6.0", "ora": "^4.0.3", + "pgtools": "^0.3.0", "raw-loader": "^4.0.1", "react": "^16.0.0", "react-dev-utils": "^10.2.1", diff --git a/packages/cli/src/commands/create-app/createApp.ts b/packages/cli/src/commands/create-app/createApp.ts index d0fc5e4450..b3ddcd3109 100644 --- a/packages/cli/src/commands/create-app/createApp.ts +++ b/packages/cli/src/commands/create-app/createApp.ts @@ -18,7 +18,7 @@ import fs from 'fs-extra'; import { promisify } from 'util'; import chalk from 'chalk'; import { Command } from 'commander'; -import inquirer, { Answers, Question, ChoiceBase, ui } from 'inquirer'; +import inquirer, { Answers, Question } from 'inquirer'; import { exec as execCb } from 'child_process'; import { resolve as resolvePath } from 'path'; import os from 'os'; @@ -113,8 +113,8 @@ export default async (cmd: Command): Promise => { }, ]; const answers: Answers = await inquirer.prompt(questions); - answers['dbTypePG'] = answers.dbType === 'PostgreSQL'; - answers['dbTypeSqlite'] = answers.dbType === 'SQLite'; + answers.dbTypePG = answers.dbType === 'PostgreSQL'; + answers.dbTypeSqlite = answers.dbType === 'SQLite'; const templateDir = paths.resolveOwn('templates/default-app'); const tempDir = resolvePath(os.tmpdir(), answers.name); diff --git a/packages/cli/templates/default-app/.eslintrc.js b/packages/cli/templates/default-app/.eslintrc.js index 13573efa9c..e351352491 100644 --- a/packages/cli/templates/default-app/.eslintrc.js +++ b/packages/cli/templates/default-app/.eslintrc.js @@ -1,3 +1,3 @@ module.exports = { - extends: [require.resolve('@backstage/cli/config/eslint')], + root: true, }; diff --git a/packages/cli/templates/default-app/packages/backend/src/index.ts.hbs b/packages/cli/templates/default-app/packages/backend/src/index.ts.hbs index d1cf3d1ada..5e98afb6d1 100644 --- a/packages/cli/templates/default-app/packages/backend/src/index.ts.hbs +++ b/packages/cli/templates/default-app/packages/backend/src/index.ts.hbs @@ -29,7 +29,7 @@ import { } from '@backstage/backend-common'; import { ConfigReader, AppConfig } from '@backstage/config'; import { loadConfig } from '@backstage/config-loader'; -import knex from 'knex'; +import knex, { PgConnectionConfig } from 'knex'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; import identity from './plugins/identity'; @@ -49,11 +49,12 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { client: 'pg', useNullAsDefault: true, connection: { - host: process.env.PG_HOST, - user: process.env.PG_USER, - password: process.env.PG_PASSWORD, + port: process.env.POSTGRES_PORT, + host: process.env.POSTGRES_HOST, + user: process.env.POSTGRES_USER, + password: process.env.POSTGRES_PASSWORD, database: `backstage_plugin_${plugin}`, - }, + } as PgConnectionConfig, }; {{/if}} {{#if dbTypeSqlite}} diff --git a/yarn.lock b/yarn.lock index 6c0ff99388..85f2b5b994 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5669,6 +5669,16 @@ buffer-indexof@^1.0.0: resolved "https://registry.npmjs.org/buffer-indexof/-/buffer-indexof-1.1.1.tgz#52fabcc6a606d1a00302802648ef68f639da268c" integrity sha512-4/rOEg86jivtPTeOUUT61jJO1Ya1TrR/OkqCSZDyq84WJh3LuuiphBYJN+fm5xufIk4XAFcEwte/8WzC8If/1g== +buffer-writer@1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" + integrity sha1-Iqk2kB4wKa/NdUfrRIfOtpejvwg= + +buffer-writer@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz#ce7eb81a38f7829db09c873f2fbb792c0c98ec04" + integrity sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw== + buffer-xor@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" @@ -5872,6 +5882,11 @@ camelcase@^2.0.0: resolved "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz#7c1d16d679a1bbe59ca02cacecfb011e201f5a1f" integrity sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8= +camelcase@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" + integrity sha1-MvxLn82vhF/N9+c7uXysImHwqwo= + camelcase@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd" @@ -6169,6 +6184,15 @@ clipboard@^2.0.0: select "^1.1.2" tiny-emitter "^2.0.0" +cliui@^3.2.0: + version "3.2.0" + resolved "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d" + integrity sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0= + dependencies: + string-width "^1.0.1" + strip-ansi "^3.0.1" + wrap-ansi "^2.0.0" + cliui@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz#deefcfdb2e800784aa34f46fa08e06851c7bbbc5" @@ -7301,7 +7325,7 @@ decamelize-keys@^1.0.0: decamelize "^1.1.0" map-obj "^1.0.0" -decamelize@^1.1.0, decamelize@^1.1.2, decamelize@^1.2.0: +decamelize@^1.1.0, decamelize@^1.1.1, decamelize@^1.1.2, decamelize@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= @@ -9289,6 +9313,11 @@ generic-names@^2.0.1: dependencies: loader-utils "^1.1.0" +generic-pool@2.4.3: + version "2.4.3" + resolved "https://registry.npmjs.org/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" + integrity sha1-eAw29p360FpaBF3Te+etyhGk9v8= + genfun@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/genfun/-/genfun-5.0.0.tgz#9dd9710a06900a5c4a5bf57aca5da4e52fe76537" @@ -9299,6 +9328,11 @@ gensync@^1.0.0-beta.1: resolved "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.1.tgz#58f4361ff987e5ff6e1e7a210827aa371eaac269" integrity sha512-r8EC6NO1sngH/zdD9fiRDLdcgnbayXah+mLgManTaIZJqEC1MZstmnox8KpnI2/fxQwrp5OpCOYWLp4rBl4Jcg== +get-caller-file@^1.0.1: + version "1.0.3" + resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz#f978fa4c90d1dfe7ff2d6beda2a515e713bdcf4a" + integrity sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w== + get-caller-file@^2.0.1: version "2.0.5" resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" @@ -10585,6 +10619,11 @@ invariant@^2.2.2, invariant@^2.2.3, invariant@^2.2.4: dependencies: loose-envify "^1.0.0" +invert-kv@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" + integrity sha1-EEqOSqym09jNFXqO+L+rLXo//bY= + ip-regex@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" @@ -11626,6 +11665,11 @@ js-cookie@^2.2.1: resolved "https://registry.npmjs.org/js-cookie/-/js-cookie-2.2.1.tgz#69e106dc5d5806894562902aa5baec3744e9b2b8" integrity sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ== +js-string-escape@1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" + integrity sha1-4mJbrbwNZ8dTPp7cEGjFh65BN+8= + "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -12034,6 +12078,13 @@ lazy-universal-dotenv@^3.0.1: dotenv "^8.0.0" dotenv-expand "^5.1.0" +lcid@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz#308accafa0bc483a3867b4b6f2b9506251d1b835" + integrity sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU= + dependencies: + invert-kv "^1.0.0" + left-pad@^1.3.0: version "1.3.0" resolved "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz#5b8a3a7765dfe001261dde915589e782f8c94d1e" @@ -12303,6 +12354,11 @@ lodash._reinterpolate@^3.0.0: resolved "https://registry.npmjs.org/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d" integrity sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0= +lodash.assign@^4.1.0, lodash.assign@^4.2.0: + version "4.2.0" + resolved "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7" + integrity sha1-DZnzzNem0mHRm9rrkkUAXShYCOc= + lodash.camelcase@^4.3.0: version "4.3.0" resolved "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" @@ -13659,6 +13715,11 @@ oauth@0.9.x: resolved "https://registry.npmjs.org/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1" integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE= +object-assign@4.1.0: + version "4.1.0" + resolved "https://registry.npmjs.org/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" + integrity sha1-ejs9DpgGPUP0wD8uiubNUahog6A= + object-assign@^4, object-assign@^4.0.1, object-assign@^4.1.0, object-assign@^4.1.1: version "4.1.1" resolved "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -13903,6 +13964,13 @@ os-homedir@^1.0.0: resolved "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M= +os-locale@^1.4.0: + version "1.4.0" + resolved "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz#20f9f17ae29ed345e8bde583b13d2009803c14d9" + integrity sha1-IPnxeuKe00XoveWDsT0gCYA8FNk= + dependencies: + lcid "^1.0.0" + os-name@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/os-name/-/os-name-3.1.0.tgz#dec19d966296e1cd62d701a5a66ee1ddeae70801" @@ -14076,6 +14144,16 @@ package-json@^6.3.0: registry-url "^5.0.0" semver "^6.2.0" +packet-reader@0.3.1: + version "0.3.1" + resolved "https://registry.npmjs.org/packet-reader/-/packet-reader-0.3.1.tgz#cd62e60af8d7fea8a705ec4ff990871c46871f27" + integrity sha1-zWLmCvjX/qinBexP+ZCHHEaHHyc= + +packet-reader@1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74" + integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ== + pako@~1.0.5: version "1.0.11" resolved "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" @@ -14444,11 +14522,111 @@ performance-now@^2.1.0: resolved "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= +pg-connection-string@0.1.3, pg-connection-string@^0.1.3: + version "0.1.3" + resolved "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" + integrity sha1-2hhHsglA5C7hSSvq9l1J2RskXfc= + pg-connection-string@2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.2.0.tgz#caab4d38a9de4fdc29c9317acceed752897de41c" integrity sha512-xB/+wxcpFipUZOQcSzcgkjcNOosGhEoPSjz06jC89lv1dj7mc9bZv6wLVy8M2fVjP0a/xN0N988YDq1L0FhK3A== +pg-connection-string@^2.3.0: + version "2.3.0" + resolved "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.3.0.tgz#c13fcb84c298d0bfa9ba12b40dd6c23d946f55d6" + integrity sha512-ukMTJXLI7/hZIwTW7hGMZJ0Lj0S2XQBCJ4Shv4y1zgQ/vqVea+FLhzywvPj0ujSuofu+yA4MYHGZPTsgjBgJ+w== + +pg-int8@1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" + integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== + +pg-pool@1.*: + version "1.8.0" + resolved "https://registry.npmjs.org/pg-pool/-/pg-pool-1.8.0.tgz#f7ec73824c37a03f076f51bfdf70e340147c4f37" + integrity sha1-9+xzgkw3oD8Hb1G/33DjQBR8Tzc= + dependencies: + generic-pool "2.4.3" + object-assign "4.1.0" + +pg-pool@^3.2.1: + version "3.2.1" + resolved "https://registry.npmjs.org/pg-pool/-/pg-pool-3.2.1.tgz#5f4afc0f58063659aeefa952d36af49fa28b30e0" + integrity sha512-BQDPWUeKenVrMMDN9opfns/kZo4lxmSWhIqo+cSAF7+lfi9ZclQbr9vfnlNaPr8wYF3UYjm5X0yPAhbcgqNOdA== + +pg-protocol@^1.2.5: + version "1.2.5" + resolved "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.2.5.tgz#28a1492cde11646ff2d2d06bdee42a3ba05f126c" + integrity sha512-1uYCckkuTfzz/FCefvavRywkowa6M5FohNMF5OjKrqo9PSR8gYc8poVmwwYQaBxhmQdBjhtP514eXy9/Us2xKg== + +pg-types@1.*: + version "1.13.0" + resolved "https://registry.npmjs.org/pg-types/-/pg-types-1.13.0.tgz#75f490b8a8abf75f1386ef5ec4455ecf6b345c63" + integrity sha512-lfKli0Gkl/+za/+b6lzENajczwZHc7D5kiUCZfgm914jipD2kIOIvEkAhZ8GrW3/TUoP9w8FHjwpPObBye5KQQ== + dependencies: + pg-int8 "1.0.1" + postgres-array "~1.0.0" + postgres-bytea "~1.0.0" + postgres-date "~1.0.0" + postgres-interval "^1.1.0" + +pg-types@^2.1.0: + version "2.2.0" + resolved "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" + integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== + dependencies: + pg-int8 "1.0.1" + postgres-array "~2.0.0" + postgres-bytea "~1.0.0" + postgres-date "~1.0.4" + postgres-interval "^1.1.0" + +pg@^6.1.0: + version "6.4.2" + resolved "https://registry.npmjs.org/pg/-/pg-6.4.2.tgz#c364011060eac7a507a2ae063eb857ece910e27f" + integrity sha1-w2QBEGDqx6UHoq4GPrhX7OkQ4n8= + dependencies: + buffer-writer "1.0.1" + js-string-escape "1.0.1" + packet-reader "0.3.1" + pg-connection-string "0.1.3" + pg-pool "1.*" + pg-types "1.*" + pgpass "1.*" + semver "4.3.2" + +pg@^8.3.0: + version "8.3.0" + resolved "https://registry.npmjs.org/pg/-/pg-8.3.0.tgz#941383300d38eef51ecb88a0188cec441ab64d81" + integrity sha512-jQPKWHWxbI09s/Z9aUvoTbvGgoj98AU7FDCcQ7kdejupn/TcNpx56v2gaOTzXkzOajmOEJEdi9eTh9cA2RVAjQ== + dependencies: + buffer-writer "2.0.0" + packet-reader "1.0.0" + pg-connection-string "^2.3.0" + pg-pool "^3.2.1" + pg-protocol "^1.2.5" + pg-types "^2.1.0" + pgpass "1.x" + semver "4.3.2" + +pgpass@1.*, pgpass@1.x: + version "1.0.2" + resolved "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" + integrity sha1-Knu0G2BltnkH6R2hsHwYR8h3swY= + dependencies: + split "^1.0.0" + +pgtools@^0.3.0: + version "0.3.0" + resolved "https://registry.npmjs.org/pgtools/-/pgtools-0.3.0.tgz#ee7decf4183ada28299c63df71e1e73c95b5bc53" + integrity sha512-8NxDCJ8xJ6hOp9hVNZqxi+TZl7hM1Jc8pQyj8DlAbyaWnk5OsGwf3gB/UyDODdOguiim9QzbzPsslp//apO+Uw== + dependencies: + bluebird "^3.3.5" + pg "^6.1.0" + pg-connection-string "^0.1.3" + yargs "^5.0.0" + picomatch@^2.0.4, picomatch@^2.0.5, picomatch@^2.2.1, picomatch@^2.2.2: version "2.2.2" resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" @@ -14980,6 +15158,33 @@ postcss@^6.0.1: source-map "^0.6.1" supports-color "^5.4.0" +postgres-array@~1.0.0: + version "1.0.3" + resolved "https://registry.npmjs.org/postgres-array/-/postgres-array-1.0.3.tgz#c561fc3b266b21451fc6555384f4986d78ec80f5" + integrity sha512-5wClXrAP0+78mcsNX3/ithQ5exKvCyK5lr5NEEEeGwwM6NJdQgzIJBVxLvRW+huFpX92F2QnZ5CcokH0VhK2qQ== + +postgres-array@~2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e" + integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA== + +postgres-bytea@~1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35" + integrity sha1-AntTPAqokOJtFy1Hz5zOzFIazTU= + +postgres-date@~1.0.0, postgres-date@~1.0.4: + version "1.0.5" + resolved "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.5.tgz#710b27de5f27d550f6e80b5d34f7ba189213c2ee" + integrity sha512-pdau6GRPERdAYUQwkBnGKxEfPyhVZXG/JiS44iZWiNdSOWE09N2lUgN6yshuq6fVSon4Pm0VMXd1srUUkLe9iA== + +postgres-interval@^1.1.0: + version "1.2.0" + resolved "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695" + integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ== + dependencies: + xtend "^4.0.0" + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" @@ -16304,6 +16509,11 @@ require-directory@^2.1.1: resolved "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I= +require-main-filename@^1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1" + integrity sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE= + require-main-filename@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" @@ -16747,6 +16957,11 @@ semver-regex@^2.0.0: resolved "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== +semver@4.3.2: + version "4.3.2" + resolved "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" + integrity sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c= + semver@7.0.0: version "7.0.0" resolved "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" @@ -17524,7 +17739,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -string-width@^1.0.1: +string-width@^1.0.1, string-width@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" integrity sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M= @@ -19316,6 +19531,11 @@ whatwg-url@^8.0.0: tr46 "^2.0.2" webidl-conversions "^5.0.0" +which-module@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" + integrity sha1-u6Y8qGGUiZT/MHc2CJ47lgJsKk8= + which-module@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" @@ -19354,6 +19574,11 @@ widest-line@^3.1.0: dependencies: string-width "^4.0.0" +window-size@^0.2.0: + version "0.2.0" + resolved "https://registry.npmjs.org/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" + integrity sha1-tDFbtCFKPXBY6+7okuE/ok2YsHU= + windows-release@^3.1.0: version "3.2.0" resolved "https://registry.npmjs.org/windows-release/-/windows-release-3.2.0.tgz#8122dad5afc303d833422380680a79cdfa91785f" @@ -19408,6 +19633,14 @@ worker-rpc@^0.1.0: dependencies: microevent.ts "~0.1.1" +wrap-ansi@^2.0.0: + version "2.1.0" + resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85" + integrity sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU= + dependencies: + string-width "^1.0.1" + strip-ansi "^3.0.1" + wrap-ansi@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba" @@ -19599,6 +19832,11 @@ xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.1: resolved "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== +y18n@^3.2.1: + version "3.2.1" + resolved "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41" + integrity sha1-bRX7qITAhnnA136I53WegR4H+kE= + y18n@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b" @@ -19650,6 +19888,14 @@ yargs-parser@^15.0.1: camelcase "^5.0.0" decamelize "^1.2.0" +yargs-parser@^3.2.0: + version "3.2.0" + resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-3.2.0.tgz#5081355d19d9d0c8c5d81ada908cb4e6d186664f" + integrity sha1-UIE1XRnZ0MjF2BrakIy05tGGZk8= + dependencies: + camelcase "^3.0.0" + lodash.assign "^4.1.0" + yargs@^13.3.2: version "13.3.2" resolved "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz#ad7ffefec1aa59565ac915f82dccb38a9c31a2dd" @@ -19700,6 +19946,26 @@ yargs@^15.3.1: y18n "^4.0.0" yargs-parser "^18.1.1" +yargs@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/yargs/-/yargs-5.0.0.tgz#3355144977d05757dbb86d6e38ec056123b3a66e" + integrity sha1-M1UUSXfQV1fbuG1uOOwFYSOzpm4= + dependencies: + cliui "^3.2.0" + decamelize "^1.1.1" + get-caller-file "^1.0.1" + lodash.assign "^4.2.0" + os-locale "^1.4.0" + read-pkg-up "^1.0.1" + require-directory "^2.1.1" + require-main-filename "^1.0.1" + set-blocking "^2.0.0" + string-width "^1.0.2" + which-module "^1.0.0" + window-size "^0.2.0" + y18n "^3.2.1" + yargs-parser "^3.2.0" + yauzl@2.10.0, yauzl@^2.10.0: version "2.10.0" resolved "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" From a6b8602378c0490e8c1bfef4a589717220bb9111 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Fri, 24 Jul 2020 04:01:21 +0200 Subject: [PATCH 4/5] fix: tsc and pr comments --- packages/backend/src/index.ts | 11 ++++++----- .../default-app/packages/backend/src/index.ts.hbs | 5 +++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 8b9edec327..0da700d205 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -29,7 +29,7 @@ import { } from '@backstage/backend-common'; import { ConfigReader, AppConfig } from '@backstage/config'; import { loadConfig } from '@backstage/config-loader'; -import knex from 'knex'; +import knex, { PgConnectionConfig } from 'knex'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; import identity from './plugins/identity'; @@ -59,11 +59,12 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { client: 'pg', useNullAsDefault: true, connection: { - host: process.env.PG_HOST, - user: process.env.PG_USER, - password: process.env.PG_PASSWORD, + port: process.env.POSTGRES_PORT, + host: process.env.POSTGRES_HOST, + user: process.env.POSTGRES_USER, + password: process.env.POSTGRES_PASSWORD, database: `backstage_plugin_${plugin}`, - }, + } as PgConnectionConfig, }; } else { knexConfig = { diff --git a/packages/cli/templates/default-app/packages/backend/src/index.ts.hbs b/packages/cli/templates/default-app/packages/backend/src/index.ts.hbs index 5e98afb6d1..591fa1a28f 100644 --- a/packages/cli/templates/default-app/packages/backend/src/index.ts.hbs +++ b/packages/cli/templates/default-app/packages/backend/src/index.ts.hbs @@ -29,7 +29,12 @@ import { } from '@backstage/backend-common'; import { ConfigReader, AppConfig } from '@backstage/config'; import { loadConfig } from '@backstage/config-loader'; +{{#if dbTypePG}} import knex, { PgConnectionConfig } from 'knex'; +{{/if}} +{{#if dbTypeSqlite}} +import knex from 'knex'; +{{/if}} import auth from './plugins/auth'; import catalog from './plugins/catalog'; import identity from './plugins/identity'; From e5ed8f95d1e318044eff61f967fe1fd3b8a8a4b9 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Mon, 27 Jul 2020 14:38:07 +0200 Subject: [PATCH 5/5] fix(cli): e2e backend PR adjustments --- .github/workflows/cli.yml | 2 +- packages/backend/src/index.ts | 10 +++++----- packages/cli/e2e-test/cli-e2e-test.js | 6 ++++-- .../templates/default-app/packages/backend/README.md | 2 -- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 13b84a555f..32fb15d329 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -15,7 +15,7 @@ jobs: services: postgres: - image: postgres:10.8 + image: postgres:latest env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 0da700d205..9d2d89a049 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -50,7 +50,7 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { 'POSTGRES_USER', 'POSTGRES_HOST', 'POSTGRES_PASSWORD', - ].every(key => Object.keys(process.env).includes(key)); + ].every(key => config.getOptional(`backend.${key}`)); let knexConfig; @@ -59,10 +59,10 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { client: 'pg', useNullAsDefault: true, connection: { - port: process.env.POSTGRES_PORT, - host: process.env.POSTGRES_HOST, - user: process.env.POSTGRES_USER, - password: process.env.POSTGRES_PASSWORD, + port: config.getOptionalNumber('backend.POSTGRES_PORT'), + host: config.getString('backend.POSTGRES_HOST'), + user: config.getString('backend.POSTGRES_USER'), + password: config.getString('backend.POSTGRES_PASSWORD'), database: `backstage_plugin_${plugin}`, } as PgConnectionConfig, }; diff --git a/packages/cli/e2e-test/cli-e2e-test.js b/packages/cli/e2e-test/cli-e2e-test.js index 37c1904b6d..6bab97f66b 100644 --- a/packages/cli/e2e-test/cli-e2e-test.js +++ b/packages/cli/e2e-test/cli-e2e-test.js @@ -318,7 +318,7 @@ async function testBackendStart(appDir, isPostgres) { await waitFor(() => stdout.includes('Listening on ') || stderr !== ''); if (stderr !== '') { // Skipping the whole block - throw new Error(); + throw new Error(stderr); } print('Try to fetch entities from the backend'); @@ -328,6 +328,8 @@ async function testBackendStart(appDir, isPostgres) { ); print('Entities fetched successfully'); successful = true; + } catch (error) { + throw new Error(`Backend failed to startup: ${error}`); } finally { print('Stopping the child process'); // Kill entire process group, otherwise we'll end up with hanging serve processes @@ -338,7 +340,7 @@ async function testBackendStart(appDir, isPostgres) { await waitForExit(child); } catch (error) { if (!successful) { - throw error; + throw new Error(`Backend failed to startup: ${stderr}`); } print('Backend startup test finished successfully'); } diff --git a/packages/cli/templates/default-app/packages/backend/README.md b/packages/cli/templates/default-app/packages/backend/README.md index 90c70912f0..f94904a930 100644 --- a/packages/cli/templates/default-app/packages/backend/README.md +++ b/packages/cli/templates/default-app/packages/backend/README.md @@ -29,8 +29,6 @@ AUTH_GOOGLE_CLIENT_ID=x AUTH_GOOGLE_CLIENT_SECRET=x \ AUTH_GITHUB_CLIENT_ID=x AUTH_GITHUB_CLIENT_SECRET=x \ AUTH_OAUTH2_CLIENT_ID=x AUTH_OAUTH2_CLIENT_SECRET=x \ AUTH_OAUTH2_AUTH_URL=x AUTH_OAUTH2_TOKEN_URL=x \ -ROLLBAR_ACCOUNT_TOKEN=x \ -SENTRY_TOKEN=x \ LOG_LEVEL=debug \ yarn start ```