From 6d61fba0c5eba9cf002ba318c3205c74634f7ff3 Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Tue, 2 Jun 2020 13:53:05 +0200 Subject: [PATCH 1/9] feature: improve local database functionality 1. make locally run app use real DB with locations support 2. add shell script for filling up DB with mock data --- plugins/catalog-backend/package.json | 6 ++- plugins/catalog-backend/scripts/mock-db | 12 ++++++ .../src/database/DatabaseManager.ts | 36 ++++++++++++++---- .../src/service/standaloneApplication.ts | 17 ++++++++- .../src/service/standaloneServer.ts | 38 +++++++++++-------- .../components/CatalogTable/CatalogTable.tsx | 14 ++++++- 6 files changed, 96 insertions(+), 27 deletions(-) create mode 100644 plugins/catalog-backend/scripts/mock-db diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 5ec37295ed..a5f283abba 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -6,19 +6,21 @@ "license": "Apache-2.0", "private": true, "scripts": { - "start": "tsc-watch --onFirstSuccess \"cross-env NODE_ENV=development nodemon dist/run.js\"", + "start": "tsc-watch --onFirstSuccess \"cross-env NODE_ENV=development nodemon -r esm dist/run.js\"", "build": "tsc", "lint": "backstage-cli lint", "test": "backstage-cli test", "prepack": "backstage-cli prepack", "postpack": "backstage-cli postpack", - "clean": "backstage-cli clean" + "clean": "backstage-cli clean", + "mock-db": "sh ./scripts/mock-db" }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", "@backstage/catalog-model": "^0.1.1-alpha.6", "compression": "^1.7.4", "cors": "^2.8.5", + "esm": "^3.2.25", "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", diff --git a/plugins/catalog-backend/scripts/mock-db b/plugins/catalog-backend/scripts/mock-db new file mode 100644 index 0000000000..af5de1cb1b --- /dev/null +++ b/plugins/catalog-backend/scripts/mock-db @@ -0,0 +1,12 @@ +curl --location --request POST 'localhost:3003/locations' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "type": "github", + "target": "https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml" +}' +curl --location --request POST 'localhost:3003/locations' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "type": "github", + "target": "https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/two_components.yaml" +}' \ No newline at end of file diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 4268dc4146..9d7a79d57a 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -14,26 +14,48 @@ * limitations under the License. */ -import type { Entity, EntityPolicy } from '@backstage/catalog-model'; +import { Entity, EntityPolicy } from '@backstage/catalog-model'; import Knex from 'knex'; import lodash from 'lodash'; import path from 'path'; import { Logger } from 'winston'; -import type { IngestionModel } from '../ingestion/types'; +import { IngestionModel } from '../ingestion/types'; import { CommonDatabase } from './CommonDatabase'; -import { DatabaseLocationUpdateLogStatus } from './types'; -import type { Database, DbEntityRequest } from './types'; +import { + DatabaseLocationUpdateLogStatus, + Database, + DbEntityRequest, +} from './types'; export class DatabaseManager { public static async createDatabase( - knex: Knex, + database: Knex, logger: Logger, ): Promise { - await knex.migrate.latest({ + await database.migrate.latest({ directory: path.resolve(__dirname, 'migrations'), loadExtensions: ['.js'], }); - return new CommonDatabase(knex, logger); + return new CommonDatabase(database, logger); + } + + public static async createInMemoryDatabase( + logger: Logger, + ): Promise { + const database = Knex({ + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }); + database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { + resource.run('PRAGMA foreign_keys = ON', () => {}); + }); + + await database.migrate.latest({ + directory: path.resolve(__dirname, 'migrations'), + loadExtensions: ['.js'], + }); + return new CommonDatabase(database, logger); } private static async logUpdateSuccess( diff --git a/plugins/catalog-backend/src/service/standaloneApplication.ts b/plugins/catalog-backend/src/service/standaloneApplication.ts index 126806c751..ebd28a0a45 100644 --- a/plugins/catalog-backend/src/service/standaloneApplication.ts +++ b/plugins/catalog-backend/src/service/standaloneApplication.ts @@ -26,18 +26,26 @@ import helmet from 'helmet'; import { Logger } from 'winston'; import { EntitiesCatalog, LocationsCatalog } from '../catalog'; import { createRouter } from './router'; +import { HigherOrderOperation } from '../ingestion/types'; export interface ApplicationOptions { enableCors: boolean; entitiesCatalog: EntitiesCatalog; locationsCatalog?: LocationsCatalog; + higherOrderOperation?: HigherOrderOperation; logger: Logger; } export async function createStandaloneApplication( options: ApplicationOptions, ): Promise { - const { enableCors, entitiesCatalog, locationsCatalog, logger } = options; + const { + enableCors, + entitiesCatalog, + locationsCatalog, + logger, + higherOrderOperation, + } = options; const app = express(); app.use(helmet()); @@ -49,7 +57,12 @@ export async function createStandaloneApplication( app.use(requestLoggingHandler()); app.use( '/', - await createRouter({ entitiesCatalog, locationsCatalog, logger }), + await createRouter({ + entitiesCatalog, + locationsCatalog, + higherOrderOperation, + logger, + }), ); app.use(notFoundHandler()); app.use(errorHandler()); diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 2fd4afc7cd..8bf3554510 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -16,8 +16,14 @@ import { Server } from 'http'; import { Logger } from 'winston'; -import { StaticEntitiesCatalog } from '../catalog'; +import knex from 'knex'; import { createStandaloneApplication } from './standaloneApplication'; +import { DatabaseEntitiesCatalog } from '../catalog/DatabaseEntitiesCatalog'; +import { DatabaseManager } from '../database/DatabaseManager'; +import { DatabaseLocationsCatalog } from '../catalog/DatabaseLocationsCatalog'; +import { LocationReaders } from '../ingestion/source/LocationReaders'; +import { IngestionModels, DescriptorParsers, HigherOrderOperations } from '..'; +import { EntityPolicies } from '@backstage/catalog-model'; export interface ServerOptions { port: number; @@ -30,25 +36,27 @@ export async function startStandaloneServer( ): Promise { const logger = options.logger.child({ service: 'catalog-backend' }); - const entitiesCatalog = new StaticEntitiesCatalog([ - { - apiVersion: 'backstage.io/v1beta1', - kind: 'Component', - metadata: { name: 'c1' }, - spec: { type: 'service' }, - }, - { - apiVersion: 'backstage.io/v1beta1', - kind: 'Component', - metadata: { name: 'c2' }, - spec: { type: 'service' }, - }, - ]); + const db = await DatabaseManager.createInMemoryDatabase(logger); + + const entitiesCatalog = new DatabaseEntitiesCatalog(db); + const locationsCatalog = new DatabaseLocationsCatalog(db); + const ingestionModel = new IngestionModels( + new LocationReaders(), + new DescriptorParsers(), + new EntityPolicies(), + ); + const higherOrderOperation = new HigherOrderOperations( + entitiesCatalog, + locationsCatalog, + ingestionModel, + ); logger.debug('Creating application...'); const app = await createStandaloneApplication({ enableCors: options.enableCors, entitiesCatalog, + locationsCatalog, + higherOrderOperation, logger, }); diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx index f5fa45dc7e..35bc6c7d7c 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx @@ -17,6 +17,7 @@ import React, { FC } from 'react'; import { Component } from '../../data/component'; import { InfoCard, Progress, Table, TableColumn } from '@backstage/core'; import { Typography, Link } from '@material-ui/core'; +import EditIcon from '@material-ui/icons/Edit'; const columns: TableColumn[] = [ { @@ -66,7 +67,18 @@ const CatalogTable: FC = ({ columns={columns} options={{ paging: false }} title={`${titlePreamble} (${(components && components.length) || 0})`} - data={components} + data={components.map(({ kind, name, description }) => ({ + kind, + name, + description: ( +
+ {description} + + + +
+ ), + }))} /> ); }; From 3dfbc6e160d328fe46e07f4f2902638e4b14a775 Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Thu, 4 Jun 2020 10:12:52 +0200 Subject: [PATCH 2/9] fix: merge errors --- plugins/catalog-backend/src/service/standaloneApplication.ts | 1 - plugins/catalog-backend/src/service/standaloneServer.ts | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/service/standaloneApplication.ts b/plugins/catalog-backend/src/service/standaloneApplication.ts index db24f9d224..bdf296d06b 100644 --- a/plugins/catalog-backend/src/service/standaloneApplication.ts +++ b/plugins/catalog-backend/src/service/standaloneApplication.ts @@ -27,7 +27,6 @@ import { Logger } from 'winston'; import { EntitiesCatalog, LocationsCatalog } from '../catalog'; import { HigherOrderOperation } from '../ingestion'; import { createRouter } from './router'; -import { HigherOrderOperation } from '../ingestion/types'; export interface ApplicationOptions { enableCors: boolean; diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 8bf3554510..51fc8be829 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -15,7 +15,7 @@ */ import { Server } from 'http'; -import { Logger } from 'winston'; +import { Logger, log } from 'winston'; import knex from 'knex'; import { createStandaloneApplication } from './standaloneApplication'; import { DatabaseEntitiesCatalog } from '../catalog/DatabaseEntitiesCatalog'; @@ -49,6 +49,7 @@ export async function startStandaloneServer( entitiesCatalog, locationsCatalog, ingestionModel, + logger, ); logger.debug('Creating application...'); From da071e0afa73ed18f34425f86656e4ed486287ec Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Thu, 4 Jun 2020 10:15:53 +0200 Subject: [PATCH 3/9] fix: linting --- plugins/catalog-backend/src/service/standaloneServer.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 51fc8be829..79cf72df4d 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -15,8 +15,7 @@ */ import { Server } from 'http'; -import { Logger, log } from 'winston'; -import knex from 'knex'; +import { Logger } from 'winston'; import { createStandaloneApplication } from './standaloneApplication'; import { DatabaseEntitiesCatalog } from '../catalog/DatabaseEntitiesCatalog'; import { DatabaseManager } from '../database/DatabaseManager'; From 64673eaeba44b6ddb3b43123b0298e8b12083d96 Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Thu, 4 Jun 2020 14:03:54 +0200 Subject: [PATCH 4/9] fix: mock data script --- plugins/catalog-backend/package.json | 2 +- plugins/catalog-backend/scripts/mock-db | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 436afbfcd2..253959c61b 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -13,7 +13,7 @@ "prepack": "backstage-cli prepack", "postpack": "backstage-cli postpack", "clean": "backstage-cli clean", - "mock-db": "sh ./scripts/mock-db" + "mock-data": "./scripts/mock-data" }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", diff --git a/plugins/catalog-backend/scripts/mock-db b/plugins/catalog-backend/scripts/mock-db index af5de1cb1b..7edc8c422b 100644 --- a/plugins/catalog-backend/scripts/mock-db +++ b/plugins/catalog-backend/scripts/mock-db @@ -1,12 +1,9 @@ -curl --location --request POST 'localhost:3003/locations' \ ---header 'Content-Type: application/json' \ ---data-raw '{ - "type": "github", - "target": "https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml" -}' -curl --location --request POST 'localhost:3003/locations' \ +#!/usr/bin/env sh +curl \ +--location \ +--request POST 'localhost:3003/locations' \ --header 'Content-Type: application/json' \ --data-raw '{ "type": "github", "target": "https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/two_components.yaml" -}' \ No newline at end of file +}' From fd635a80a0e926c93bd18f1176caac7a0de30ee2 Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Thu, 4 Jun 2020 14:06:57 +0200 Subject: [PATCH 5/9] fix: improve readability --- plugins/catalog-backend/src/database/DatabaseManager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 349df0b114..f466040c3f 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -35,19 +35,19 @@ export class DatabaseManager { public static async createInMemoryDatabase( logger: Logger, ): Promise { - const database = Knex({ + const knex = Knex({ client: 'sqlite3', connection: ':memory:', useNullAsDefault: true, }); - database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { + knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { resource.run('PRAGMA foreign_keys = ON', () => {}); }); - await database.migrate.latest({ + await knex.migrate.latest({ directory: path.resolve(__dirname, 'migrations'), loadExtensions: ['.js'], }); - return new CommonDatabase(database, logger); + return new CommonDatabase(knex, logger); } } From dd37c0dd3349b6103d64f9e8fc9599a8ebd2deba Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Thu, 4 Jun 2020 14:11:09 +0200 Subject: [PATCH 6/9] fix: get rid of obsolete and duplicated code --- .../src/database/DatabaseManager.ts | 7 +------ .../src/components/CatalogTable/CatalogTable.tsx | 14 +------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index f466040c3f..8ba1292549 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -43,11 +43,6 @@ export class DatabaseManager { knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { resource.run('PRAGMA foreign_keys = ON', () => {}); }); - - await knex.migrate.latest({ - directory: path.resolve(__dirname, 'migrations'), - loadExtensions: ['.js'], - }); - return new CommonDatabase(knex, logger); + return DatabaseManager.createDatabase(knex, logger); } } diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx index 35bc6c7d7c..f5fa45dc7e 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx @@ -17,7 +17,6 @@ import React, { FC } from 'react'; import { Component } from '../../data/component'; import { InfoCard, Progress, Table, TableColumn } from '@backstage/core'; import { Typography, Link } from '@material-ui/core'; -import EditIcon from '@material-ui/icons/Edit'; const columns: TableColumn[] = [ { @@ -67,18 +66,7 @@ const CatalogTable: FC = ({ columns={columns} options={{ paging: false }} title={`${titlePreamble} (${(components && components.length) || 0})`} - data={components.map(({ kind, name, description }) => ({ - kind, - name, - description: ( -
- {description} - - - -
- ), - }))} + data={components} /> ); }; From 443faf58b0e046c7658ee7bcf942e0f79d9b3b36 Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Thu, 4 Jun 2020 14:22:30 +0200 Subject: [PATCH 7/9] fix: rename database to knex for better readability --- plugins/catalog-backend/src/database/DatabaseManager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 8ba1292549..f0fbe92600 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -22,14 +22,14 @@ import { Database } from './types'; export class DatabaseManager { public static async createDatabase( - database: Knex, + knex: Knex, logger: Logger, ): Promise { - await database.migrate.latest({ + await knex.migrate.latest({ directory: path.resolve(__dirname, 'migrations'), loadExtensions: ['.js'], }); - return new CommonDatabase(database, logger); + return new CommonDatabase(knex, logger); } public static async createInMemoryDatabase( From 9922c4f5bc5bf410c0d9117e253f8d21d113382a Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Thu, 4 Jun 2020 14:23:32 +0200 Subject: [PATCH 8/9] fix: rename mock db script --- plugins/catalog-backend/scripts/mock-db | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 plugins/catalog-backend/scripts/mock-db diff --git a/plugins/catalog-backend/scripts/mock-db b/plugins/catalog-backend/scripts/mock-db deleted file mode 100644 index 7edc8c422b..0000000000 --- a/plugins/catalog-backend/scripts/mock-db +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env sh -curl \ ---location \ ---request POST 'localhost:3003/locations' \ ---header 'Content-Type: application/json' \ ---data-raw '{ - "type": "github", - "target": "https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/two_components.yaml" -}' From 109e5cf7c29c3f47cc57024f136b455506f609ca Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Thu, 4 Jun 2020 14:30:52 +0200 Subject: [PATCH 9/9] fix: add mock data script --- plugins/catalog-backend/scripts/mock-data | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 plugins/catalog-backend/scripts/mock-data diff --git a/plugins/catalog-backend/scripts/mock-data b/plugins/catalog-backend/scripts/mock-data new file mode 100644 index 0000000000..7edc8c422b --- /dev/null +++ b/plugins/catalog-backend/scripts/mock-data @@ -0,0 +1,9 @@ +#!/usr/bin/env sh +curl \ +--location \ +--request POST 'localhost:3003/locations' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "type": "github", + "target": "https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/two_components.yaml" +}'