diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index fb2814d640..e5c8c296f6 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -15,12 +15,28 @@ */ import { - DatabaseCatalog, + DatabaseItemsCatalog, + DatabaseLocationsCatalog, + DatabaseManager, createRouter, + runPeriodically, + LocationReaders, + DescriptorParsers, } from '@backstage/plugin-catalog-backend'; import { PluginEnvironment } from '../types'; -export default async function({ logger, database }: PluginEnvironment) { - const catalog = await DatabaseCatalog.create(database, logger); - return await createRouter({ catalog, logger }); +export default async function ({ logger, database }: PluginEnvironment) { + const reader = LocationReaders.create(); + const parser = DescriptorParsers.create(); + + const db = await DatabaseManager.createDatabase(database); + runPeriodically( + () => DatabaseManager.refreshLocations(db, reader, parser, logger), + 10000, + ); + + const itemsCatalog = new DatabaseItemsCatalog(db); + const locationsCatalog = new DatabaseLocationsCatalog(db); + + return await createRouter({ itemsCatalog, locationsCatalog, logger }); } diff --git a/plugins/catalog-backend/fixtures/component1.yaml b/plugins/catalog-backend/fixtures/one_component.yaml similarity index 81% rename from plugins/catalog-backend/fixtures/component1.yaml rename to plugins/catalog-backend/fixtures/one_component.yaml index b0b54e0902..e25d9bfdc2 100644 --- a/plugins/catalog-backend/fixtures/component1.yaml +++ b/plugins/catalog-backend/fixtures/one_component.yaml @@ -1,6 +1,6 @@ apiVersion: catalog.backstage.io/v1 kind: Component metadata: - name: component1 + name: component3 spec: type: service diff --git a/plugins/catalog-backend/fixtures/two_components.yaml b/plugins/catalog-backend/fixtures/two_components.yaml new file mode 100644 index 0000000000..83c2aad6ea --- /dev/null +++ b/plugins/catalog-backend/fixtures/two_components.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: catalog.backstage.io/v1 +kind: Component +metadata: + name: component1 +spec: + type: service +--- +apiVersion: catalog.backstage.io/v1 +kind: Component +metadata: + name: component2 +spec: + type: service diff --git a/plugins/catalog-backend/src/catalog/CatalogLogic.test.ts b/plugins/catalog-backend/src/catalog/CatalogLogic.test.ts deleted file mode 100644 index 5234136b55..0000000000 --- a/plugins/catalog-backend/src/catalog/CatalogLogic.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { PassThrough } from 'stream'; -import winston from 'winston'; -import { CatalogLogic } from './CatalogLogic'; -import { Catalog } from './types'; - -describe('CatalogLogic', () => { - const logger = winston.createLogger({ - transports: [new winston.transports.Stream({ stream: new PassThrough() })], - }); - - describe('refreshLocations', () => { - it('works with no locations added', async () => { - const catalog = ({ - addOrUpdateComponent: jest.fn(), - locations: jest.fn().mockResolvedValue([]), - } as unknown) as Catalog; - const locationReader = jest.fn(); - - await expect( - CatalogLogic.refreshLocations(catalog, locationReader, logger), - ).resolves.toBeUndefined(); - expect(locationReader).not.toHaveBeenCalled(); - }); - - it('can update a single location', async () => { - const catalog = ({ - addOrUpdateComponent: jest.fn(), - locations: jest.fn().mockResolvedValue([ - { - id: '123', - type: 'some', - target: 'thing', - }, - ]), - } as unknown) as Catalog; - - const locationReader = jest.fn().mockResolvedValue([ - { - name: 'c1', - }, - ]); - - await expect( - CatalogLogic.refreshLocations(catalog, locationReader, logger), - ).resolves.toBeUndefined(); - expect(locationReader).toHaveBeenCalledTimes(1); - expect(locationReader).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ type: 'some' }), - ); - expect(catalog.addOrUpdateComponent).toHaveBeenCalledTimes(1); - expect(catalog.addOrUpdateComponent).toHaveBeenNthCalledWith( - 1, - '123', - expect.objectContaining({ name: 'c1' }), - ); - }); - }); -}); diff --git a/plugins/catalog-backend/src/catalog/CatalogLogic.ts b/plugins/catalog-backend/src/catalog/CatalogLogic.ts deleted file mode 100644 index 686aba0c40..0000000000 --- a/plugins/catalog-backend/src/catalog/CatalogLogic.ts +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Logger } from 'winston'; -import { LocationReader } from '../ingestion/types'; -import { Catalog } from './types'; - -export class CatalogLogic { - public static startRefreshLoop( - catalog: Catalog, - reader: LocationReader, - logger: Logger, - ): () => void { - let cancel: () => void; - let cancelled = false; - const cancellationPromise = new Promise((resolve) => { - cancel = () => { - resolve(); - cancelled = true; - }; - }); - - const startRefresh = async () => { - while (!cancelled) { - await CatalogLogic.refreshLocations(catalog, reader, logger); - await Promise.race([ - new Promise((resolve) => setTimeout(resolve, 10000)), - cancellationPromise, - ]); - } - }; - startRefresh(); - - return cancel!; - } - - public static async refreshLocations( - catalog: Catalog, - reader: LocationReader, - logger: Logger, - ): Promise { - const locations = await catalog.locations(); - for (const location of locations) { - try { - logger.debug(`Attempting refresh of location: ${location.id}`); - const componentDescriptors = await reader(location); - for (const componentDescriptor of componentDescriptors) { - await catalog.addOrUpdateComponent(location.id, componentDescriptor); - } - } catch (e) { - logger.debug(`Failed to update location "${location.id}", ${e}`); - } - } - } -} diff --git a/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts deleted file mode 100644 index 87ff5a503f..0000000000 --- a/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { NotFoundError } from '@backstage/backend-common'; -import Knex from 'knex'; -import path from 'path'; -import { v4 as uuidv4 } from 'uuid'; -import { Logger } from 'winston'; -import { ComponentDescriptor } from '../descriptors'; -import { readLocation } from '../ingestion'; -import { CatalogLogic } from './CatalogLogic'; -import { AddLocationRequest, Catalog, Component, Location } from './types'; - -export class DatabaseCatalog implements Catalog { - static async create( - database: Knex, - logger: Logger, - ): Promise { - await database.migrate.latest({ - directory: path.resolve(__dirname, '..', 'migrations'), - loadExtensions: ['.js'], - }); - - const databaseCatalog = new DatabaseCatalog(database, logger); - CatalogLogic.startRefreshLoop(databaseCatalog, readLocation, logger); - - return databaseCatalog; - } - - constructor( - private readonly database: Knex, - private readonly logger: Logger, - ) {} - - async addOrUpdateComponent( - locationId: string, - descriptor: ComponentDescriptor, - ): Promise { - const component: Component = { - name: descriptor.metadata.name, - }; - - await this.database.transaction(async (tx) => { - // TODO(freben): Currently, several locations can compete for the same component - const count = await tx('components') - .where({ name: component.name }) - .update({ ...component, locationId }); - if (!count) { - await tx('components').insert({ - ...component, - id: uuidv4(), - locationId, - }); - } - }); - } - - async components(): Promise { - return await this.database('components').select(); - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async component(name: string): Promise { - const items = await this.database('components').where({ name }).select(); - if (!items.length) { - throw new NotFoundError(`Found no component with name ${name}`); - } - return items[0]; - } - - async addLocation(location: AddLocationRequest): Promise { - const id = uuidv4(); - const { type, target } = location; - await this.database('locations').insert({ id, type, target }); - return await this.location(id); - } - - async removeLocation(id: string): Promise { - const result = await this.database('locations').where({ id }).del(); - - if (!result) { - throw new NotFoundError(`Found no location with ID ${id}`); - } - } - - async location(id: string): Promise { - const items = await this.database('locations').where({ id }).select(); - if (!items.length) { - throw new NotFoundError(`Found no location with ID ${id}`); - } - return items[0]; - } - - async locations(): Promise { - return await this.database('locations').select(); - } -} diff --git a/plugins/catalog-backend/src/ingestion/util.ts b/plugins/catalog-backend/src/catalog/DatabaseItemsCatalog.ts similarity index 57% rename from plugins/catalog-backend/src/ingestion/util.ts rename to plugins/catalog-backend/src/catalog/DatabaseItemsCatalog.ts index ede6b1b21a..6221ba4d4c 100644 --- a/plugins/catalog-backend/src/ingestion/util.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseItemsCatalog.ts @@ -14,17 +14,19 @@ * limitations under the License. */ -import { Location } from '../catalog/types'; -import { ComponentDescriptor } from '../descriptors'; -import { readFileLocation } from './fileLocation'; +import { Database } from '../database'; +import { Component, ItemsCatalog } from './types'; -export async function readLocation( - location: Location, -): Promise { - switch (location.type) { - case 'file': - return await readFileLocation(location.target); - default: - throw new Error(`Unknown type "${location.type}"`); +export class DatabaseItemsCatalog implements ItemsCatalog { + constructor(private readonly database: Database) {} + + async components(): Promise { + const items = await this.database.components(); + return items; + } + + async component(name: string): Promise { + const item = await this.database.component(name); + return item; } } diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts new file mode 100644 index 0000000000..e9f1512074 --- /dev/null +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Database } from '../database'; +import { AddLocation, Location, LocationsCatalog } from './types'; + +export class DatabaseLocationsCatalog implements LocationsCatalog { + constructor(private readonly database: Database) {} + + async addLocation(location: AddLocation): Promise { + const added = await this.database.addLocation(location); + return added; + } + + async removeLocation(id: string): Promise { + await this.database.removeLocation(id); + } + + async locations(): Promise { + const items = await this.database.locations(); + return items; + } + + async location(id: string): Promise { + const item = await this.location(id); + return item; + } +} diff --git a/plugins/catalog-backend/src/catalog/StaticCatalog.ts b/plugins/catalog-backend/src/catalog/StaticCatalog.ts deleted file mode 100644 index 28d6cff9a3..0000000000 --- a/plugins/catalog-backend/src/catalog/StaticCatalog.ts +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { NotFoundError } from '@backstage/backend-common'; -import { v4 as uuidv4 } from 'uuid'; -import { ComponentDescriptor } from '../descriptors'; -import { AddLocationRequest, Catalog, Component, Location } from './types'; - -export class StaticCatalog implements Catalog { - private _components: Component[]; - private _locations: Location[]; - - constructor(components: Component[], locations: Location[]) { - this._components = components; - this._locations = locations; - } - - async addOrUpdateComponent( - locationId: string, - descriptor: ComponentDescriptor, - ): Promise { - this._components = this._components - .filter((c) => c.name !== descriptor.metadata.name) - .concat([{ name: descriptor.metadata.name }]); - } - - async components(): Promise { - return this._components.slice(); - } - - async component(name: string): Promise { - const item = this._components.find((i) => i.name === name); - if (!item) { - throw new NotFoundError(`Found no component with name ${name}`); - } - return item; - } - - async addLocation(location: AddLocationRequest): Promise { - const l = { id: uuidv4(), type: location.type, target: location.target }; - this._locations.push(l); - return l; - } - - async removeLocation(id: string): Promise { - this._locations = this._locations.filter((l) => l.id !== id); - } - - async location(id: string): Promise { - const location = this._locations.find((l) => l.id === id); - if (!location) { - throw new NotFoundError(`Found no location with ID ${id}`); - } - return location; - } - - async locations(): Promise { - return this._locations; - } -} diff --git a/plugins/catalog-backend/src/catalog/StaticItemsCatalog.ts b/plugins/catalog-backend/src/catalog/StaticItemsCatalog.ts new file mode 100644 index 0000000000..0544ea9ba8 --- /dev/null +++ b/plugins/catalog-backend/src/catalog/StaticItemsCatalog.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NotFoundError } from '@backstage/backend-common'; +import { Component, ItemsCatalog } from './types'; + +export class StaticItemsCatalog implements ItemsCatalog { + private _components: Component[]; + + constructor(components: Component[]) { + this._components = components; + } + + async components(): Promise { + return this._components.slice(); + } + + async component(name: string): Promise { + const item = this._components.find((i) => i.name === name); + if (!item) { + throw new NotFoundError(`Found no component with name ${name}`); + } + return item; + } +} diff --git a/plugins/catalog-backend/src/catalog/index.ts b/plugins/catalog-backend/src/catalog/index.ts index b4bc07c6bb..5db68ce4e5 100644 --- a/plugins/catalog-backend/src/catalog/index.ts +++ b/plugins/catalog-backend/src/catalog/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ -export * from './DatabaseCatalog'; -export * from './StaticCatalog'; +export * from './DatabaseItemsCatalog'; +export * from './DatabaseLocationsCatalog'; +export * from './StaticItemsCatalog'; export * from './types'; diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 3a7b685003..81b31a465c 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -15,38 +15,46 @@ */ import * as yup from 'yup'; -import { ComponentDescriptor } from '../descriptors'; + +// +// Items +// export type Component = { + id: string; + locationId?: string; name: string; }; +export type ItemsCatalog = { + components(): Promise; + component(id: string): Promise; +}; + +// +// Locations +// + export type Location = { id: string; type: string; target: string; }; -export type AddLocationRequest = { +export type AddLocation = { type: string; target: string; }; -export const addLocationRequestShape: yup.Schema = yup +export const addLocationSchema: yup.Schema = yup .object({ type: yup.string().required(), target: yup.string().required(), }) .noUnknown(); -export type Catalog = { - addOrUpdateComponent( - locationId: string, - descriptor: ComponentDescriptor, - ): Promise; - components(): Promise; - component(id: string): Promise; - addLocation(location: AddLocationRequest): Promise; +export type LocationsCatalog = { + addLocation(location: AddLocation): Promise; removeLocation(id: string): Promise; locations(): Promise; location(id: string): Promise; diff --git a/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts b/plugins/catalog-backend/src/database/Database.test.ts similarity index 58% rename from plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts rename to plugins/catalog-backend/src/database/Database.test.ts index 98a5e9acd4..c375d13fdb 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts +++ b/plugins/catalog-backend/src/database/Database.test.ts @@ -16,11 +16,10 @@ import knex from 'knex'; import path from 'path'; -import { PassThrough } from 'stream'; -import winston from 'winston'; -import { DatabaseCatalog } from './DatabaseCatalog'; +import { Database } from './Database'; +import { AddDatabaseLocation, DatabaseLocation } from './types'; -describe('DatabaseCatalog', () => { +describe('Database', () => { const database = knex({ client: 'sqlite3', connection: ':memory:', @@ -30,33 +29,33 @@ describe('DatabaseCatalog', () => { resource.run('PRAGMA foreign_keys = ON', () => {}); }); - const logger = winston.createLogger({ - transports: [new winston.transports.Stream({ stream: new PassThrough() })], - }); - beforeEach(async () => { await database.migrate.latest({ - directory: path.resolve(__dirname, '..', 'migrations'), + directory: path.resolve(__dirname, 'migrations'), loadExtensions: ['.ts'], }); }); it('manages locations', async () => { - const catalog = new DatabaseCatalog(database, logger); - const input = { type: 'a', target: 'b' }; - const output = { id: expect.anything(), type: 'a', target: 'b' }; + const db = new Database(database); + const input: AddDatabaseLocation = { type: 'a', target: 'b' }; + const output: DatabaseLocation = { + id: expect.anything(), + type: 'a', + target: 'b', + }; - await catalog.addLocation(input); + await db.addLocation(input); - const locations = await catalog.locations(); + const locations = await db.locations(); expect(locations).toEqual([output]); - const location = await catalog.location(locations[0].id); + const location = await db.location(locations[0].id); expect(location).toEqual(output); - await catalog.removeLocation(locations[0].id); + await db.removeLocation(locations[0].id); - await expect(catalog.locations()).resolves.toEqual([]); - await expect(catalog.location(locations[0].id)).rejects.toThrow( + await expect(db.locations()).resolves.toEqual([]); + await expect(db.location(locations[0].id)).rejects.toThrow( /Found no location/, ); }); diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts new file mode 100644 index 0000000000..ed11f091fe --- /dev/null +++ b/plugins/catalog-backend/src/database/Database.ts @@ -0,0 +1,96 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NotFoundError } from '@backstage/backend-common'; +import Knex from 'knex'; +import { v4 as uuidv4 } from 'uuid'; +import { + AddDatabaseComponent, + AddDatabaseLocation, + DatabaseComponent, + DatabaseLocation, +} from './types'; + +export class Database { + constructor(private readonly database: Knex) {} + + async addOrUpdateComponent(component: AddDatabaseComponent): Promise { + await this.database.transaction(async (tx) => { + // TODO(freben): Currently, several locations can compete for the same component + // TODO(freben): If locationId is unset in the input, it won't be overwritten - should we instead replace with null? + const count = await tx('components') + .where({ name: component.name }) + .update({ ...component }); + if (!count) { + await tx('components').insert({ + ...component, + id: uuidv4(), + }); + } + }); + } + + async components(): Promise { + return await this.database('components') + .orderBy('name') + .select(); + } + + async component(name: string): Promise { + const items = await this.database('components') + .where({ name }) + .select(); + if (!items.length) { + throw new NotFoundError(`Found no component with name ${name}`); + } + return items[0]; + } + + async addLocation(location: AddDatabaseLocation): Promise { + const id = uuidv4(); + const { type, target } = location; + await this.database('locations').insert({ + id, + type, + target, + }); + return await this.location(id); + } + + async removeLocation(id: string): Promise { + const result = await this.database('locations') + .where({ id }) + .del(); + + if (!result) { + throw new NotFoundError(`Found no location with ID ${id}`); + } + } + + async location(id: string): Promise { + const items = await this.database('locations') + .where({ id }) + .select(); + if (!items.length) { + throw new NotFoundError(`Found no location with ID ${id}`); + } + return items[0]; + } + + async locations(): Promise { + return await this.database('locations').select(); + } +} diff --git a/plugins/catalog-backend/src/database/DatabaseManager.test.ts b/plugins/catalog-backend/src/database/DatabaseManager.test.ts new file mode 100644 index 0000000000..77c9a463e2 --- /dev/null +++ b/plugins/catalog-backend/src/database/DatabaseManager.test.ts @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PassThrough } from 'stream'; +import winston from 'winston'; +import { + ComponentDescriptor, + DescriptorParser, + LocationReader, +} from '../ingestion'; +import { Database } from './Database'; +import { DatabaseManager } from './DatabaseManager'; +import { DatabaseLocation } from './types'; + +describe('DatabaseManager', () => { + const logger = winston.createLogger({ + transports: [new winston.transports.Stream({ stream: new PassThrough() })], + }); + + describe('refreshLocations', () => { + it('works with no locations added', async () => { + const db = ({ + addOrUpdateComponent: jest.fn(), + locations: jest.fn().mockResolvedValue([]), + } as unknown) as Database; + const reader: LocationReader = { + read: jest.fn(), + }; + const parser: DescriptorParser = { + parse: jest.fn(), + }; + + await expect( + DatabaseManager.refreshLocations(db, reader, parser, logger), + ).resolves.toBeUndefined(); + expect(reader.read).not.toHaveBeenCalled(); + expect(parser.parse).not.toHaveBeenCalled(); + }); + + it('can update a single location', async () => { + const db = ({ + addOrUpdateComponent: jest.fn(), + locations: jest.fn(() => + Promise.resolve([ + { + id: '123', + type: 'some', + target: 'thing', + } as DatabaseLocation, + ]), + ), + } as unknown) as Database; + + const desc: ComponentDescriptor = { + metadata: { name: 'c1' }, + spec: { type: 'service' }, + }; + const reader: LocationReader = { + read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), + }; + const parser: DescriptorParser = { + parse: jest.fn(() => + Promise.resolve({ kind: 'Component', component: desc }), + ), + }; + + await expect( + DatabaseManager.refreshLocations(db, reader, parser, logger), + ).resolves.toBeUndefined(); + expect(reader.read).toHaveBeenCalledTimes(1); + expect(reader.read).toHaveBeenNthCalledWith(1, 'some', 'thing'); + expect(db.addOrUpdateComponent).toHaveBeenCalledTimes(1); + expect(db.addOrUpdateComponent).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ locationId: '123', name: 'c1' }), + ); + }); + }); +}); diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts new file mode 100644 index 0000000000..03ed066d49 --- /dev/null +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Knex from 'knex'; +import path from 'path'; +import { Logger } from 'winston'; +import { DescriptorParser, LocationReader } from '../ingestion'; +import { Database } from './Database'; +import { AddDatabaseComponent } from './types'; + +export class DatabaseManager { + public static async createDatabase(database: Knex): Promise { + await database.migrate.latest({ + directory: path.resolve(__dirname, 'migrations'), + loadExtensions: ['.js'], + }); + return new Database(database); + } + + public static async refreshLocations( + database: Database, + reader: LocationReader, + parser: DescriptorParser, + logger: Logger, + ): Promise { + const locations = await database.locations(); + for (const location of locations) { + try { + logger.debug( + `Refreshing location ${location.id} type "${location.type}" target "${location.target}"`, + ); + + const readerOutput = await reader.read(location.type, location.target); + for (const readerItem of readerOutput) { + if (readerItem.type === 'error') { + logger.debug(readerItem.error); + continue; + } + + const parserOutput = await parser.parse(readerItem.data); + if (parserOutput.kind === 'Component') { + const component = parserOutput.component; + const dbc: AddDatabaseComponent = { + locationId: location.id, + name: component.metadata.name, + }; + await database.addOrUpdateComponent(dbc); + } + } + } catch (e) { + // TODO(freben): Store trace log of these events, or at least the + // latest status, per location + logger.debug(`Failed to refresh location ${location.id}, ${e}`); + } + } + } +} diff --git a/plugins/catalog-backend/src/index.test.ts b/plugins/catalog-backend/src/database/index.ts similarity index 84% rename from plugins/catalog-backend/src/index.test.ts rename to plugins/catalog-backend/src/database/index.ts index b3e2f19771..616808fb67 100644 --- a/plugins/catalog-backend/src/index.test.ts +++ b/plugins/catalog-backend/src/database/index.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -describe('test', () => { - it('unbreaks the test runner', () => { - expect(true).toBeTruthy(); - }); -}); +export * from './Database'; +export * from './DatabaseManager'; +export * from './types'; diff --git a/plugins/catalog-backend/src/migrations/20200511113813_init.ts b/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts similarity index 100% rename from plugins/catalog-backend/src/migrations/20200511113813_init.ts rename to plugins/catalog-backend/src/database/migrations/20200511113813_init.ts diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts new file mode 100644 index 0000000000..ef7807b524 --- /dev/null +++ b/plugins/catalog-backend/src/database/types.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as yup from 'yup'; + +export type DatabaseComponent = { + id: string; + locationId?: string; + name: string; +}; + +export type AddDatabaseComponent = { + locationId?: string; + name: string; +}; + +export const addDatabaseComponentSchema: yup.Schema = yup + .object({ + locationId: yup.string().optional(), + name: yup.string().required(), + }) + .noUnknown(); + +export type DatabaseLocation = { + id: string; + type: string; + target: string; +}; + +export type AddDatabaseLocation = { + type: string; + target: string; +}; + +export const addDatabaseLocationSchema: yup.Schema = yup + .object({ + type: yup.string().required(), + target: yup.string().required(), + }) + .noUnknown(); diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index 16f59d2c17..89fd4d37ed 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -15,4 +15,7 @@ */ export * from './catalog'; +export * from './database'; +export * from './ingestion'; export * from './service/router'; +export * from './util'; diff --git a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts b/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts new file mode 100644 index 0000000000..37f7ac1233 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ComponentDescriptorV1Parser } from './descriptors/ComponentDescriptorV1Parser'; +import { parseDescriptorEnvelope } from './descriptors/DescriptorEnvelope'; +import { EnvelopeParser } from './descriptors/types'; +import { DescriptorParser, ParserOutput } from './types'; + +export class DescriptorParsers implements DescriptorParser { + static create(): DescriptorParser { + return new DescriptorParsers([new ComponentDescriptorV1Parser()]); + } + + constructor(private readonly parsers: EnvelopeParser[]) {} + + async parse(descriptor: object): Promise { + const envelope = await parseDescriptorEnvelope(descriptor); + + for (const parser of this.parsers) { + const parsed = await parser.tryParse(envelope); + if (parsed) { + return parsed; + } + } + + throw new Error( + `Unsupported object ${envelope.apiVersion}, ${envelope.kind}`, + ); + } +} diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts new file mode 100644 index 0000000000..676c318086 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FileLocationSource } from './sources/FileLocationSource'; +import { LocationSource } from './sources/types'; +import { LocationReader, ReaderOutput } from './types'; + +export class LocationReaders implements LocationReader { + static create(): LocationReader { + return new LocationReaders({ + file: new FileLocationSource(), + }); + } + + constructor(private readonly sources: Record) {} + + async read(type: string, target: string): Promise { + const source = this.sources[type]; + if (!source) { + throw new Error(`Unknown location type ${type}`); + } + + return source.read(target); + } +} diff --git a/plugins/catalog-backend/src/descriptors/component.ts b/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1Parser.ts similarity index 51% rename from plugins/catalog-backend/src/descriptors/component.ts rename to plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1Parser.ts index abe75a0050..bb08dd15e4 100644 --- a/plugins/catalog-backend/src/descriptors/component.ts +++ b/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1Parser.ts @@ -15,9 +15,11 @@ */ import * as yup from 'yup'; -import { DescriptorEnvelope } from './envelope'; +import { ParserOutput } from '../types'; +import { DescriptorEnvelope } from './DescriptorEnvelope'; +import { EnvelopeParser } from './types'; -export type ComponentDescriptor = { +export type ComponentDescriptorV1 = { metadata: { name: string; }; @@ -26,11 +28,7 @@ export type ComponentDescriptor = { }; }; -const componentDescriptorSchema: yup.Schema = yup.object({ - kind: yup - .string() - .required() - .matches(/^Component$/), +const schema: yup.Schema = yup.object({ metadata: yup.object({ name: yup.string().required(), }), @@ -39,17 +37,27 @@ const componentDescriptorSchema: yup.Schema = yup.object({ }), }); -export async function parseComponentDescriptor( - envelope: DescriptorEnvelope, -): Promise { - let componentDescriptor; - try { - componentDescriptor = await componentDescriptorSchema.validate(envelope, { - strict: true, - }); - } catch (e) { - throw new Error(`Malformed component, ${e}`); - } +export class ComponentDescriptorV1Parser implements EnvelopeParser { + async tryParse( + envelope: DescriptorEnvelope, + ): Promise { + if ( + envelope.apiVersion !== 'catalog.backstage.io/v1' || + envelope.kind !== 'Component' + ) { + return undefined; + } - return [componentDescriptor]; + let component; + try { + component = await schema.validate(envelope, { strict: true }); + } catch (e) { + throw new Error(`Malformed component, ${e}`); + } + + return { + kind: 'Component', + component, + }; + } } diff --git a/plugins/catalog-backend/src/descriptors/envelope.ts b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelope.ts similarity index 80% rename from plugins/catalog-backend/src/descriptors/envelope.ts rename to plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelope.ts index dd78d860ee..90193bcc7e 100644 --- a/plugins/catalog-backend/src/descriptors/envelope.ts +++ b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelope.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import yaml from 'yaml'; import * as yup from 'yup'; export type DescriptorEnvelope = { @@ -24,6 +23,7 @@ export type DescriptorEnvelope = { spec?: object; }; +// The schema of the envelope that's common to all versions/kinds const descriptorEnvelopeSchema: yup.Schema = yup .object({ apiVersion: yup.string().required(), @@ -33,20 +33,12 @@ const descriptorEnvelopeSchema: yup.Schema = yup }) .noUnknown(); +// Validate some raw structured data as a descriptor envelope export async function parseDescriptorEnvelope( - rawYaml: string, + data: object, ): Promise { - let descriptor; try { - descriptor = yaml.parse(rawYaml); - } catch (e) { - throw new Error(`Malformed YAML, ${e}`); - } - - try { - return await descriptorEnvelopeSchema.validate(descriptor, { - strict: true, - }); + return await descriptorEnvelopeSchema.validate(data, { strict: true }); } catch (e) { throw new Error(`Malformed envelope, ${e}`); } diff --git a/plugins/catalog-backend/src/ingestion/descriptors/types.ts b/plugins/catalog-backend/src/ingestion/descriptors/types.ts new file mode 100644 index 0000000000..64900c9ad4 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptors/types.ts @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ParserOutput } from '../types'; +import { DescriptorEnvelope } from './DescriptorEnvelope'; + +export type EnvelopeParser = { + /** + * Parses and validates a single envelope into its materialized type. + * + * @param envelope A descriptor envelope + * @returns A materialized type, or undefined if the given version/kind is + * not handled by this parser + * @throws An Error if the type was not properly formatted + */ + tryParse(envelope: DescriptorEnvelope): Promise; +}; diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index e96be6822a..ca6f2dd2be 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -14,6 +14,6 @@ * limitations under the License. */ -export * from './fileLocation'; +export * from './DescriptorParsers'; +export * from './LocationReaders'; export * from './types'; -export * from './util'; diff --git a/plugins/catalog-backend/src/ingestion/fileLocation.ts b/plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts similarity index 53% rename from plugins/catalog-backend/src/ingestion/fileLocation.ts rename to plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts index e856e701d3..67d889e7e5 100644 --- a/plugins/catalog-backend/src/ingestion/fileLocation.ts +++ b/plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts @@ -15,21 +15,23 @@ */ import fs from 'fs-extra'; -import { ComponentDescriptor, parseDescriptor } from '../descriptors'; +import { ReaderOutput } from '../types'; +import { LocationSource } from './types'; +import { readDescriptorYaml } from './util'; -export async function readFileLocation( - target: string, -): Promise { - let rawYaml; - try { - rawYaml = await fs.readFile(target, 'utf8'); - } catch (e) { - throw new Error(`Unable to read "${target}", ${e}`); - } +export class FileLocationSource implements LocationSource { + async read(target: string): Promise { + let rawYaml; + try { + rawYaml = await fs.readFile(target, 'utf8'); + } catch (e) { + throw new Error(`Unable to read "${target}", ${e}`); + } - try { - return parseDescriptor(rawYaml); - } catch (e) { - throw new Error(`Malformed descriptor at "${target}", ${e}`); + try { + return readDescriptorYaml(rawYaml); + } catch (e) { + throw new Error(`Malformed descriptor at "${target}", ${e}`); + } } } diff --git a/plugins/catalog-backend/src/descriptors/util.ts b/plugins/catalog-backend/src/ingestion/sources/types.ts similarity index 60% rename from plugins/catalog-backend/src/descriptors/util.ts rename to plugins/catalog-backend/src/ingestion/sources/types.ts index 20118c7d97..c078693a19 100644 --- a/plugins/catalog-backend/src/descriptors/util.ts +++ b/plugins/catalog-backend/src/ingestion/sources/types.ts @@ -14,13 +14,15 @@ * limitations under the License. */ -import { ComponentDescriptor, parseComponentDescriptor } from './component'; -import { parseDescriptorEnvelope } from './envelope'; +import { ReaderOutput } from '../types'; -// TODO(freben): Temporary helper that ignores the kind -export async function parseDescriptor( - rawYaml: string, -): Promise { - const env = await parseDescriptorEnvelope(rawYaml); - return await parseComponentDescriptor(env); -} +export type LocationSource = { + /** + * Reads the contents of a single location. + * + * @param target The location target to read + * @returns The parsed contents, as an array of unverified descriptors + * @throws An error if the location target could not be read + */ + read(target: string): Promise; +}; diff --git a/plugins/catalog-backend/src/ingestion/sources/util.ts b/plugins/catalog-backend/src/ingestion/sources/util.ts new file mode 100644 index 0000000000..cccbeb92b3 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/sources/util.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { ReaderOutput } from '../types'; + +export function readDescriptorYaml(data: string): ReaderOutput[] { + let documents; + try { + documents = yaml.parseAllDocuments(data); + } catch (e) { + throw new Error(`Could not parse YAML data, ${e}`); + } + + const result: ReaderOutput[] = []; + + for (const document of documents) { + if (document.contents) { + if (document.errors?.length) { + result.push({ + type: 'error', + error: new Error(`Malformed YAML document, ${document.errors[0]}`), + }); + } else { + const json = document.toJSON(); + if (typeof json !== 'object' || Array.isArray(json)) { + result.push({ + type: 'error', + error: new Error(`Malformed descriptor, expected object at root`), + }); + } else { + result.push({ + type: 'data', + data: json, + }); + } + } + } + } + + return result; +} diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index 426188ed2c..11de959967 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -14,9 +14,39 @@ * limitations under the License. */ -import { Location } from '../catalog'; -import { ComponentDescriptor } from '../descriptors'; +import { ComponentDescriptorV1 } from './descriptors/ComponentDescriptorV1Parser'; -export type LocationReader = ( - location: Location, -) => Promise; +export type ComponentDescriptor = ComponentDescriptorV1; + +export type ParserOutput = { + kind: 'Component'; + component: ComponentDescriptor; +}; + +export type DescriptorParser = { + /** + * Parses and validates a single raw descriptor. + * + * @param descriptor A raw descriptor object + * @returns A structure describing the parsed and validated descriptor + * @throws An Error if the descriptor was malformed + */ + parse(descriptor: object): Promise; +}; + +export type ReaderOutput = + | { type: 'error'; error: Error } + | { type: 'data'; data: object }; + +export type LocationReader = { + /** + * Reads the contents of a single location. + * + * @param type The type of location to read + * @param target The location target (type-specific) + * @returns The parsed contents, as an array of unverified descriptors or + * errors where the individual documents could not be parsed. + * @throws An error if the location as a whole could not be read + */ + read(type: string, target: string): Promise; +}; diff --git a/plugins/catalog-backend/src/run.ts b/plugins/catalog-backend/src/run.ts index a2c2601258..9c086081a7 100644 --- a/plugins/catalog-backend/src/run.ts +++ b/plugins/catalog-backend/src/run.ts @@ -14,15 +14,15 @@ * limitations under the License. */ -import yn from 'yn'; import { getRootLogger } from '@backstage/backend-common'; +import yn from 'yn'; import { startStandaloneServer } from './service/standaloneServer'; const port = process.env.PLUGIN_PORT ? Number(process.env.PLUGIN_PORT) : 3003; const enableCors = yn(process.env.PLUGIN_CORS, { default: false }); const logger = getRootLogger(); -startStandaloneServer({ port, enableCors, logger }).catch(err => { +startStandaloneServer({ port, enableCors, logger }).catch((err) => { logger.error(err); process.exit(1); }); diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index 3bdbedb766..479c4c2c1f 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -14,83 +14,62 @@ * limitations under the License. */ -import { InputError } from '@backstage/backend-common'; -import express, { Request } from 'express'; +import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; -import yup from 'yup'; -import { addLocationRequestShape, Catalog } from '../catalog'; +import { addLocationSchema, ItemsCatalog, LocationsCatalog } from '../catalog'; +import { validateRequestBody } from './util'; export interface RouterOptions { - catalog: Catalog; + itemsCatalog?: ItemsCatalog; + locationsCatalog?: LocationsCatalog; logger: Logger; } -async function validateRequestBody( - req: Request, - schema: yup.Schema, -): Promise { - const contentType = req.header('content-type'); - if (!contentType) { - throw new InputError('Content-Type missing'); - } else if (!contentType.match(/^application\/json($|;)/)) { - throw new InputError('Illegal Content-Type'); - } - - const body = req.body; - if (!body) { - throw new InputError('Missing request body'); - } - - try { - await schema.validate(body, { strict: true }); - } catch (e) { - throw new InputError(`Malformed request: ${e}`); - } - - return body as T; -} - export async function createRouter( options: RouterOptions, ): Promise { - const catalog = options.catalog; + const { itemsCatalog, locationsCatalog } = options; const logger = options.logger.child({ plugin: 'catalog' }); const router = Router(); - // Components - router - .get('/components', async (req, res) => { - const components = await catalog.components(); - res.status(200).send(components); - }) - .get('/components/:id', async (req, res) => { - const { id } = req.params; - const component = await catalog.component(id); - res.status(200).send(component); - }); + if (itemsCatalog) { + // Components + router + .get('/components', async (req, res) => { + const components = await itemsCatalog.components(); + res.status(200).send(components); + }) + .get('/components/:id', async (req, res) => { + const { id } = req.params; + const component = await itemsCatalog.component(id); + res.status(200).send(component); + }); + } // Locations - router - .post('/locations', async (req, res) => { - const input = await validateRequestBody(req, addLocationRequestShape); - const output = await catalog.addLocation(input); - res.status(201).send(output); - }) - .get('/locations', async (req, res) => { - const output = await catalog.locations(); - res.status(200).send(output); - }) - .get('/locations/:id', async (req, res) => { - const { id } = req.params; - const output = await catalog.location(id); - res.status(200).send(output); - }) - .delete('/locations/:id', async (req, res) => { - const { id } = req.params; - await catalog.removeLocation(id); - res.status(200).send(); - }); + if (locationsCatalog) { + router + .post('/locations', async (req, res) => { + const input = await validateRequestBody(req, addLocationSchema); + const output = await locationsCatalog.addLocation(input); + res.status(201).send(output); + }) + .get('/locations', async (req, res) => { + const output = await locationsCatalog.locations(); + res.status(200).send(output); + }) + .get('/locations/:id', async (req, res) => { + const { id } = req.params; + const output = await locationsCatalog.location(id); + res.status(200).send(output); + }) + .delete('/locations/:id', async (req, res) => { + const { id } = req.params; + await locationsCatalog.removeLocation(id); + res.status(200).send(); + }); + } const app = express(); app.set('logger', logger); diff --git a/plugins/catalog-backend/src/service/standaloneApplication.ts b/plugins/catalog-backend/src/service/standaloneApplication.ts index da39be8410..0592b0ab42 100644 --- a/plugins/catalog-backend/src/service/standaloneApplication.ts +++ b/plugins/catalog-backend/src/service/standaloneApplication.ts @@ -24,19 +24,20 @@ import cors from 'cors'; import express from 'express'; import helmet from 'helmet'; import { Logger } from 'winston'; -import { Catalog } from '../catalog'; +import { ItemsCatalog, LocationsCatalog } from '../catalog'; import { createRouter } from './router'; export interface ApplicationOptions { enableCors: boolean; - catalog: Catalog; + itemsCatalog: ItemsCatalog; + locationsCatalog?: LocationsCatalog; logger: Logger; } export async function createStandaloneApplication( options: ApplicationOptions, ): Promise { - const { enableCors, catalog, logger } = options; + const { enableCors, itemsCatalog, locationsCatalog, logger } = options; const app = express(); app.use(helmet()); @@ -46,7 +47,7 @@ export async function createStandaloneApplication( app.use(compression()); app.use(express.json()); app.use(requestLoggingHandler()); - app.use('/', await createRouter({ catalog, logger })); + app.use('/', await createRouter({ itemsCatalog, locationsCatalog, logger })); app.use(notFoundHandler()); app.use(errorHandler()); diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index d5c46d7293..1a208eb32d 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -16,7 +16,7 @@ import { Server } from 'http'; import { Logger } from 'winston'; -import { StaticCatalog } from '../catalog'; +import { StaticItemsCatalog } from '../catalog'; import { createStandaloneApplication } from './standaloneApplication'; export interface ServerOptions { @@ -30,20 +30,15 @@ export async function startStandaloneServer( ): Promise { const logger = options.logger.child({ service: 'catalog-backend' }); - const catalog = new StaticCatalog( - [ - { name: 'component1' }, - { name: 'component2' }, - { name: 'component3' }, - { name: 'component4' }, - ], - [], - ); + const itemsCatalog = new StaticItemsCatalog([ + { id: '1', name: 'component1' }, + { id: '2', name: 'component2' }, + ]); logger.debug('Creating application...'); const app = await createStandaloneApplication({ enableCors: options.enableCors, - catalog, + itemsCatalog, logger, }); diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts new file mode 100644 index 0000000000..4c37154c48 --- /dev/null +++ b/plugins/catalog-backend/src/service/util.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { Request } from 'express'; +import yup from 'yup'; + +export async function validateRequestBody( + req: Request, + schema: yup.Schema, +): Promise { + const contentType = req.header('content-type'); + if (!contentType) { + throw new InputError('Content-Type missing'); + } else if (!contentType.match(/^application\/json($|;)/)) { + throw new InputError('Illegal Content-Type'); + } + + const body = req.body; + if (!body) { + throw new InputError('Missing request body'); + } + + try { + await schema.validate(body, { strict: true }); + } catch (e) { + throw new InputError(`Malformed request: ${e}`); + } + + return body as T; +} diff --git a/plugins/catalog-backend/src/descriptors/index.ts b/plugins/catalog-backend/src/util/index.ts similarity index 88% rename from plugins/catalog-backend/src/descriptors/index.ts rename to plugins/catalog-backend/src/util/index.ts index db3fd4c938..6f0505d8c5 100644 --- a/plugins/catalog-backend/src/descriptors/index.ts +++ b/plugins/catalog-backend/src/util/index.ts @@ -14,6 +14,4 @@ * limitations under the License. */ -export * from './component'; -export * from './envelope'; -export * from './util'; +export * from './runPeriodically'; diff --git a/plugins/catalog-backend/src/util/runPeriodically.ts b/plugins/catalog-backend/src/util/runPeriodically.ts new file mode 100644 index 0000000000..2d02287540 --- /dev/null +++ b/plugins/catalog-backend/src/util/runPeriodically.ts @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Runs a function repeatedly, with a fixed wait between invocations. + * + * Supports async functions, and silently ignores exceptions and rejections. + * + * @param fn The function to run. May return a Promise. + * @param delayMs The delay between a completed function invocation and the + * next. + * @returns A function that, when called, stops the invocation loop. + */ +export function runPeriodically(fn: () => any, delayMs: number): () => void { + let cancel: () => void; + let cancelled = false; + const cancellationPromise = new Promise((resolve) => { + cancel = () => { + resolve(); + cancelled = true; + }; + }); + + const startRefresh = async () => { + while (!cancelled) { + try { + await fn(); + } catch { + // ignore intentionally + } + + await Promise.race([ + new Promise((resolve) => setTimeout(resolve, delayMs)), + cancellationPromise, + ]); + } + }; + startRefresh(); + + return cancel!; +}