diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 2c6459e071..fb2814d640 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -20,7 +20,7 @@ import { } from '@backstage/plugin-catalog-backend'; import { PluginEnvironment } from '../types'; -export default async function ({ logger, database }: PluginEnvironment) { - const catalog = await DatabaseCatalog.create(database); +export default async function({ logger, database }: PluginEnvironment) { + const catalog = await DatabaseCatalog.create(database, logger); return await createRouter({ catalog, logger }); } diff --git a/plugins/catalog-backend/fixtures/component1.yaml b/plugins/catalog-backend/fixtures/component1.yaml new file mode 100644 index 0000000000..8db5cfc851 --- /dev/null +++ b/plugins/catalog-backend/fixtures/component1.yaml @@ -0,0 +1 @@ +name: Component1 diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 88b8048f7b..1a819c28b3 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -17,11 +17,13 @@ "cors": "^2.8.5", "express": "^4.17.1", "express-promise-router": "^3.0.3", + "fs-extra": "^9.0.0", "helmet": "^3.22.0", "knex": "^0.21.1", "morgan": "^1.10.0", "uuid": "^8.0.0", "winston": "^3.2.1", + "yaml": "^1.9.2", "yup": "^0.28.5" }, "devDependencies": { diff --git a/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts new file mode 100644 index 0000000000..038314e653 --- /dev/null +++ b/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import knex from 'knex'; +import path from 'path'; +import { PassThrough } from 'stream'; +import winston from 'winston'; +import { DatabaseCatalog } from './DatabaseCatalog'; + +describe('DatabaseCatalog', () => { + const database = knex({ + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }); + + const logger = winston.createLogger({ + transports: [new winston.transports.Stream({ stream: new PassThrough() })], + }); + + beforeEach(async () => { + await database.migrate.latest({ + directory: path.resolve(__dirname, '..', 'migrations'), + loadExtensions: ['.ts'], + }); + }); + + it('instantiates', () => { + const catalog = new DatabaseCatalog(database, logger); + expect(catalog).toBeDefined(); + }); + + describe(`refreshLocations`, () => { + it('works with no locations added', async () => { + const catalog = new DatabaseCatalog(database, logger); + await expect(catalog.refreshLocations()).resolves.toBeUndefined(); + }); + }); +}); diff --git a/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts index 4ef99108da..d28278c496 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts @@ -15,29 +15,79 @@ */ import { NotFoundError } from '@backstage/backend-common'; -import path from 'path'; import Knex from 'knex'; +import path from 'path'; +import fs from 'fs-extra'; import { v4 as uuidv4 } from 'uuid'; -import { AddLocationRequest, Component, Catalog, Location } from './types'; +import { Logger } from 'winston'; +import { + AddLocationRequest, + Catalog, + Component, + Location, + componentShape, +} from './types'; +import YAML from 'yaml'; + +async function readLocation(location: Location): Promise { + switch (location.type) { + case 'file': { + let parsed; + try { + const raw = await fs.readFile(location.target, 'utf8'); + parsed = YAML.parse(raw); + } catch (e) { + throw new Error(`Unable to read "${location.target}", ${e}`); + } + + try { + return [await componentShape.validate(parsed, { strict: true })]; + } catch (e) { + throw new Error( + `Malformed file contents at "${location.target}", ${e}`, + ); + } + } + + default: + throw new Error(`Unknown type "${location.type}"`); + } +} export class DatabaseCatalog implements Catalog { - static async create(database: Knex): Promise { + static async create( + database: Knex, + logger: Logger, + ): Promise { await database.migrate.latest({ directory: path.resolve(__dirname, '..', 'migrations'), loadExtensions: ['.js'], }); - return new DatabaseCatalog(database); + const databaseCatalog = new DatabaseCatalog(database, logger); + + const startRefresh = async () => { + for (;;) { + await databaseCatalog.refreshLocations(); + await new Promise(r => setTimeout(r, 10000)); + } + }; + startRefresh(); + + return databaseCatalog; } - constructor(private readonly database: Knex) {} + constructor( + private readonly database: Knex, + private readonly logger: Logger, + ) {} async components(): Promise { throw new Error('Not supported'); } // eslint-disable-next-line @typescript-eslint/no-unused-vars - async component(id: string): Promise { + async component(name: string): Promise { throw new Error('Not supported'); } @@ -49,15 +99,42 @@ export class DatabaseCatalog implements Catalog { } async removeLocation(id: string): Promise { - const result = await this.database('locations').where({ id }).del(); + const result = await this.database('locations') + .where({ id }) + .del(); if (!result) { throw new NotFoundError(`Found no location with ID ${id}`); } } + async refreshLocations(): Promise { + const locations = await this.locations(); + for (const location of locations) { + try { + this.logger.debug(`Attempting refresh of location: ${location.id}`); + const components = await readLocation(location); + for (const component of components) { + await this.database.transaction(async tx => { + await tx('components') + .insert(component) + .catch(() => + tx('components') + .where({ name: component.name }) + .update(component), + ); + }); + } + } catch (e) { + this.logger.debug(`Failed to update location "${location.id}", ${e}`); + } + } + } + async location(id: string): Promise { - const items = await this.database('locations').where({ id }).select(); + const items = await this.database('locations') + .where({ id }) + .select(); if (!items.length) { throw new NotFoundError(`Found no location with ID ${id}`); } diff --git a/plugins/catalog-backend/src/catalog/StaticCatalog.ts b/plugins/catalog-backend/src/catalog/StaticCatalog.ts index 2003982681..43229929b8 100644 --- a/plugins/catalog-backend/src/catalog/StaticCatalog.ts +++ b/plugins/catalog-backend/src/catalog/StaticCatalog.ts @@ -31,10 +31,10 @@ export class StaticCatalog implements Catalog { return this._components.slice(); } - async component(id: string): Promise { - const item = this._components.find((i) => i.id === id); + async component(name: string): Promise { + const item = this._components.find(i => i.name === name); if (!item) { - throw new NotFoundError(`Found no component with ID ${id}`); + throw new NotFoundError(`Found no component with name ${name}`); } return item; } @@ -46,11 +46,11 @@ export class StaticCatalog implements Catalog { } async removeLocation(id: string): Promise { - this._locations = this._locations.filter((l) => l.id !== id); + this._locations = this._locations.filter(l => l.id !== id); } async location(id: string): Promise { - const location = this._locations.find((l) => l.id === id); + const location = this._locations.find(l => l.id === id); if (!location) { throw new NotFoundError(`Found no location with ID ${id}`); } diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index f8b00626f0..abac94345d 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -17,9 +17,15 @@ import * as yup from 'yup'; export type Component = { - id: string; + name: string; }; +export const componentShape: yup.Schema = yup + .object({ + name: yup.string().required(), + }) + .unknown(); + export type Location = { id: string; type: string; diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 4607f8ca25..d5c46d7293 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -32,10 +32,10 @@ export async function startStandaloneServer( const catalog = new StaticCatalog( [ - { id: 'component1' }, - { id: 'component2' }, - { id: 'component3' }, - { id: 'component4' }, + { name: 'component1' }, + { name: 'component2' }, + { name: 'component3' }, + { name: 'component4' }, ], [], ); diff --git a/yarn.lock b/yarn.lock index 7fa6355394..4168fce334 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22136,6 +22136,13 @@ yaml@^1.7.2: dependencies: "@babel/runtime" "^7.8.7" +yaml@^1.9.2: + version "1.9.2" + resolved "https://registry.npmjs.org/yaml/-/yaml-1.9.2.tgz#f0cfa865f003ab707663e4f04b3956957ea564ed" + integrity sha512-HPT7cGGI0DuRcsO51qC1j9O16Dh1mZ2bnXwsi0jrSpsLz0WxOLSLXfkABVl6bZO629py3CU+OMJtpNHDLB97kg== + dependencies: + "@babel/runtime" "^7.9.2" + yargs-parser@^10.0.0: version "10.1.0" resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-10.1.0.tgz#7202265b89f7e9e9f2e5765e0fe735a905edbaa8"