diff --git a/plugins/catalog-backend/src/catalog/CatalogLogic.test.ts b/plugins/catalog-backend/src/catalog/CatalogLogic.test.ts new file mode 100644 index 0000000000..38f1088b98 --- /dev/null +++ b/plugins/catalog-backend/src/catalog/CatalogLogic.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PassThrough } from 'stream'; +import winston from 'winston'; +import { CatalogLogic } from './CatalogLogic'; +import { Catalog } from './types'; + +describe('CatalogLogic', () => { + const logger = winston.createLogger({ + transports: [new winston.transports.Stream({ stream: new PassThrough() })], + }); + + describe('refreshLocations', () => { + it('works with no locations added', async () => { + const catalog = ({ + addOrUpdateComponent: jest.fn(), + locations: jest.fn().mockResolvedValue([]), + } as unknown) as Catalog; + const locationReader = jest.fn(); + + await expect( + CatalogLogic.refreshLocations(catalog, locationReader, logger), + ).resolves.toBeUndefined(); + expect(locationReader).not.toHaveBeenCalled(); + }); + + it('can update a single location', async () => { + const catalog = ({ + addOrUpdateComponent: jest.fn(), + locations: jest.fn().mockResolvedValue([ + { + id: '123', + type: 'some', + target: 'thing', + }, + ]), + } as unknown) as Catalog; + + const locationReader = jest.fn().mockResolvedValue([ + { + name: 'c1', + }, + ]); + + await expect( + CatalogLogic.refreshLocations(catalog, locationReader, logger), + ).resolves.toBeUndefined(); + expect(locationReader).toHaveBeenCalledTimes(1); + expect(locationReader).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ type: 'some' }), + ); + expect(catalog.addOrUpdateComponent).toHaveBeenCalledTimes(1); + expect(catalog.addOrUpdateComponent).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ name: 'c1' }), + ); + }); + }); +}); diff --git a/plugins/catalog-backend/src/catalog/CatalogLogic.ts b/plugins/catalog-backend/src/catalog/CatalogLogic.ts new file mode 100644 index 0000000000..ed195386d0 --- /dev/null +++ b/plugins/catalog-backend/src/catalog/CatalogLogic.ts @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Logger } from 'winston'; +import { LocationReader } from '../ingestion/types'; +import { Catalog } from './types'; + +export class CatalogLogic { + public static startRefreshLoop( + catalog: Catalog, + reader: LocationReader, + logger: Logger, + ): () => void { + let cancel: () => void; + let cancelled = false; + const cancellationPromise = new Promise((resolve) => { + cancel = () => { + resolve(); + cancelled = true; + }; + }); + + const startRefresh = async () => { + while (!cancelled) { + await CatalogLogic.refreshLocations(catalog, reader, logger); + await Promise.race([ + new Promise((resolve) => setTimeout(resolve, 10000)), + cancellationPromise, + ]); + } + }; + startRefresh(); + + return cancel!; + } + + public static async refreshLocations( + catalog: Catalog, + reader: LocationReader, + logger: Logger, + ): Promise { + const locations = await catalog.locations(); + for (const location of locations) { + try { + logger.debug(`Attempting refresh of location: ${location.id}`); + const components = await reader(location); + for (const component of components) { + await catalog.addOrUpdateComponent(component); + } + } catch (e) { + logger.debug(`Failed to update location "${location.id}", ${e}`); + } + } + } +} diff --git a/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts index 038314e653..7a474246d9 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts @@ -38,15 +38,23 @@ describe('DatabaseCatalog', () => { }); }); - it('instantiates', () => { + it('manages locations', async () => { const catalog = new DatabaseCatalog(database, logger); - expect(catalog).toBeDefined(); - }); + const input = { type: 'a', target: 'b' }; + const output = { id: expect.anything(), type: 'a', target: 'b' }; - describe(`refreshLocations`, () => { - it('works with no locations added', async () => { - const catalog = new DatabaseCatalog(database, logger); - await expect(catalog.refreshLocations()).resolves.toBeUndefined(); - }); + await catalog.addLocation(input); + + const locations = await catalog.locations(); + expect(locations).toEqual([output]); + const location = await catalog.location(locations[0].id); + expect(location).toEqual(output); + + await catalog.removeLocation(locations[0].id); + + await expect(catalog.locations()).resolves.toEqual([]); + await expect(catalog.location(locations[0].id)).rejects.toThrow( + /Found no location/, + ); }); }); diff --git a/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts index 51cccf2286..39a0719201 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts @@ -20,6 +20,7 @@ import path from 'path'; import { v4 as uuidv4 } from 'uuid'; import { Logger } from 'winston'; import { readLocation } from '../ingestion'; +import { CatalogLogic } from './CatalogLogic'; import { AddLocationRequest, Catalog, Component, Location } from './types'; export class DatabaseCatalog implements Catalog { @@ -33,14 +34,7 @@ export class DatabaseCatalog implements Catalog { }); const databaseCatalog = new DatabaseCatalog(database, logger); - - const startRefresh = async () => { - for (;;) { - await databaseCatalog.refreshLocations(); - await new Promise(r => setTimeout(r, 10000)); - } - }; - startRefresh(); + CatalogLogic.startRefreshLoop(databaseCatalog, readLocation, logger); return databaseCatalog; } @@ -50,13 +44,28 @@ export class DatabaseCatalog implements Catalog { private readonly logger: Logger, ) {} + async addOrUpdateComponent(component: Component): Promise { + await this.database.transaction(async (tx) => { + await tx('components') + .insert(component) + .catch(() => + tx('components').where({ name: component.name }).update(component), + ); + }); + return component; + } + async components(): Promise { - throw new Error('Not supported'); + return await this.database('components').select(); } // eslint-disable-next-line @typescript-eslint/no-unused-vars async component(name: string): Promise { - throw new Error('Not supported'); + const items = await this.database('components').where({ name }).select(); + if (!items.length) { + throw new NotFoundError(`Found no component with name ${name}`); + } + return items[0]; } async addLocation(location: AddLocationRequest): Promise { @@ -67,42 +76,15 @@ export class DatabaseCatalog implements Catalog { } async removeLocation(id: string): Promise { - const result = await this.database('locations') - .where({ id }) - .del(); + const result = await this.database('locations').where({ id }).del(); if (!result) { throw new NotFoundError(`Found no location with ID ${id}`); } } - async refreshLocations(): Promise { - const locations = await this.locations(); - for (const location of locations) { - try { - this.logger.debug(`Attempting refresh of location: ${location.id}`); - const components = await readLocation(location); - for (const component of components) { - await this.database.transaction(async tx => { - await tx('components') - .insert(component) - .catch(() => - tx('components') - .where({ name: component.name }) - .update(component), - ); - }); - } - } catch (e) { - this.logger.debug(`Failed to update location "${location.id}", ${e}`); - } - } - } - async location(id: string): Promise { - const items = await this.database('locations') - .where({ id }) - .select(); + const items = await this.database('locations').where({ id }).select(); if (!items.length) { throw new NotFoundError(`Found no location with ID ${id}`); } diff --git a/plugins/catalog-backend/src/catalog/StaticCatalog.ts b/plugins/catalog-backend/src/catalog/StaticCatalog.ts index 43229929b8..b98798f644 100644 --- a/plugins/catalog-backend/src/catalog/StaticCatalog.ts +++ b/plugins/catalog-backend/src/catalog/StaticCatalog.ts @@ -16,7 +16,7 @@ import { NotFoundError } from '@backstage/backend-common'; import { v4 as uuidv4 } from 'uuid'; -import { AddLocationRequest, Component, Catalog, Location } from './types'; +import { AddLocationRequest, Catalog, Component, Location } from './types'; export class StaticCatalog implements Catalog { private _components: Component[]; @@ -27,12 +27,19 @@ export class StaticCatalog implements Catalog { this._locations = locations; } + async addOrUpdateComponent(component: Component): Promise { + this._components = this._components + .filter((c) => c.name !== component.name) + .concat([component]); + return component; + } + async components(): Promise { return this._components.slice(); } async component(name: string): Promise { - const item = this._components.find(i => i.name === name); + const item = this._components.find((i) => i.name === name); if (!item) { throw new NotFoundError(`Found no component with name ${name}`); } @@ -46,11 +53,11 @@ export class StaticCatalog implements Catalog { } async removeLocation(id: string): Promise { - this._locations = this._locations.filter(l => l.id !== id); + this._locations = this._locations.filter((l) => l.id !== id); } async location(id: string): Promise { - const location = this._locations.find(l => l.id === id); + const location = this._locations.find((l) => l.id === id); if (!location) { throw new NotFoundError(`Found no location with ID ${id}`); } diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index e91c5f8267..ed33f35b3d 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -39,10 +39,11 @@ export const addLocationRequestShape: yup.Schema = yup .noUnknown(); export type Catalog = { + addOrUpdateComponent(component: Component): Promise; components(): Promise; component(id: string): Promise; addLocation(location: AddLocationRequest): Promise; removeLocation(id: string): Promise; - location(id: string): Promise; locations(): Promise; + location(id: string): Promise; }; diff --git a/plugins/catalog-backend/src/descriptors/component.ts b/plugins/catalog-backend/src/descriptors/component.ts index cb0c30aa61..0ecaf77c66 100644 --- a/plugins/catalog-backend/src/descriptors/component.ts +++ b/plugins/catalog-backend/src/descriptors/component.ts @@ -15,8 +15,8 @@ */ import * as yup from 'yup'; -import { DescriptorEnvelope } from './envelope'; import { Component } from '../catalog/types'; +import { DescriptorEnvelope } from './envelope'; export type ComponentDescriptor = { metadata: { diff --git a/plugins/catalog-backend/src/ingestion/fileLocation.ts b/plugins/catalog-backend/src/ingestion/fileLocation.ts index 524721035e..3c1b985356 100644 --- a/plugins/catalog-backend/src/ingestion/fileLocation.ts +++ b/plugins/catalog-backend/src/ingestion/fileLocation.ts @@ -15,8 +15,8 @@ */ import fs from 'fs-extra'; -import { parseDescriptor } from '../descriptors'; import { Component } from '../catalog/types'; +import { parseDescriptor } from '../descriptors'; export async function readFileLocation(target: string): Promise { let rawYaml; diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index 775fe29f84..e96be6822a 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -15,4 +15,5 @@ */ export * from './fileLocation'; +export * from './types'; export * from './util'; diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts new file mode 100644 index 0000000000..5808c7e592 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, Location } from '../catalog'; + +export type LocationReader = (location: Location) => Promise;