paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 19 Packages Projects Releases Wiki Activity

Merge pull request #847 from spotify/mob/refreshlogic

Browse Source 
Move out the refresh logic into a separate class
This commit is contained in:
Fredrik Adelöw
2020-05-14 11:44:11 +02:00
committed by GitHub
parent 47b3efffe9 922e783150
commit 3077fa9965
10 changed files with 214 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/catalog/CatalogLogic.test.ts
+74
View File
@@ -0,0 +1,74 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { PassThrough } from 'stream';
import winston from 'winston';
import { CatalogLogic } from './CatalogLogic';
import { Catalog } from './types';
describe('CatalogLogic', () => {
const logger = winston.createLogger({
transports: [new winston.transports.Stream({ stream: new PassThrough() })],
});
describe('refreshLocations', () => {
it('works with no locations added', async () => {
const catalog = ({
addOrUpdateComponent: jest.fn(),
locations: jest.fn().mockResolvedValue([]),
} as unknown) as Catalog;
const locationReader = jest.fn();
await expect(
CatalogLogic.refreshLocations(catalog, locationReader, logger),
).resolves.toBeUndefined();
expect(locationReader).not.toHaveBeenCalled();
});
it('can update a single location', async () => {
const catalog = ({
addOrUpdateComponent: jest.fn(),
locations: jest.fn().mockResolvedValue([
{
id: '123',
type: 'some',
target: 'thing',
},
]),
} as unknown) as Catalog;
const locationReader = jest.fn().mockResolvedValue([
{
name: 'c1',
},
]);
await expect(
CatalogLogic.refreshLocations(catalog, locationReader, logger),
).resolves.toBeUndefined();
expect(locationReader).toHaveBeenCalledTimes(1);
expect(locationReader).toHaveBeenNthCalledWith(
1,
expect.objectContaining({ type: 'some' }),
);
expect(catalog.addOrUpdateComponent).toHaveBeenCalledTimes(1);
expect(catalog.addOrUpdateComponent).toHaveBeenNthCalledWith(
1,
expect.objectContaining({ name: 'c1' }),
);
});
});
});
plugins/catalog-backend/src/catalog/CatalogLogic.ts
+68
View File
@@ -0,0 +1,68 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Logger } from 'winston';
import { LocationReader } from '../ingestion/types';
import { Catalog } from './types';
export class CatalogLogic {
public static startRefreshLoop(
catalog: Catalog,
reader: LocationReader,
logger: Logger,
): () => void {
let cancel: () => void;
let cancelled = false;
const cancellationPromise = new Promise((resolve) => {
cancel = () => {
resolve();
cancelled = true;
};
});
const startRefresh = async () => {
while (!cancelled) {
await CatalogLogic.refreshLocations(catalog, reader, logger);
await Promise.race([
new Promise((resolve) => setTimeout(resolve, 10000)),
cancellationPromise,
]);
}
};
startRefresh();
return cancel!;
}
public static async refreshLocations(
catalog: Catalog,
reader: LocationReader,
logger: Logger,
): Promise<void> {
const locations = await catalog.locations();
for (const location of locations) {
try {
logger.debug(`Attempting refresh of location: ${location.id}`);
const components = await reader(location);
for (const component of components) {
await catalog.addOrUpdateComponent(component);
}
} catch (e) {
logger.debug(`Failed to update location "${location.id}", ${e}`);
}
}
}
}
plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts
+16 -8
View File
@@ -38,15 +38,23 @@ describe('DatabaseCatalog', () => {
});
});
it('instantiates', () => {
it('manages locations', async () => {
const catalog = new DatabaseCatalog(database, logger);
expect(catalog).toBeDefined();
});
const input = { type: 'a', target: 'b' };
const output = { id: expect.anything(), type: 'a', target: 'b' };
describe(`refreshLocations`, () => {
it('works with no locations added', async () => {
const catalog = new DatabaseCatalog(database, logger);
await expect(catalog.refreshLocations()).resolves.toBeUndefined();
});
await catalog.addLocation(input);
const locations = await catalog.locations();
expect(locations).toEqual([output]);
const location = await catalog.location(locations[0].id);
expect(location).toEqual(output);
await catalog.removeLocation(locations[0].id);
await expect(catalog.locations()).resolves.toEqual([]);
await expect(catalog.location(locations[0].id)).rejects.toThrow(
/Found no location/,
);
});
});
plugins/catalog-backend/src/catalog/DatabaseCatalog.ts
+21 -39
View File
@@ -20,6 +20,7 @@ import path from 'path';
import { v4 as uuidv4 } from 'uuid';
import { Logger } from 'winston';
import { readLocation } from '../ingestion';
import { CatalogLogic } from './CatalogLogic';
import { AddLocationRequest, Catalog, Component, Location } from './types';
export class DatabaseCatalog implements Catalog {
@@ -33,14 +34,7 @@ export class DatabaseCatalog implements Catalog {
});
const databaseCatalog = new DatabaseCatalog(database, logger);
const startRefresh = async () => {
for (;;) {
await databaseCatalog.refreshLocations();
await new Promise(r => setTimeout(r, 10000));
}
};
startRefresh();
CatalogLogic.startRefreshLoop(databaseCatalog, readLocation, logger);
return databaseCatalog;
}
@@ -50,13 +44,28 @@ export class DatabaseCatalog implements Catalog {
private readonly logger: Logger,
) {}
async addOrUpdateComponent(component: Component): Promise<Component> {
await this.database.transaction(async (tx) => {
await tx('components')
.insert(component)
.catch(() =>
tx('components').where({ name: component.name }).update(component),
);
});
return component;
}
async components(): Promise<Component[]> {
throw new Error('Not supported');
return await this.database('components').select();
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async component(name: string): Promise<Component> {
throw new Error('Not supported');
const items = await this.database('components').where({ name }).select();
if (!items.length) {
throw new NotFoundError(`Found no component with name ${name}`);
}
return items[0];
}
async addLocation(location: AddLocationRequest): Promise<Location> {
@@ -67,42 +76,15 @@ export class DatabaseCatalog implements Catalog {
}
async removeLocation(id: string): Promise<void> {
const result = await this.database('locations')
.where({ id })
.del();
const result = await this.database('locations').where({ id }).del();
if (!result) {
throw new NotFoundError(`Found no location with ID ${id}`);
}
}
async refreshLocations(): Promise<void> {
const locations = await this.locations();
for (const location of locations) {
try {
this.logger.debug(`Attempting refresh of location: ${location.id}`);
const components = await readLocation(location);
for (const component of components) {
await this.database.transaction(async tx => {
await tx('components')
.insert(component)
.catch(() =>
tx('components')
.where({ name: component.name })
.update(component),
);
});
}
} catch (e) {
this.logger.debug(`Failed to update location "${location.id}", ${e}`);
}
}
}
async location(id: string): Promise<Location> {
const items = await this.database('locations')
.where({ id })
.select();
const items = await this.database('locations').where({ id }).select();
if (!items.length) {
throw new NotFoundError(`Found no location with ID ${id}`);
}
plugins/catalog-backend/src/catalog/StaticCatalog.ts
+11 -4
View File
@@ -16,7 +16,7 @@
import { NotFoundError } from '@backstage/backend-common';
import { v4 as uuidv4 } from 'uuid';
import { AddLocationRequest, Component, Catalog, Location } from './types';
import { AddLocationRequest, Catalog, Component, Location } from './types';
export class StaticCatalog implements Catalog {
private _components: Component[];
@@ -27,12 +27,19 @@ export class StaticCatalog implements Catalog {
this._locations = locations;
}
async addOrUpdateComponent(component: Component): Promise<Component> {
this._components = this._components
.filter((c) => c.name !== component.name)
.concat([component]);
return component;
}
async components(): Promise<Component[]> {
return this._components.slice();
}
async component(name: string): Promise<Component> {
const item = this._components.find(i => i.name === name);
const item = this._components.find((i) => i.name === name);
if (!item) {
throw new NotFoundError(`Found no component with name ${name}`);
}
@@ -46,11 +53,11 @@ export class StaticCatalog implements Catalog {
}
async removeLocation(id: string): Promise<void> {
this._locations = this._locations.filter(l => l.id !== id);
this._locations = this._locations.filter((l) => l.id !== id);
}
async location(id: string): Promise<Location> {
const location = this._locations.find(l => l.id === id);
const location = this._locations.find((l) => l.id === id);
if (!location) {
throw new NotFoundError(`Found no location with ID ${id}`);
}
plugins/catalog-backend/src/catalog/types.ts
+2 -1
View File
@@ -39,10 +39,11 @@ export const addLocationRequestShape: yup.Schema<AddLocationRequest> = yup
.noUnknown();
export type Catalog = {
addOrUpdateComponent(component: Component): Promise<Component>;
components(): Promise<Component[]>;
component(id: string): Promise<Component>;
addLocation(location: AddLocationRequest): Promise<Location>;
removeLocation(id: string): Promise<void>;
location(id: string): Promise<Location>;
locations(): Promise<Location[]>;
location(id: string): Promise<Location>;
};
plugins/catalog-backend/src/descriptors/component.ts
+1 -1
View File
@@ -15,8 +15,8 @@
*/
import * as yup from 'yup';
import { DescriptorEnvelope } from './envelope';
import { Component } from '../catalog/types';
import { DescriptorEnvelope } from './envelope';
export type ComponentDescriptor = {
metadata: {
plugins/catalog-backend/src/ingestion/fileLocation.ts
+1 -1
View File
@@ -15,8 +15,8 @@
*/
import fs from 'fs-extra';
import { parseDescriptor } from '../descriptors';
import { Component } from '../catalog/types';
import { parseDescriptor } from '../descriptors';
export async function readFileLocation(target: string): Promise<Component[]> {
let rawYaml;
plugins/catalog-backend/src/ingestion/index.ts
+1
View File
@@ -15,4 +15,5 @@
*/
export * from './fileLocation';
export * from './types';
export * from './util';
plugins/catalog-backend/src/ingestion/types.ts
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Component, Location } from '../catalog';
export type LocationReader = (location: Location) => Promise<Component[]>;