diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 41e4226001..fdac6dd938 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -29,8 +29,9 @@ import { useHotCleanup } from '@backstage/backend-common'; export default async function createPlugin({ logger, database, + config, }: PluginEnvironment) { - const locationReader = new LocationReaders(logger); + const locationReader = new LocationReaders({ logger, config }); const db = await DatabaseManager.createDatabase(database, { logger }); const entitiesCatalog = new DatabaseEntitiesCatalog(db); diff --git a/plugins/catalog-backend/migrations/20200809202832_add_bootstrap_location.js b/plugins/catalog-backend/migrations/20200809202832_add_bootstrap_location.js new file mode 100644 index 0000000000..3d13a4219a --- /dev/null +++ b/plugins/catalog-backend/migrations/20200809202832_add_bootstrap_location.js @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + // Adds a single 'bootstrap' location that can be used to trigger work in processors. + // This is primarily here to fulfill foreign key constraints. + await knex('locations').insert({ + id: 'bootstrap', + type: 'bootstrap', + target: 'bootstrap', + }); +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + await knex('locations') + .where({ + id: 'bootstrap', + }) + .del(); +}; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 2445ee206e..4fbe982de4 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -23,6 +23,7 @@ "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.18", "@backstage/catalog-model": "^0.1.1-alpha.18", + "@backstage/config": "^0.1.1-alpha.18", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts index 679e5b43c1..456f06b334 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts @@ -17,6 +17,12 @@ import { DatabaseManager } from '../database'; import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; +const bootstrapLocation = { + id: 'bootstrap', + type: 'bootstrap', + target: 'bootstrap', +}; + describe('DatabaseLocationsCatalog', () => { let catalog: DatabaseLocationsCatalog; @@ -35,9 +41,12 @@ describe('DatabaseLocationsCatalog', () => { await expect( catalog.location('dd12620d-0436-422f-93bd-929aa0788123'), ).resolves.toEqual(expect.objectContaining({ data: location })); - await expect(catalog.locations()).resolves.toEqual([ - expect.objectContaining({ data: location }), - ]); + await expect(catalog.locations()).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ data: location }), + expect.objectContaining({ data: bootstrapLocation }), + ]), + ); }); it('does not return duplicates of rows because of logs', async () => { @@ -60,11 +69,12 @@ describe('DatabaseLocationsCatalog', () => { catalog.logUpdateSuccess(location1.id), ).resolves.toBeUndefined(); const locations = await catalog.locations(); - expect(locations.length).toBe(2); + expect(locations.length).toBe(3); expect(locations).toEqual( expect.arrayContaining([ expect.objectContaining({ data: location1 }), expect.objectContaining({ data: location2 }), + expect.objectContaining({ data: bootstrapLocation }), ]), ); }); diff --git a/plugins/catalog-backend/src/database/CommonDatabase.test.ts b/plugins/catalog-backend/src/database/CommonDatabase.test.ts index 21dde4ca74..94dfc25aa4 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.test.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.test.ts @@ -24,6 +24,15 @@ import type { DbLocationsRowWithStatus, } from './types'; +const bootstrapLocation = { + id: 'bootstrap', + type: 'bootstrap', + target: 'bootstrap', + message: null, + status: null, + timestamp: null, +}; + describe('CommonDatabase', () => { let db: Database; let entityRequest: DbEntityRequest; @@ -85,8 +94,12 @@ describe('CommonDatabase', () => { await db.addLocation(input); const locations = await db.locations(); - expect(locations).toEqual([output]); - const location = await db.location(locations[0].id); + expect(locations).toEqual( + expect.arrayContaining([output, bootstrapLocation]), + ); + const location = await db.location( + locations.find(l => l.id !== 'bootstrap')!.id, + ); expect(location).toEqual(output); // If we add 2 new update log events, @@ -105,20 +118,21 @@ describe('CommonDatabase', () => { DatabaseLocationUpdateLogStatus.FAIL, ); - expect(await db.locations()).toEqual([ - { - ...output, - status: DatabaseLocationUpdateLogStatus.FAIL, - timestamp: expect.any(String), - }, - ]); - - await db.transaction(tx => db.removeLocation(tx, locations[0].id)); - - await expect(db.locations()).resolves.toEqual([]); - await expect(db.location(locations[0].id)).rejects.toThrow( - /Found no location/, + await expect(db.locations()).resolves.toEqual( + expect.arrayContaining([ + bootstrapLocation, + { + ...output, + status: DatabaseLocationUpdateLogStatus.FAIL, + timestamp: expect.any(String), + }, + ]), ); + + await db.transaction(tx => db.removeLocation(tx, location.id)); + + await expect(db.locations()).resolves.toEqual([bootstrapLocation]); + await expect(db.location(location.id)).rejects.toThrow(/Found no location/); }); describe('addEntity', () => { diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index e5ef410456..7e91af7c59 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -15,6 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { Config, ConfigReader } from '@backstage/config'; import { Entity, EntityPolicies, @@ -31,6 +32,7 @@ import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor' import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; +import { StaticLocationProcessor } from './processors/StaticLocationProcessor'; import * as result from './processors/results'; import { LocationProcessor, @@ -47,6 +49,12 @@ import { LocationReader, ReadLocationResult } from './types'; // The max amount of nesting depth of generated work items const MAX_DEPTH = 10; +type Options = { + logger?: Logger; + config?: Config; + processors?: LocationProcessor[]; +}; + /** * Implements the reading of a location through a series of processor tasks. */ @@ -54,10 +62,16 @@ export class LocationReaders implements LocationReader { private readonly logger: Logger; private readonly processors: LocationProcessor[]; - static defaultProcessors( - entityPolicy: EntityPolicy = new EntityPolicies(), - ): LocationProcessor[] { + static defaultProcessors(options: { + config?: Config; + entityPolicy?: EntityPolicy; + }): LocationProcessor[] { + const { + config = new ConfigReader({}, 'missing-config'), + entityPolicy = new EntityPolicies(), + } = options; return [ + StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), new GithubReaderProcessor(), new GithubApiReaderProcessor(), @@ -71,10 +85,11 @@ export class LocationReaders implements LocationReader { ]; } - constructor( - logger: Logger = getVoidLogger(), - processors: LocationProcessor[] = LocationReaders.defaultProcessors(), - ) { + constructor({ + logger = getVoidLogger(), + config, + processors = LocationReaders.defaultProcessors({ config }), + }: Options) { this.logger = logger; this.processors = processors; } diff --git a/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts new file mode 100644 index 0000000000..6a2d1096cc --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import * as result from './results'; +import { Config } from '@backstage/config'; +import { LocationProcessorEmit } from './types'; + +export class StaticLocationProcessor implements StaticLocationProcessor { + static fromConfig(config: Config): StaticLocationProcessor { + const locations: LocationSpec[] = []; + + const lConfigs = config.getOptionalConfigArray('catalog.locations') ?? []; + for (const lConfig of lConfigs) { + const type = lConfig.getString('type'); + const target = lConfig.getString('target'); + locations.push({ type, target }); + } + + return new StaticLocationProcessor(locations); + } + + constructor(private readonly staticLocations: LocationSpec[]) {} + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'bootstrap') { + return false; + } + + for (const staticLocation of this.staticLocations) { + emit(result.location(staticLocation, false)); + } + + return true; + } +} diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 5c7ccb23f5..0b532d1664 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { createServiceBuilder } from '@backstage/backend-common'; +import { + createServiceBuilder, + loadBackendConfig, +} from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import { Server } from 'http'; import { Logger } from 'winston'; import { HigherOrderOperations } from '..'; @@ -34,12 +38,13 @@ export async function startStandaloneServer( options: ServerOptions, ): Promise { const logger = options.logger.child({ service: 'catalog-backend' }); + const config = ConfigReader.fromConfigs(await loadBackendConfig()); logger.debug('Creating application...'); const db = await DatabaseManager.createInMemoryDatabase({ logger }); const entitiesCatalog = new DatabaseEntitiesCatalog(db); const locationsCatalog = new DatabaseLocationsCatalog(db); - const locationReader = new LocationReaders(); + const locationReader = new LocationReaders({ logger, config }); const higherOrderOperation = new HigherOrderOperations( entitiesCatalog, locationsCatalog,