diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 150b9d1826..906e8c2d11 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -32,7 +32,6 @@ export default async function createPlugin({ logger, database, }: PluginEnvironment) { - const policy = new EntityPolicies(); const ingestionModel = new IngestionModels( new LocationReaders(), new DescriptorParsers(), @@ -40,19 +39,17 @@ export default async function createPlugin({ ); const db = await DatabaseManager.createDatabase(database, logger); - runPeriodically( - () => DatabaseManager.refreshLocations(db, ingestionModel, policy, logger), - 10000, - ); - const entitiesCatalog = new DatabaseEntitiesCatalog(db); const locationsCatalog = new DatabaseLocationsCatalog(db); const higherOrderOperation = new HigherOrderOperations( entitiesCatalog, locationsCatalog, ingestionModel, + logger, ); + runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000); + return await createRouter({ entitiesCatalog, locationsCatalog, diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index 6a7a51a168..1ec1ebf6e5 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -38,8 +38,8 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { async entityByName( kind: string, - name: string, namespace: string | undefined, + name: string, ): Promise { return await this.database.transaction(tx => this.entityByNameInternal(tx, kind, name, namespace), diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index 066fab8464..a1e76679f7 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -16,7 +16,10 @@ import { Location } from '@backstage/catalog-model'; import type { Database } from '../database'; -import { DatabaseLocationUpdateLogEvent } from '../database/types'; +import { + DatabaseLocationUpdateLogEvent, + DatabaseLocationUpdateLogStatus, +} from '../database/types'; import { LocationResponse, LocationsCatalog } from './types'; export class DatabaseLocationsCatalog implements LocationsCatalog { @@ -63,4 +66,28 @@ export class DatabaseLocationsCatalog implements LocationsCatalog { data, }; } + + async logUpdateSuccess( + locationId: string, + entityName?: string, + ): Promise { + await this.database.addLocationUpdateLogEvent( + locationId, + DatabaseLocationUpdateLogStatus.SUCCESS, + entityName, + ); + } + + async logUpdateFailure( + locationId: string, + error?: Error, + entityName?: string, + ): Promise { + await this.database.addLocationUpdateLogEvent( + locationId, + DatabaseLocationUpdateLogStatus.FAIL, + entityName, + error?.message, + ); + } } diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 0499b8a409..b0d3dde0fb 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -62,4 +62,10 @@ export type LocationsCatalog = { locations(): Promise; location(id: string): Promise; locationHistory(id: string): Promise; + logUpdateSuccess(locationId: string, entityName?: string): Promise; + logUpdateFailure( + locationId: string, + error?: Error, + entityName?: string, + ): Promise; }; diff --git a/plugins/catalog-backend/src/database/DatabaseManager.test.ts b/plugins/catalog-backend/src/database/DatabaseManager.test.ts deleted file mode 100644 index 0c28524f8d..0000000000 --- a/plugins/catalog-backend/src/database/DatabaseManager.test.ts +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getVoidLogger } from '@backstage/backend-common'; -import type { Entity, EntityPolicy } from '@backstage/catalog-model'; -import Knex from 'knex'; -import type { IngestionModel } from '../ingestion/types'; -import { DatabaseManager } from './DatabaseManager'; -import { DatabaseLocationUpdateLogStatus } from './types'; -import type { - Database, - DbLocationsRow, - DbLocationsRowWithStatus, -} from './types'; - -describe('DatabaseManager', () => { - describe('refreshLocations', () => { - it('works with no locations added', async () => { - const db = ({ - locations: jest.fn().mockResolvedValue([]), - } as unknown) as Database; - const reader: IngestionModel = { - readLocation: jest.fn(), - }; - const policy: EntityPolicy = { - enforce: jest.fn(), - }; - - await expect( - DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), - ).resolves.toBeUndefined(); - expect(reader.readLocation).not.toHaveBeenCalled(); - expect(policy.enforce).not.toHaveBeenCalled(); - }); - - it('can update a single location', async () => { - const location: DbLocationsRowWithStatus = { - id: '123', - type: 'some', - target: 'thing', - message: '', - status: DatabaseLocationUpdateLogStatus.SUCCESS, - timestamp: new Date(314159265).toISOString(), - }; - const desc: Entity = { - apiVersion: 'backstage.io/v1beta1', - kind: 'Component', - metadata: { name: 'c1' }, - spec: { type: 'service' }, - }; - - const tx = (undefined as unknown) as Knex.Transaction; - - const db = ({ - transaction: jest.fn(f => f(tx)), - entity: jest.fn(() => Promise.resolve(undefined)), - addEntity: jest.fn(), - locations: jest.fn(() => Promise.resolve([location])), - addLocationUpdateLogEvent: jest.fn(), - } as Partial) as Database; - - const reader: IngestionModel = { - readLocation: jest.fn(() => - Promise.resolve([{ type: 'data', data: desc }]), - ), - }; - const policy: EntityPolicy = { - enforce: jest.fn(() => Promise.resolve(desc)), - }; - - await expect( - DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), - ).resolves.toBeUndefined(); - expect(reader.readLocation).toHaveBeenCalledTimes(1); - expect(reader.readLocation).toHaveBeenNthCalledWith(1, 'some', 'thing'); - expect(db.addEntity).toHaveBeenCalledTimes(1); - expect(db.addEntity).toHaveBeenNthCalledWith(1, undefined, { - locationId: '123', - entity: expect.objectContaining({ - metadata: expect.objectContaining({ name: 'c1' }), - }), - }); - }); - - it('logs successful updates', async () => { - const tx = (undefined as unknown) as Knex.Transaction; - - const db = ({ - transaction: jest.fn(f => f(tx)), - addEntity: jest.fn(), - entity: jest.fn(() => Promise.resolve(undefined)), - locations: jest.fn(() => - Promise.resolve([ - { - id: '123', - type: 'some', - target: 'thing', - } as DbLocationsRow, - ]), - ), - addLocationUpdateLogEvent: jest.fn(), - } as unknown) as Database; - - const desc: Entity = { - apiVersion: 'backstage.io/v1beta1', - kind: 'Component', - metadata: { name: 'c1' }, - spec: { type: 'service' }, - }; - const reader: IngestionModel = { - readLocation: jest.fn(() => - Promise.resolve([{ type: 'data', data: desc }]), - ), - }; - const policy: EntityPolicy = { - enforce: jest.fn(() => Promise.resolve(desc)), - }; - - await expect( - DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), - ).resolves.toBeUndefined(); - - expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( - 1, - '123', - DatabaseLocationUpdateLogStatus.SUCCESS, - 'c1', - ); - - expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( - 2, - '123', - DatabaseLocationUpdateLogStatus.SUCCESS, - undefined, - ); - }); - - it('logs unsuccessful updates when parser fails', async () => { - const tx = (undefined as unknown) as Knex.Transaction; - - const db = ({ - transaction: jest.fn(f => f(tx)), - addEntity: jest.fn(), - locations: jest.fn(() => - Promise.resolve([ - { - id: '123', - type: 'some', - target: 'thing', - } as DbLocationsRow, - ]), - ), - addLocationUpdateLogEvent: jest.fn(), - } as unknown) as Database; - - const desc: Entity = { - apiVersion: 'backstage.io/v1beta1', - kind: 'Component', - metadata: { name: 'c1' }, - spec: { type: 'service' }, - }; - const reader: IngestionModel = { - readLocation: jest.fn(() => - Promise.resolve([{ type: 'data', data: desc }]), - ), - }; - const policy: EntityPolicy = { - enforce: jest.fn(() => - Promise.reject(new Error('parser error message')), - ), - }; - - await expect( - DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), - ).resolves.toBeUndefined(); - - expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( - 1, - '123', - DatabaseLocationUpdateLogStatus.FAIL, - 'c1', - 'parser error message', - ); - - expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( - 2, - '123', - DatabaseLocationUpdateLogStatus.SUCCESS, - undefined, - ); - }); - - it('logs unsuccessful updates when reader fails', async () => { - const tx = (undefined as unknown) as Knex.Transaction; - - const db = ({ - transaction: jest.fn(f => f(tx)), - addEntity: jest.fn(), - locations: jest.fn(() => - Promise.resolve([ - { - id: '123', - type: 'some', - target: 'thing', - } as DbLocationsRow, - ]), - ), - addLocationUpdateLogEvent: jest.fn(), - } as unknown) as Database; - - const reader: IngestionModel = { - readLocation: jest.fn(() => - Promise.reject([{ type: 'error', error: new Error('test message') }]), - ), - }; - const policy: EntityPolicy = { - enforce: jest.fn(() => - Promise.reject(new Error('parser error message')), - ), - }; - - await expect( - DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), - ).resolves.toBeUndefined(); - - expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( - 1, - '123', - DatabaseLocationUpdateLogStatus.FAIL, - undefined, - undefined, - ); - }); - }); -}); diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 4268dc4146..bc7c7b8dae 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -14,15 +14,11 @@ * limitations under the License. */ -import type { Entity, EntityPolicy } from '@backstage/catalog-model'; import Knex from 'knex'; -import lodash from 'lodash'; import path from 'path'; import { Logger } from 'winston'; -import type { IngestionModel } from '../ingestion/types'; import { CommonDatabase } from './CommonDatabase'; -import { DatabaseLocationUpdateLogStatus } from './types'; -import type { Database, DbEntityRequest } from './types'; +import type { Database } from './types'; export class DatabaseManager { public static async createDatabase( @@ -35,182 +31,4 @@ export class DatabaseManager { }); return new CommonDatabase(knex, logger); } - - private static async logUpdateSuccess( - database: Database, - locationId: string, - entityName?: string, - ) { - return database.addLocationUpdateLogEvent( - locationId, - DatabaseLocationUpdateLogStatus.SUCCESS, - entityName, - ); - } - - private static async logUpdateFailure( - database: Database, - locationId: string, - error?: Error, - entityName?: string, - ) { - return database.addLocationUpdateLogEvent( - locationId, - DatabaseLocationUpdateLogStatus.FAIL, - entityName, - error?.message, - ); - } - - public static async refreshLocations( - database: Database, - ingestionModel: IngestionModel, - entityPolicy: EntityPolicy, - logger: Logger, - ): Promise { - const locations = await database.locations(); - for (const location of locations) { - try { - logger.debug( - `Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`, - ); - - const readerOutput = await ingestionModel.readLocation( - location.type, - location.target, - ); - - for (const readerItem of readerOutput) { - if (readerItem.type === 'error') { - logger.info(readerItem.error); - continue; - } - - try { - const entity = await entityPolicy.enforce(readerItem.data); - await DatabaseManager.refreshSingleEntity( - database, - location.id, - entity, - logger, - ); - await DatabaseManager.logUpdateSuccess( - database, - location.id, - entity.metadata.name, - ); - } catch (error) { - await DatabaseManager.logUpdateFailure( - database, - location.id, - error, - readerItem.data.metadata.name, - ); - } - } - await DatabaseManager.logUpdateSuccess( - database, - location.id, - undefined, - ); - } catch (error) { - logger.debug( - `Failed to refresh location id="${location.id}", ${error}`, - ); - await DatabaseManager.logUpdateFailure(database, location.id, error); - } - } - } - - private static async refreshSingleEntity( - database: Database, - locationId: string, - entity: Entity, - logger: Logger, - ): Promise { - const { kind } = entity; - const { name, namespace } = entity.metadata || {}; - if (!name) { - throw new Error('Entities without names are not yet supported'); - } - - const request: DbEntityRequest = { - locationId: locationId, - entity: entity, - }; - - logger.debug( - `Read entity kind="${kind}" name="${name}" namespace="${namespace}"`, - ); - - await database.transaction(async tx => { - const previous = await database.entity(tx, kind, name, namespace); - if (!previous) { - logger.debug(`No such entity found, adding`); - await database.addEntity(tx, request); - } else if ( - !DatabaseManager.entitiesAreEqual(previous.entity, request.entity) - ) { - logger.debug(`Different from existing entity, updating`); - await database.updateEntity(tx, request); - } else { - logger.debug(`Equal to existing entity, skipping update`); - } - }); - } - - private static entitiesAreEqual(previous: Entity, next: Entity) { - if ( - previous.apiVersion !== next.apiVersion || - previous.kind !== next.kind || - !lodash.isEqual(previous.spec, next.spec) // Accept that {} !== undefined - ) { - return false; - } - - // Since the next annotations get merged into the previous, extract only - // the overlapping keys and check if their values match. - if (next.metadata.annotations) { - if (!previous.metadata.annotations) { - return false; - } - if ( - !lodash.isEqual( - next.metadata.annotations, - lodash.pick( - previous.metadata.annotations, - Object.keys(next.metadata.annotations), - ), - ) - ) { - return false; - } - } - - const e1 = lodash.cloneDeep(previous); - const e2 = lodash.cloneDeep(next); - - if (!e1.metadata.labels) { - e1.metadata.labels = {}; - } - if (!e2.metadata.labels) { - e2.metadata.labels = {}; - } - - // Remove generated fields - delete e1.metadata.uid; - delete e1.metadata.etag; - delete e1.metadata.generation; - delete e2.metadata.uid; - delete e2.metadata.etag; - delete e2.metadata.generation; - - // Remove already compared things - delete e1.metadata.annotations; - delete e1.spec; - delete e2.metadata.annotations; - delete e2.spec; - - return lodash.isEqual(e1, e2); - } } diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts index 7f1e890fbf..bd8dbedb27 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts @@ -14,10 +14,13 @@ * limitations under the License. */ +import { getVoidLogger } from '@backstage/backend-common'; +import { Entity, Location } from '@backstage/catalog-model'; import { EntitiesCatalog, LocationsCatalog } from '../catalog'; -import { IngestionModel } from './types'; +import { LocationUpdateStatus } from '../catalog/types'; +import { DatabaseLocationUpdateLogStatus } from '../database/types'; import { HigherOrderOperations } from './HigherOrderOperations'; -import { Entity } from '@backstage/catalog-model'; +import { IngestionModel } from './types'; describe('HigherOrderOperations', () => { let entitiesCatalog: jest.Mocked; @@ -39,6 +42,8 @@ describe('HigherOrderOperations', () => { locations: jest.fn(), location: jest.fn(), locationHistory: jest.fn(), + logUpdateSuccess: jest.fn(), + logUpdateFailure: jest.fn(), }; ingestionModel = { readLocation: jest.fn(), @@ -47,6 +52,7 @@ describe('HigherOrderOperations', () => { entitiesCatalog, locationsCatalog, ingestionModel, + getVoidLogger(), ); }); @@ -140,4 +146,147 @@ describe('HigherOrderOperations', () => { expect(locationsCatalog.addLocation).not.toBeCalled(); }); }); + + describe('refreshLocations', () => { + it('works with no locations added', async () => { + locationsCatalog.locations.mockResolvedValue([]); + + await expect( + higherOrderOperation.refreshAllLocations(), + ).resolves.toBeUndefined(); + + expect(locationsCatalog.locations).toHaveBeenCalledTimes(1); + expect(ingestionModel.readLocation).not.toHaveBeenCalled(); + expect(entitiesCatalog.addOrUpdateEntity).not.toHaveBeenCalled(); + }); + + it('can update a single location where a matching entity did not exist', async () => { + const locationStatus: LocationUpdateStatus = { + message: '', + status: DatabaseLocationUpdateLogStatus.SUCCESS, + timestamp: new Date(314159265).toISOString(), + }; + const location: Location = { + id: '123', + type: 'some', + target: 'thing', + }; + const desc: Entity = { + apiVersion: 'backstage.io/v1beta1', + kind: 'Component', + metadata: { name: 'c1' }, + spec: { type: 'service' }, + }; + + locationsCatalog.locations.mockResolvedValue([ + { currentStatus: locationStatus, data: location }, + ]); + ingestionModel.readLocation.mockResolvedValue([ + { type: 'data', data: desc }, + ]); + entitiesCatalog.entityByName.mockResolvedValue(undefined); + entitiesCatalog.addOrUpdateEntity.mockResolvedValue(desc); + + await expect( + higherOrderOperation.refreshAllLocations(), + ).resolves.toBeUndefined(); + + expect(locationsCatalog.locations).toHaveBeenCalledTimes(1); + expect(ingestionModel.readLocation).toHaveBeenCalledTimes(1); + expect(ingestionModel.readLocation).toHaveBeenNthCalledWith( + 1, + 'some', + 'thing', + ); + expect(entitiesCatalog.entityByName).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.entityByName).toHaveBeenNthCalledWith( + 1, + 'Component', + undefined, + 'c1', + ); + expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + metadata: expect.objectContaining({ name: 'c1' }), + }), + '123', + ); + }); + + it('logs successful updates', async () => { + const locationStatus: LocationUpdateStatus = { + message: '', + status: DatabaseLocationUpdateLogStatus.SUCCESS, + timestamp: new Date(314159265).toISOString(), + }; + const location: Location = { + id: '123', + type: 'some', + target: 'thing', + }; + const desc: Entity = { + apiVersion: 'backstage.io/v1beta1', + kind: 'Component', + metadata: { name: 'c1' }, + spec: { type: 'service' }, + }; + + locationsCatalog.locations.mockResolvedValue([ + { currentStatus: locationStatus, data: location }, + ]); + ingestionModel.readLocation.mockResolvedValue([ + { type: 'data', data: desc }, + ]); + entitiesCatalog.entityByName.mockResolvedValue(undefined); + entitiesCatalog.addOrUpdateEntity.mockResolvedValue(desc); + + await expect( + higherOrderOperation.refreshAllLocations(), + ).resolves.toBeUndefined(); + + expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledTimes(2); + expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith( + '123', + undefined, + ); + expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith( + '123', + 'c1', + ); + }); + + it('logs unsuccessful updates when reader fails', async () => { + const locationStatus: LocationUpdateStatus = { + message: '', + status: DatabaseLocationUpdateLogStatus.SUCCESS, + timestamp: new Date(314159265).toISOString(), + }; + const location: Location = { + id: '123', + type: 'some', + target: 'thing', + }; + + locationsCatalog.locations.mockResolvedValue([ + { currentStatus: locationStatus, data: location }, + ]); + ingestionModel.readLocation.mockRejectedValue( + new Error('reader error message'), + ); + + await expect( + higherOrderOperation.refreshAllLocations(), + ).resolves.toBeUndefined(); + + expect(ingestionModel.readLocation).toHaveBeenCalledTimes(1); + expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledTimes(1); + expect(locationsCatalog.logUpdateSuccess).not.toHaveBeenCalled(); + expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledWith( + '123', + expect.objectContaining({ message: 'reader error message' }), + ); + }); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts index e28d1e722a..e9051d1f78 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts @@ -16,10 +16,12 @@ import { InputError } from '@backstage/backend-common'; import { Entity, Location, LocationSpec } from '@backstage/catalog-model'; +import lodash from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import { EntitiesCatalog, LocationsCatalog } from '../catalog'; import { IngestionModel } from '../ingestion'; import { AddLocationResult, HigherOrderOperation } from './types'; +import { Logger } from 'winston'; const LOCATION_ANNOTATION = 'backstage.io/managed-by-location'; @@ -34,15 +36,18 @@ export class HigherOrderOperations implements HigherOrderOperation { private readonly entitiesCatalog: EntitiesCatalog; private readonly locationsCatalog: LocationsCatalog; private readonly ingestionModel: IngestionModel; + private readonly logger: Logger; constructor( entitiesCatalog: EntitiesCatalog, locationsCatalog: LocationsCatalog, ingestionModel: IngestionModel, + logger: Logger, ) { this.entitiesCatalog = entitiesCatalog; this.locationsCatalog = locationsCatalog; this.ingestionModel = ingestionModel; + this.logger = logger; } /** @@ -111,4 +116,154 @@ export class HigherOrderOperations implements HigherOrderOperation { return { location, entities: outputEntities }; } + + /** + * Goes through all registered locations, and performs a refresh of each one. + * + * Entities are read from their respective sources, are parsed and validated + * according to the entity policy, and get inserted or updated in the catalog. + * Entities that have disappeared from their location are left orphaned, + * without changes. + */ + async refreshAllLocations(): Promise { + const startTimestamp = new Date().valueOf(); + this.logger.info('Beginning locations refresh'); + + const locations = await this.locationsCatalog.locations(); + this.logger.info(`Visiting ${locations.length} locations`); + + for (const { data: location } of locations) { + this.logger.debug( + `Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`, + ); + try { + await this.refreshSingleLocation(location); + await this.locationsCatalog.logUpdateSuccess(location.id, undefined); + } catch (e) { + this.logger.debug( + `Failed to refresh location id="${location.id}" type="${location.type}" target="${location.target}", ${e}`, + ); + await this.locationsCatalog.logUpdateFailure(location.id, e); + } + } + + const endTimestamp = new Date().valueOf(); + const duration = ((endTimestamp - startTimestamp) / 1000).toFixed(1); + this.logger.debug(`Completed locations refresh in ${duration} seconds`); + } + + // Performs a full refresh of a single location + private async refreshSingleLocation(location: Location) { + const readerOutput = await this.ingestionModel.readLocation( + location.type, + location.target, + ); + + for (const readerItem of readerOutput) { + if (readerItem.type === 'error') { + this.logger.debug( + `Failed item in location id="${location.id}" type="${location.type}" target="${location.target}", ${readerItem.error}`, + ); + continue; + } + + const entity = readerItem.data; + this.logger.debug( + `Read entity kind="${entity.kind}" name="${ + entity.metadata.name + }" namespace="${entity.metadata.namespace || ''}"`, + ); + + try { + const previous = await this.entitiesCatalog.entityByName( + entity.kind, + entity.metadata.namespace, + entity.metadata.name, + ); + + if (!previous) { + this.logger.debug(`No such entity found, adding`); + await this.entitiesCatalog.addOrUpdateEntity(entity, location.id); + } else if (!this.entitiesAreEqual(previous, entity)) { + this.logger.debug(`Different from existing entity, updating`); + await this.entitiesCatalog.addOrUpdateEntity(entity, location.id); + } else { + this.logger.debug(`Equal to existing entity, skipping update`); + } + + await this.locationsCatalog.logUpdateSuccess( + location.id, + entity.metadata.name, + ); + } catch (error) { + this.logger.debug( + `Failed refresh of entity kind="${entity.kind}" name="${ + entity.metadata.name + }" namespace="${entity.metadata.namespace || ''}", ${error}`, + ); + + await this.locationsCatalog.logUpdateFailure( + location.id, + error, + entity.metadata.name, + ); + } + } + } + + // Compares entities, ignoring generated and irrelevant data + private entitiesAreEqual(previous: Entity, next: Entity): boolean { + if ( + previous.apiVersion !== next.apiVersion || + previous.kind !== next.kind || + !lodash.isEqual(previous.spec, next.spec) // Accept that {} !== undefined + ) { + return false; + } + + // Since the next annotations get merged into the previous, extract only + // the overlapping keys and check if their values match. + if (next.metadata.annotations) { + if (!previous.metadata.annotations) { + return false; + } + if ( + !lodash.isEqual( + next.metadata.annotations, + lodash.pick( + previous.metadata.annotations, + Object.keys(next.metadata.annotations), + ), + ) + ) { + return false; + } + } + + const e1 = lodash.cloneDeep(previous); + const e2 = lodash.cloneDeep(next); + + if (!e1.metadata.labels) { + e1.metadata.labels = {}; + } + if (!e2.metadata.labels) { + e2.metadata.labels = {}; + } + + // Remove generated fields + delete e1.metadata.uid; + delete e1.metadata.etag; + delete e1.metadata.generation; + delete e2.metadata.uid; + delete e2.metadata.etag; + delete e2.metadata.generation; + + // Remove already compared things + delete e1.metadata.annotations; + delete e1.spec; + delete e2.metadata.annotations; + delete e2.spec; + + return lodash.isEqual(e1, e2); + } } diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts index d26476bca0..0efc0a3be4 100644 --- a/plugins/catalog-backend/src/service/router.test.ts +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -43,6 +43,8 @@ describe('createRouter', () => { locations: jest.fn(), location: jest.fn(), locationHistory: jest.fn(), + logUpdateSuccess: jest.fn(), + logUpdateFailure: jest.fn(), }; higherOrderOperation = { addLocation: jest.fn(),