From 200c31c100c65615e4e980cd408f9b55dfdb9bb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sun, 16 Jan 2022 14:34:51 +0100 Subject: [PATCH] remove the legacy folder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../catalog/DatabaseEntitiesCatalog.test.ts | 426 ---------- .../legacy/catalog/DatabaseEntitiesCatalog.ts | 377 --------- .../catalog/DatabaseLocationsCatalog.test.ts | 81 -- .../catalog/DatabaseLocationsCatalog.ts | 91 -- .../src/legacy/catalog/index.ts | 24 - .../src/legacy/catalog/types.ts | 62 -- .../legacy/database/CommonDatabase.test.ts | 789 ------------------ .../src/legacy/database/CommonDatabase.ts | 624 -------------- .../src/legacy/database/DatabaseManager.ts | 109 --- .../src/legacy/database/index.ts | 32 - .../src/legacy/database/search.test.ts | 138 --- .../src/legacy/database/search.ts | 176 ---- .../src/legacy/database/types.ts | 223 ----- plugins/catalog-backend/src/legacy/index.ts | 20 - .../ingestion/HigherOrderOperations.test.ts | 415 --------- .../legacy/ingestion/HigherOrderOperations.ts | 218 ----- .../src/legacy/ingestion/LocationReaders.ts | 353 -------- .../src/legacy/ingestion/index.ts | 26 - .../src/legacy/ingestion/types.ts | 76 -- .../src/legacy/service/CatalogBuilder.test.ts | 260 ------ .../src/legacy/service/CatalogBuilder.ts | 373 --------- .../src/legacy/service/index.ts | 19 - .../src/legacy/service/router.test.ts | 529 ------------ .../src/legacy/service/router.ts | 254 ------ 24 files changed, 5695 deletions(-) delete mode 100644 plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.test.ts delete mode 100644 plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.ts delete mode 100644 plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts delete mode 100644 plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts delete mode 100644 plugins/catalog-backend/src/legacy/catalog/index.ts delete mode 100644 plugins/catalog-backend/src/legacy/catalog/types.ts delete mode 100644 plugins/catalog-backend/src/legacy/database/CommonDatabase.test.ts delete mode 100644 plugins/catalog-backend/src/legacy/database/CommonDatabase.ts delete mode 100644 plugins/catalog-backend/src/legacy/database/DatabaseManager.ts delete mode 100644 plugins/catalog-backend/src/legacy/database/index.ts delete mode 100644 plugins/catalog-backend/src/legacy/database/search.test.ts delete mode 100644 plugins/catalog-backend/src/legacy/database/search.ts delete mode 100644 plugins/catalog-backend/src/legacy/database/types.ts delete mode 100644 plugins/catalog-backend/src/legacy/index.ts delete mode 100644 plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts delete mode 100644 plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.ts delete mode 100644 plugins/catalog-backend/src/legacy/ingestion/LocationReaders.ts delete mode 100644 plugins/catalog-backend/src/legacy/ingestion/index.ts delete mode 100644 plugins/catalog-backend/src/legacy/ingestion/types.ts delete mode 100644 plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts delete mode 100644 plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts delete mode 100644 plugins/catalog-backend/src/legacy/service/index.ts delete mode 100644 plugins/catalog-backend/src/legacy/service/router.test.ts delete mode 100644 plugins/catalog-backend/src/legacy/service/router.ts diff --git a/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.test.ts b/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.test.ts deleted file mode 100644 index e2db8a0fc0..0000000000 --- a/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.test.ts +++ /dev/null @@ -1,426 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getVoidLogger } from '@backstage/backend-common'; -import { Entity, LOCATION_ANNOTATION } from '@backstage/catalog-model'; -import { Database, DatabaseManager, Transaction } from '../database'; -import { basicEntityFilter } from '../../service/request'; -import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog'; -import { EntityUpsertRequest } from '../../catalog/types'; - -describe('DatabaseEntitiesCatalog', () => { - let db: jest.Mocked; - let transaction: jest.Mocked; - - beforeAll(() => { - db = { - transaction: jest.fn(), - addEntities: jest.fn(), - updateEntity: jest.fn(), - entities: jest.fn(), - entityByName: jest.fn(), - entityByUid: jest.fn(), - removeEntityByUid: jest.fn(), - setRelations: jest.fn(), - addLocation: jest.fn(), - removeLocation: jest.fn(), - location: jest.fn(), - locations: jest.fn(), - locationHistory: jest.fn(), - addLocationUpdateLogEvent: jest.fn(), - }; - transaction = { - rollback: jest.fn(), - }; - }); - - beforeEach(() => { - jest.resetAllMocks(); - db.transaction.mockImplementation(async f => f(transaction)); - }); - - describe('batchAddOrUpdateEntities', () => { - it('adds when no given uid and no matching by name', async () => { - const entity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { - name: 'c', - namespace: 'd', - }, - }; - - db.entities.mockResolvedValue({ - entities: [], - pageInfo: { hasNextPage: false }, - }); - db.addEntities.mockResolvedValue([ - { entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } }, - ]); - - const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger()); - const result = await catalog.batchAddOrUpdateEntities([ - { entity, relations: [] }, - ]); - - expect(db.entities).toHaveBeenCalledTimes(1); - expect(db.entities).toHaveBeenCalledWith(expect.anything(), { - filter: basicEntityFilter({ - kind: 'b', - 'metadata.namespace': 'd', - 'metadata.name': 'c', - }), - }); - expect(db.addEntities).toHaveBeenCalledTimes(1); - expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [ - { entity: expect.anything(), relations: [] }, - ]); - expect(result).toEqual([{ entityId: 'u' }]); - }); - - it('dry run of add operation', async () => { - const entity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { - name: 'c', - namespace: 'd', - }, - }; - db.entities.mockResolvedValue({ - entities: [], - pageInfo: { hasNextPage: false }, - }); - db.addEntities.mockResolvedValue([ - { entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } }, - ]); - - const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger()); - const result = await catalog.batchAddOrUpdateEntities( - [{ entity, relations: [] }], - { dryRun: true }, - ); - - expect(db.entities).toHaveBeenCalledTimes(1); - expect(db.entities).toHaveBeenCalledWith(expect.anything(), { - filter: basicEntityFilter({ - kind: 'b', - 'metadata.namespace': 'd', - 'metadata.name': 'c', - }), - }); - expect(db.addEntities).toHaveBeenCalledTimes(1); - expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [ - { entity: expect.anything(), relations: [] }, - ]); - expect(transaction.rollback).toBeCalledTimes(1); - expect(result).toEqual([{ entityId: 'u' }]); - }); - - it('output modified entities', async () => { - const entity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { - name: 'c', - namespace: 'd', - annotations: { - [LOCATION_ANNOTATION]: 'mock', - }, - }, - }; - const dbEntity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { - name: 'c', - namespace: 'd', - description: 'changes', - uid: 'u', - annotations: { - [LOCATION_ANNOTATION]: 'mock', - }, - }, - }; - db.entities.mockResolvedValue({ - entities: [{ entity: dbEntity }], - pageInfo: { hasNextPage: false }, - }); - db.addEntities.mockResolvedValue([ - { entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } }, - ]); - - const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger()); - const result = await catalog.batchAddOrUpdateEntities( - [{ entity, relations: [] }], - { outputEntities: true }, - ); - - expect(db.entities).toHaveBeenCalledTimes(2); - expect(db.addEntities).toHaveBeenCalledTimes(1); - expect(result).toEqual([ - { - entityId: 'u', - entity: dbEntity, - }, - ]); - }); - - it('updates when given uid', async () => { - const entity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { - uid: 'u', - name: 'c', - namespace: 'd', - }, - spec: { - x: 'b', - }, - }; - const existing = { - entity: { - apiVersion: 'a', - kind: 'b', - metadata: { - uid: 'u', - etag: 'e', - generation: 1, - name: 'c', - namespace: 'd', - }, - spec: { - x: 'a', - }, - }, - }; - - db.entities.mockResolvedValue({ - entities: [existing], - pageInfo: { hasNextPage: false }, - }); - db.entityByUid.mockResolvedValue(existing); - db.updateEntity.mockResolvedValue({ entity }); - - const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger()); - const result = await catalog.batchAddOrUpdateEntities([ - { entity, relations: [] }, - ]); - - expect(db.entities).toHaveBeenCalledTimes(1); - expect(db.entities).toHaveBeenCalledWith(expect.anything(), { - filter: basicEntityFilter({ - kind: 'b', - 'metadata.namespace': 'd', - 'metadata.name': 'c', - }), - }); - expect(db.entityByName).not.toHaveBeenCalled(); - expect(db.entityByUid).toHaveBeenCalledTimes(1); - expect(db.entityByUid).toHaveBeenCalledWith(transaction, 'u'); - expect(db.updateEntity).toHaveBeenCalledTimes(1); - expect(db.updateEntity).toHaveBeenCalledWith( - transaction, - { - entity: { - apiVersion: 'a', - kind: 'b', - metadata: { - uid: 'u', - etag: expect.any(String), - generation: 2, - name: 'c', - namespace: 'd', - }, - spec: { - x: 'b', - }, - }, - relations: [], - }, - 'e', - 1, - ); - expect(result).toEqual([{ entityId: 'u' }]); - }); - - it('update when no given uid and matching by name', async () => { - const added: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { - name: 'c', - namespace: 'd', - }, - spec: { - x: 'b', - }, - }; - const existing = { - entity: { - apiVersion: 'a', - kind: 'b', - metadata: { - uid: 'u', - etag: 'e', - generation: 1, - name: 'c', - namespace: 'd', - }, - spec: { - x: 'a', - }, - }, - }; - - db.entities.mockResolvedValue({ - entities: [existing], - pageInfo: { hasNextPage: false }, - }); - db.entityByName.mockResolvedValue(existing); - db.updateEntity.mockResolvedValue(existing); - - const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger()); - const result = await catalog.batchAddOrUpdateEntities([ - { entity: added, relations: [] }, - ]); - - expect(db.entities).toHaveBeenCalledTimes(1); - expect(db.entities).toHaveBeenCalledWith(expect.anything(), { - filter: basicEntityFilter({ - kind: 'b', - 'metadata.namespace': 'd', - 'metadata.name': 'c', - }), - }); - expect(db.entityByName).toHaveBeenCalledTimes(1); - expect(db.entityByName).toHaveBeenCalledWith(transaction, { - kind: 'b', - namespace: 'd', - name: 'c', - }); - expect(db.updateEntity).toHaveBeenCalledTimes(1); - expect(db.updateEntity).toHaveBeenCalledWith( - transaction, - { - entity: { - apiVersion: 'a', - kind: 'b', - metadata: { - uid: 'u', - etag: expect.any(String), - generation: 2, - name: 'c', - namespace: 'd', - }, - spec: { - x: 'b', - }, - }, - relations: [], - }, - 'e', - 1, - ); - expect(result).toEqual([{ entityId: 'u' }]); - }); - - it('should not update if entity is unchanged', async () => { - const entity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { - uid: 'u', - name: 'c', - namespace: 'd', - }, - spec: { - x: 'a', - }, - }; - - db.entities.mockResolvedValue({ - entities: [{ entity }], - pageInfo: { hasNextPage: false }, - }); - db.entityByUid.mockResolvedValue({ entity }); - db.updateEntity.mockResolvedValue({ entity }); - - const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger()); - const result = await catalog.batchAddOrUpdateEntities([ - { entity, relations: [] }, - ]); - - expect(db.entities).toHaveBeenCalledTimes(1); - expect(db.entities).toHaveBeenCalledWith(expect.anything(), { - filter: basicEntityFilter({ - kind: 'b', - 'metadata.namespace': 'd', - 'metadata.name': 'c', - }), - }); - expect(db.entityByName).not.toHaveBeenCalled(); - expect(db.entityByUid).not.toHaveBeenCalled(); - expect(db.updateEntity).not.toHaveBeenCalled(); - expect(db.setRelations).toHaveBeenCalledTimes(1); - expect(db.setRelations).toHaveBeenCalledWith(expect.anything(), 'u', []); - expect(result).toEqual([{ entityId: 'u' }]); - }); - - it('both adds and updates', async () => { - const catalog = new DatabaseEntitiesCatalog( - await DatabaseManager.createTestDatabase(), - getVoidLogger(), - ); - const entities: EntityUpsertRequest[] = []; - for (let i = 0; i < 300; ++i) { - entities.push({ - entity: { - apiVersion: 'a', - kind: 'k', - metadata: { name: `n${i}` }, - }, - relations: [], - }); - } - - await catalog.batchAddOrUpdateEntities(entities); - const afterFirst = await catalog.entities(); - expect(afterFirst.entities.length).toBe(300); - - entities[40].entity.metadata.op = 'changed'; - entities.push({ - entity: { - apiVersion: 'a', - kind: 'k', - metadata: { name: `n300`, op: 'added' }, - }, - relations: [], - }); - - await catalog.batchAddOrUpdateEntities(entities); - const afterSecond = await catalog.entities(); - expect(afterSecond.entities.length).toBe(301); - expect( - afterSecond.entities.find(e => e.metadata.op === 'changed'), - ).toBeDefined(); - expect( - afterSecond.entities.find(e => e.metadata.op === 'added'), - ).toBeDefined(); - }, 10000); - }); -}); diff --git a/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.ts deleted file mode 100644 index 8776a5a4dc..0000000000 --- a/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.ts +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Entity, - entityHasChanges, - generateUpdatedEntity, - getEntityName, - LOCATION_ANNOTATION, - serializeEntityRef, -} from '@backstage/catalog-model'; -import { ConflictError } from '@backstage/errors'; -import { chunk, groupBy } from 'lodash'; -import limiterFactory from 'p-limit'; -import { Logger } from 'winston'; -import type { Database, DbEntityResponse, Transaction } from '../database'; -import { DbEntitiesRequest } from '../database/types'; -import { basicEntityFilter } from '../../service/request'; -import { durationText } from '../../util/timing'; -import type { - EntitiesCatalog, - EntitiesRequest, - EntitiesResponse, - EntityUpsertRequest, - EntityUpsertResponse, -} from '../../catalog/types'; - -type BatchContext = { - kind: string; - namespace: string; - locationId?: string; -}; - -// Some locations return tens or hundreds of thousands of entities. To make -// those payloads more manageable, we break work apart in batches of this -// many entities and write them to storage per batch. -const BATCH_SIZE = 100; - -// When writing large batches, there's an increasing chance of contention in -// the form of conflicts where we compete with other writes. Each batch gets -// this many attempts at being written before giving up. -const BATCH_ATTEMPTS = 3; - -// The number of batches that may be ongoing at the same time. -const BATCH_CONCURRENCY = 3; - -/** @deprecated This was part of the legacy catalog engine */ -export class DatabaseEntitiesCatalog implements EntitiesCatalog { - constructor( - private readonly database: Database, - private readonly logger: Logger, - ) {} - - async entities(request?: EntitiesRequest): Promise { - const dbRequest: DbEntitiesRequest = { - filter: request?.filter, - pagination: request?.pagination, - }; - - const dbResponse = await this.database.transaction(tx => - this.database.entities(tx, dbRequest), - ); - - const entities = dbResponse.entities.map(e => - request?.fields ? request.fields(e.entity) : e.entity, - ); - - return { - entities, - pageInfo: dbResponse.pageInfo, - }; - } - - async removeEntityByUid(uid: string): Promise { - await this.database.transaction(async tx => { - await this.database.removeEntityByUid(tx, uid); - }); - } - - async batchAddOrUpdateEntities( - requests: EntityUpsertRequest[], - options?: { - locationId?: string; - dryRun?: boolean; - outputEntities?: boolean; - }, - ): Promise { - // Group the requests by unique kind+namespace combinations. The reason for - // this is that the change detection and merging logic requires finding - // pre-existing versions of the entities in the database. Those queries are - // easier and faster to make if every batch revolves around a single kind- - // namespace pair. - const requestsByKindAndNamespace = groupBy(requests, ({ entity }) => { - const name = getEntityName(entity); - return `${name.kind}:${name.namespace}`.toLowerCase(); - }); - - // Go through the requests in reasonable batch sizes. Sometimes, sources - // produce tens of thousands of entities, and those are too large batch - // sizes to reasonably send to the database. - const batches = Object.values(requestsByKindAndNamespace) - .map(request => chunk(request, BATCH_SIZE)) - .flat(); - - // Bound the number of concurrent batches. We want a bit of concurrency for - // performance reasons, but not so much that we starve the connection pool - // or start thrashing. - const limiter = limiterFactory(BATCH_CONCURRENCY); - const tasks = batches.map(batch => - limiter(async () => { - // Retry the batch write a few times to deal with contention - for (let attempt = 1; ; ++attempt) { - try { - return this.batchAddOrUpdateEntitiesSingleBatch(batch, options); - } catch (e) { - if (e instanceof ConflictError && attempt < BATCH_ATTEMPTS) { - this.logger.warn( - `Failed to write batch at attempt ${attempt}/${BATCH_ATTEMPTS}, ${e}`, - ); - } else { - throw e; - } - } - } - }), - ); - - const responses = await Promise.all(tasks); - return responses.flat(); - } - - // Defines the actual logic of running a single batch. All of these share a - // common kind and namespace. - private async batchAddOrUpdateEntitiesSingleBatch( - batch: EntityUpsertRequest[], - options?: { - locationId?: string; - dryRun?: boolean; - outputEntities?: boolean; - }, - ) { - const { kind, namespace } = getEntityName(batch[0].entity); - const context = { - kind, - namespace, - locationId: options?.locationId, - }; - - this.logger.debug( - `Considering batch ${serializeEntityRef( - batch[0].entity, - )}-${serializeEntityRef(batch[batch.length - 1].entity)} (${ - batch.length - } entries)`, - ); - - return this.database.transaction(async tx => { - const { toAdd, toUpdate, toIgnore } = await this.analyzeBatch( - batch, - context, - tx, - ); - - let responses = new Array(); - if (toAdd.length) { - const items = await this.batchAdd(toAdd, context, tx); - responses.push(...items); - } - if (toUpdate.length) { - const items = await this.batchUpdate(toUpdate, context, tx); - responses.push(...items); - } - for (const { entity, relations } of toIgnore) { - // TODO(Rugvip): We currently always update relations, but we - // likely want to figure out a way to avoid that - const entityId = entity.metadata.uid; - if (entityId) { - await this.database.setRelations(tx, entityId, relations); - responses.push({ entityId }); - } - } - - if (options?.outputEntities && responses.length > 0) { - const writtenEntities = await this.database.entities(tx, { - filter: basicEntityFilter({ - 'metadata.uid': responses.map(e => e.entityId), - }), - }); - responses = writtenEntities.entities.map(e => ({ - entityId: e.entity.metadata.uid!, - entity: e.entity, - })); - } - - // If this is only a dry run, cancel the database transaction even if it - // was successful. - if (options?.dryRun) { - await tx.rollback(); - this.logger.debug(`Performed successful dry run of adding entities`); - } - - return responses; - }); - } - - // Given a batch of entities that were just read from a location, take them - // into consideration by comparing against the existing catalog entities and - // produce the list of entities to be added, and the list of entities to be - // updated - private async analyzeBatch( - requests: EntityUpsertRequest[], - { kind, namespace }: BatchContext, - tx: Transaction, - ): Promise<{ - toAdd: EntityUpsertRequest[]; - toUpdate: EntityUpsertRequest[]; - toIgnore: EntityUpsertRequest[]; - }> { - const markTimestamp = process.hrtime(); - - // Here we make use of the fact that all of the entities share kind and - // namespace within a batch - const names = requests.map(({ entity }) => entity.metadata.name); - const oldEntitiesResponse = await this.database.entities(tx, { - filter: basicEntityFilter({ - kind: kind, - 'metadata.namespace': namespace, - 'metadata.name': names, - }), - }); - - const oldEntitiesByName = new Map( - oldEntitiesResponse.entities.map(e => [e.entity.metadata.name, e.entity]), - ); - - const toAdd: EntityUpsertRequest[] = []; - const toUpdate: EntityUpsertRequest[] = []; - const toIgnore: EntityUpsertRequest[] = []; - - for (const request of requests) { - const newEntity = request.entity; - const oldEntity = oldEntitiesByName.get(newEntity.metadata.name); - const newLocation = newEntity.metadata.annotations?.[LOCATION_ANNOTATION]; - const oldLocation = - oldEntity?.metadata.annotations?.[LOCATION_ANNOTATION]; - if (!oldEntity) { - toAdd.push(request); - } else if (oldLocation !== newLocation) { - this.logger.warn( - `Rejecting write of entity ${serializeEntityRef( - newEntity, - )} from ${newLocation} because entity existed from ${oldLocation}`, - ); - toIgnore.push(request); - } else if (entityHasChanges(oldEntity, newEntity)) { - // TODO(freben): This currently uses addOrUpdateEntity under the hood, - // but should probably calculate the end result entity right here - // instead and call a dedicated batch update database method - toUpdate.push(request); - } else { - // Use the existing entity to ensure that we're able to read it back by uid if needed - toIgnore.push({ ...request, entity: oldEntity }); - } - } - - this.logger.debug( - `Found ${toAdd.length} entities to add, ${ - toUpdate.length - } entities to update in ${durationText(markTimestamp)}`, - ); - - return { toAdd, toUpdate, toIgnore }; - } - - // Efficiently adds the given entities to storage, under the assumption that - // they do not conflict with any existing entities - private async batchAdd( - requests: EntityUpsertRequest[], - { locationId }: BatchContext, - tx: Transaction, - ): Promise { - const markTimestamp = process.hrtime(); - - const res = await this.database.addEntities( - tx, - requests.map(({ entity, relations }) => ({ - locationId, - entity, - relations, - })), - ); - - const responses = res.map(({ entity }) => ({ - entityId: entity.metadata.uid!, - })); - - this.logger.debug( - `Added ${requests.length} entities in ${durationText(markTimestamp)}`, - ); - - return responses; - } - - // Efficiently updates the given entities into storage, under the assumption - // that there already exist entities with the same names - private async batchUpdate( - requests: EntityUpsertRequest[], - { locationId }: BatchContext, - tx: Transaction, - ): Promise { - const markTimestamp = process.hrtime(); - const responses: EntityUpsertResponse[] = []; - - // TODO(freben): Still not batched - for (const request of requests) { - const res = await this.addOrUpdateEntity(tx, request, locationId); - const entityId = res.metadata.uid!; - responses.push({ entityId }); - } - - this.logger.debug( - `Updated ${requests.length} entities in ${durationText(markTimestamp)}`, - ); - - return responses; - } - - // TODO(freben): Incorporate this into batchUpdate which is the only caller - private async addOrUpdateEntity( - tx: Transaction, - { entity, relations }: EntityUpsertRequest, - locationId?: string, - ): Promise { - // Find a matching (by uid, or by compound name, depending on the given - // entity) existing entity, to know whether to update or add - const existing = entity.metadata.uid - ? await this.database.entityByUid(tx, entity.metadata.uid) - : await this.database.entityByName(tx, getEntityName(entity)); - - // If it's an update, run the algorithm for annotation merging, updating - // etag/generation, etc. - let response: DbEntityResponse; - if (existing) { - const updated = generateUpdatedEntity(existing.entity, entity); - response = await this.database.updateEntity( - tx, - { locationId, entity: updated, relations }, - existing.entity.metadata.etag, - existing.entity.metadata.generation, - ); - } else { - const added = await this.database.addEntities(tx, [ - { locationId, entity, relations }, - ]); - response = added[0]; - } - - return response.entity; - } - - async entityAncestry(): Promise { - throw new Error('Not implemented'); - } -} diff --git a/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts b/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts deleted file mode 100644 index 8bd8e86b8e..0000000000 --- a/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { DatabaseManager } from '../database'; -import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; - -const bootstrapLocation = { - id: expect.any(String), - type: 'bootstrap', - target: 'bootstrap', -}; - -describe('DatabaseLocationsCatalog', () => { - let catalog: DatabaseLocationsCatalog; - - beforeEach(async () => { - const db = await DatabaseManager.createTestDatabase(); - catalog = new DatabaseLocationsCatalog(db); - }); - - it('can add a location', async () => { - const location = { - id: 'dd12620d-0436-422f-93bd-929aa0788123', - type: 'valid_type', - target: 'valid_target', - }; - await expect(catalog.addLocation(location)).resolves.toEqual(location); - await expect( - catalog.location('dd12620d-0436-422f-93bd-929aa0788123'), - ).resolves.toEqual(expect.objectContaining({ data: location })); - await expect(catalog.locations()).resolves.toEqual( - expect.arrayContaining([ - expect.objectContaining({ data: location }), - expect.objectContaining({ data: bootstrapLocation }), - ]), - ); - }); - - it('does not return duplicates of rows because of logs', async () => { - const location1 = { - id: 'dd12620d-0436-422f-93bd-929aa0788123', - type: 'valid_type', - target: 'valid_target1', - }; - const location2 = { - id: '1a89c479-1a33-4f27-8927-6090ba488c42', - type: 'valid_type', - target: 'valid_target2', - }; - await expect(catalog.addLocation(location1)).resolves.toEqual(location1); - await expect(catalog.addLocation(location2)).resolves.toEqual(location2); - await expect( - catalog.logUpdateSuccess(location1.id), - ).resolves.toBeUndefined(); - await expect( - catalog.logUpdateSuccess(location1.id), - ).resolves.toBeUndefined(); - const locations = await catalog.locations(); - expect(locations.length).toBe(3); - expect(locations).toEqual( - expect.arrayContaining([ - expect.objectContaining({ data: location1 }), - expect.objectContaining({ data: location2 }), - expect.objectContaining({ data: bootstrapLocation }), - ]), - ); - }); -}); diff --git a/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts deleted file mode 100644 index 1273e193b2..0000000000 --- a/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Location } from '@backstage/catalog-model'; -import type { Database } from '../database'; -import { - DatabaseLocationUpdateLogEvent, - DatabaseLocationUpdateLogStatus, -} from '../database/types'; -import { LocationResponse, LocationsCatalog } from './types'; - -/** @deprecated This was part of the legacy catalog engine */ -export class DatabaseLocationsCatalog implements LocationsCatalog { - constructor(private readonly database: Database) {} - - async addLocation(location: Location): Promise { - return await this.database.transaction( - async tx => await this.database.addLocation(tx, location), - ); - } - - async removeLocation(id: string): Promise { - await this.database.transaction(tx => this.database.removeLocation(tx, id)); - } - - async locations(): Promise { - const items = await this.database.locations(); - return items.map(({ message, status, timestamp, ...data }) => ({ - currentStatus: { - message, - status, - timestamp, - }, - data, - })); - } - - async locationHistory(id: string): Promise { - return this.database.locationHistory(id); - } - - async location(id: string): Promise { - const { message, status, timestamp, ...data } = - await this.database.location(id); - return { - currentStatus: { - message, - status, - timestamp, - }, - data, - }; - } - - async logUpdateSuccess( - locationId: string, - entityName?: string | string[], - ): Promise { - await this.database.addLocationUpdateLogEvent( - locationId, - DatabaseLocationUpdateLogStatus.SUCCESS, - entityName, - ); - } - - async logUpdateFailure( - locationId: string, - error?: Error, - entityName?: string, - ): Promise { - await this.database.addLocationUpdateLogEvent( - locationId, - DatabaseLocationUpdateLogStatus.FAIL, - entityName, - error?.message, - ); - } -} diff --git a/plugins/catalog-backend/src/legacy/catalog/index.ts b/plugins/catalog-backend/src/legacy/catalog/index.ts deleted file mode 100644 index 96828d03d1..0000000000 --- a/plugins/catalog-backend/src/legacy/catalog/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog'; -export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; -export type { - LocationResponse, - LocationsCatalog, - LocationUpdateLogEvent, - LocationUpdateStatus, -} from './types'; diff --git a/plugins/catalog-backend/src/legacy/catalog/types.ts b/plugins/catalog-backend/src/legacy/catalog/types.ts deleted file mode 100644 index 7ca1f3620a..0000000000 --- a/plugins/catalog-backend/src/legacy/catalog/types.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Location } from '@backstage/catalog-model'; - -// -// Locations -// - -/** @deprecated This was part of the legacy catalog engine */ -export type LocationUpdateStatus = { - timestamp: string | null; - status: string | null; - message: string | null; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type LocationUpdateLogEvent = { - id: string; - status: 'fail' | 'success'; - location_id: string; - entity_name: string; - created_at?: string; - message?: string; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type LocationResponse = { - data: Location; - currentStatus: LocationUpdateStatus; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type LocationsCatalog = { - addLocation(location: Location): Promise; - removeLocation(id: string): Promise; - locations(): Promise; - location(id: string): Promise; - locationHistory(id: string): Promise; - logUpdateSuccess( - locationId: string, - entityName?: string | string[], - ): Promise; - logUpdateFailure( - locationId: string, - error?: Error, - entityName?: string, - ): Promise; -}; diff --git a/plugins/catalog-backend/src/legacy/database/CommonDatabase.test.ts b/plugins/catalog-backend/src/legacy/database/CommonDatabase.test.ts deleted file mode 100644 index 8b72b470a6..0000000000 --- a/plugins/catalog-backend/src/legacy/database/CommonDatabase.test.ts +++ /dev/null @@ -1,789 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Entity, Location, parseEntityRef } from '@backstage/catalog-model'; -import { ConflictError } from '@backstage/errors'; -import { basicEntityFilter } from '../../service/request'; -import { DatabaseManager } from './DatabaseManager'; -import type { - DbEntityRequest, - DbEntityResponse, - DbLocationsRowWithStatus, -} from './types'; -import { Database, DatabaseLocationUpdateLogStatus } from './types'; - -const bootstrapLocation = { - id: expect.any(String), - type: 'bootstrap', - target: 'bootstrap', - message: null, - status: null, - timestamp: null, -}; - -describe('CommonDatabase', () => { - let db: Database; - let entityRequest: DbEntityRequest; - let entityResponse: DbEntityResponse; - - beforeEach(async () => { - db = await DatabaseManager.createTestDatabase(); - - entityRequest = { - entity: { - apiVersion: 'a', - kind: 'b', - metadata: { - name: 'c', - namespace: 'd', - labels: { e: 'f' }, - annotations: { g: 'h' }, - }, - spec: { i: 'j' }, - }, - relations: [], - }; - - entityResponse = { - locationId: undefined, - entity: { - apiVersion: 'a', - kind: 'b', - metadata: { - uid: expect.anything(), - etag: expect.anything(), - generation: expect.anything(), - name: 'c', - namespace: 'd', - labels: { e: 'f' }, - annotations: { - g: 'h', - }, - }, - spec: { i: 'j' }, - }, - }; - }); - - it('manages locations', async () => { - const input: Location = { - id: 'dd12620d-0436-422f-93bd-929aa0788123', - type: 'a', - target: 'b', - }; - const output: DbLocationsRowWithStatus = { - id: 'dd12620d-0436-422f-93bd-929aa0788123', - type: 'a', - target: 'b', - message: null, - status: null, - timestamp: null, - }; - - await db.transaction(async tx => await db.addLocation(tx, input)); - - const locations = await db.locations(); - expect(locations).toEqual( - expect.arrayContaining([output, bootstrapLocation]), - ); - const location = await db.location( - locations.find(l => l.type !== 'bootstrap')!.id, - ); - expect(location).toEqual(output); - - // If we add 2 new update log events, - // this should not result in location duplication - // due to incorrect join in DB - await db.addLocationUpdateLogEvent( - 'dd12620d-0436-422f-93bd-929aa0788123', - DatabaseLocationUpdateLogStatus.SUCCESS, - ); - - // Have a second in-between - // To avoid having same timestamp on event - await new Promise(res => setTimeout(res, 1000)); - await db.addLocationUpdateLogEvent( - 'dd12620d-0436-422f-93bd-929aa0788123', - DatabaseLocationUpdateLogStatus.FAIL, - ); - - await expect(db.locations()).resolves.toEqual( - expect.arrayContaining([ - bootstrapLocation, - { - ...output, - status: DatabaseLocationUpdateLogStatus.FAIL, - timestamp: expect.anything(), - }, - ]), - ); - - await db.transaction(tx => db.removeLocation(tx, location.id)); - - await expect(db.locations()).resolves.toEqual([bootstrapLocation]); - await expect(db.location(location.id)).rejects.toThrow(/Found no location/); - }); - - it('refuses to remove the bootstrap location', async () => { - const input: Location = { - id: 'dd12620d-0436-422f-93bd-929aa0788123', - type: 'bootstrap', - target: 'bootstrap', - }; - - const output = await db.transaction( - async tx => await db.addLocation(tx, input), - ); - - await expect( - db.transaction(async tx => await db.removeLocation(tx, output.id)), - ).rejects.toThrow(ConflictError); - }); - - describe('addEntities', () => { - it('happy path: adds entities to empty database', async () => { - const result = await db.transaction(tx => - db.addEntities(tx, [entityRequest]), - ); - expect(result).toEqual([entityResponse]); - }); - - it('rejects adding the same-named entity twice', async () => { - const req: DbEntityRequest[] = [ - { - entity: { - apiVersion: 'av1', - kind: 'k1', - metadata: { name: 'n1', namespace: 'ns1' }, - }, - relations: [], - }, - { - entity: { - apiVersion: 'av1', - kind: 'k1', - metadata: { name: 'n1', namespace: 'ns1' }, - }, - relations: [], - }, - ]; - await expect( - db.transaction(tx => db.addEntities(tx, req)), - ).rejects.toThrow(ConflictError); - }); - - it('rejects adding the almost-same-namespace entity twice', async () => { - const req: DbEntityRequest[] = [ - { - entity: { - apiVersion: 'av1', - kind: 'k1', - metadata: { name: 'n1', namespace: 'ns1' }, - }, - relations: [], - }, - { - entity: { - apiVersion: 'av1', - kind: 'k1', - metadata: { name: 'n1', namespace: 'nS1' }, - }, - relations: [], - }, - ]; - await expect( - db.transaction(tx => db.addEntities(tx, req)), - ).rejects.toThrow(ConflictError); - }); - - it('accepts adding the same-named entity twice if on different namespaces', async () => { - const req: DbEntityRequest[] = [ - { - entity: { - apiVersion: 'av1', - kind: 'k1', - metadata: { name: 'n1', namespace: 'ns1' }, - }, - relations: [], - }, - { - entity: { - apiVersion: 'av1', - kind: 'k1', - metadata: { name: 'n1', namespace: 'ns2' }, - }, - relations: [], - }, - ]; - await expect( - db.transaction(tx => db.addEntities(tx, req)), - ).resolves.toEqual([ - { - entity: expect.objectContaining({ - metadata: expect.objectContaining({ - namespace: 'ns1', - uid: expect.any(String), - etag: expect.any(String), - generation: expect.any(Number), - }), - }), - }, - { - entity: expect.objectContaining({ - metadata: expect.objectContaining({ - namespace: 'ns2', - uid: expect.any(String), - etag: expect.any(String), - generation: expect.any(Number), - }), - }), - }, - ]); - }); - }); - - describe('locationHistory', () => { - it('outputs the history correctly', async () => { - const location: Location = { - id: 'dd12620d-0436-422f-93bd-929aa0788123', - type: 'a', - target: 'b', - }; - - await db.transaction(async tx => await db.addLocation(tx, location)); - - await db.addLocationUpdateLogEvent( - 'dd12620d-0436-422f-93bd-929aa0788123', - DatabaseLocationUpdateLogStatus.SUCCESS, - ); - await db.addLocationUpdateLogEvent( - 'dd12620d-0436-422f-93bd-929aa0788123', - DatabaseLocationUpdateLogStatus.FAIL, - undefined, - 'Something went wrong', - ); - - const result = await db.locationHistory( - 'dd12620d-0436-422f-93bd-929aa0788123', - ); - expect(result).toEqual( - expect.arrayContaining([ - { - created_at: expect.anything(), - entity_name: null, - id: expect.anything(), - location_id: 'dd12620d-0436-422f-93bd-929aa0788123', - message: null, - status: DatabaseLocationUpdateLogStatus.SUCCESS, - }, - { - created_at: expect.anything(), - entity_name: null, - id: expect.anything(), - location_id: 'dd12620d-0436-422f-93bd-929aa0788123', - message: 'Something went wrong', - status: DatabaseLocationUpdateLogStatus.FAIL, - }, - ]), - ); - }); - }); - - describe('updateEntity', () => { - it('can read and no-op-update an entity', async () => { - const [added] = await db.transaction(tx => - db.addEntities(tx, [entityRequest]), - ); - const updated = await db.transaction(tx => - db.updateEntity(tx, { entity: added.entity, relations: [] }), - ); - expect(updated.entity.apiVersion).toEqual(added.entity.apiVersion); - expect(updated.entity.kind).toEqual(added.entity.kind); - expect(updated.entity.metadata.etag).toEqual(added.entity.metadata.etag); - expect(updated.entity.metadata.generation).toEqual( - added.entity.metadata.generation, - ); - expect(updated.entity.metadata.name).toEqual(added.entity.metadata.name); - expect(updated.entity.metadata.namespace).toEqual( - added.entity.metadata.namespace, - ); - }); - - it('can update name if uid matches', async () => { - const [added] = await db.transaction(tx => - db.addEntities(tx, [entityRequest]), - ); - added.entity.metadata.name! = 'new!'; - const updated = await db.transaction(tx => - db.updateEntity(tx, { entity: added.entity, relations: [] }), - ); - expect(updated.entity.metadata.name).toEqual('new!'); - }); - - it('fails to update an entity if etag does not match', async () => { - const [added] = await db.transaction(tx => - db.addEntities(tx, [entityRequest]), - ); - await expect( - db.transaction(tx => - db.updateEntity( - tx, - { entity: added.entity, relations: [] }, - 'garbage', - ), - ), - ).rejects.toThrow(ConflictError); - }); - - it('fails to update an entity if generation does not match', async () => { - const [added] = await db.transaction(tx => - db.addEntities(tx, [entityRequest]), - ); - await expect( - db.transaction(tx => - db.updateEntity( - tx, - { entity: added.entity, relations: [] }, - undefined, - 1e20, - ), - ), - ).rejects.toThrow(ConflictError); - }); - }); - - describe('entities', () => { - it('can get all entities with empty filters list', async () => { - const e1: Entity = { - apiVersion: 'a', - kind: 'k1', - metadata: { name: 'n' }, - }; - const e2: Entity = { - apiVersion: 'c', - kind: 'k2', - metadata: { name: 'n' }, - spec: { c: null }, - }; - await db.transaction(async tx => { - await db.addEntities(tx, [ - { entity: e1, relations: [] }, - { entity: e2, relations: [] }, - ]); - }); - const result = await db.transaction(async tx => db.entities(tx)); - expect(result.entities.length).toEqual(2); - expect(result.entities).toEqual( - expect.arrayContaining([ - { - locationId: undefined, - entity: expect.objectContaining({ kind: 'k1' }), - }, - { - locationId: undefined, - entity: expect.objectContaining({ kind: 'k2' }), - }, - ]), - ); - }); - - it('can get all specific entities for matching filters (naive case)', async () => { - const entities: Entity[] = [ - { apiVersion: 'a', kind: 'k1', metadata: { name: 'n' } }, - { - apiVersion: 'a', - kind: 'k2', - metadata: { name: 'n' }, - spec: { c: 'some' }, - }, - { - apiVersion: 'a', - kind: 'k3', - metadata: { name: 'n' }, - spec: { c: null }, - }, - ]; - - await db.transaction(async tx => { - await db.addEntities( - tx, - entities.map(entity => ({ entity, relations: [] })), - ); - }); - - const response = await db.transaction(async tx => - db.entities(tx, { - filter: basicEntityFilter({ kind: 'k2', 'spec.c': 'some' }), - }), - ); - - expect(response.entities).toEqual([ - { - locationId: undefined, - entity: expect.objectContaining({ kind: 'k2' }), - }, - ]); - }); - - it('can get all specific entities for matching filters case insensitively', async () => { - const entities: Entity[] = [ - { - apiVersion: 'A', - kind: 'K1', - metadata: { name: 'N' }, - spec: { c: 'SOME' }, - }, - { - apiVersion: 'a', - kind: 'k2', - metadata: { name: 'n' }, - spec: { c: 'Some' }, - }, - { - apiVersion: 'a', - kind: 'k3', - metadata: { name: 'n' }, - spec: { c: 'somE' }, - }, - ]; - - await db.transaction(async tx => { - await db.addEntities( - tx, - entities.map(entity => ({ entity, relations: [] })), - ); - }); - - const rows = await db.transaction(async tx => - db.entities(tx, { - filter: basicEntityFilter({ ApiVersioN: 'A', 'spEc.C': 'some' }), - }), - ); - - expect(rows.entities.length).toEqual(3); - expect(rows.entities).toEqual( - expect.arrayContaining([ - { - locationId: undefined, - entity: expect.objectContaining({ kind: 'K1' }), - }, - { - locationId: undefined, - entity: expect.objectContaining({ kind: 'k2' }), - }, - { - locationId: undefined, - entity: expect.objectContaining({ kind: 'k3' }), - }, - ]), - ); - }); - - it('can get all specific entities for matching existence filters', async () => { - const entities: Entity[] = [ - { - apiVersion: 'A', - kind: 'K1', - metadata: { - name: 'N', - annotations: { - foo: 'bar', - }, - }, - spec: { c: 'SOME' }, - }, - { - apiVersion: 'a', - kind: 'k2', - metadata: { - name: 'N', - annotations: { - foo: 'bar', - }, - }, - spec: { c: 'Some' }, - }, - { - apiVersion: 'a', - kind: 'k3', - metadata: { name: 'n' }, - spec: { c: 'somE' }, - }, - ]; - - await db.transaction(async tx => { - await db.addEntities( - tx, - entities.map(entity => ({ entity, relations: [] })), - ); - }); - - const existRows = await db.transaction(async tx => - db.entities(tx, { - filter: { - anyOf: [ - { - allOf: [{ key: 'metadata.annotations.foo' }], - }, - ], - }, - }), - ); - - expect(existRows.entities.length).toEqual(2); - expect(existRows.entities).toEqual( - expect.arrayContaining([ - { - locationId: undefined, - entity: expect.objectContaining({ kind: 'K1' }), - }, - { - locationId: undefined, - entity: expect.objectContaining({ kind: 'k2' }), - }, - ]), - ); - }); - }); - - describe('setRelations', () => { - it('adds a relation for an entity', async () => { - const mockRelations = [ - { - source: { - kind: entityRequest.entity.kind, - namespace: entityRequest.entity.metadata.namespace!, - name: entityRequest.entity.metadata.name, - }, - target: { - kind: 'component', - namespace: 'asd', - name: 'bleb', - }, - type: 'child', - }, - ]; - - const entityId = await db.transaction(async tx => { - const [{ entity }] = await db.addEntities(tx, [entityRequest]); - - await db.setRelations(tx, entity?.metadata?.uid!, mockRelations); - return entity.metadata.uid; - }); - - const returnedEntity1 = await db.transaction(tx => - db.entityByUid(tx, entityId!), - ); - expect(returnedEntity1?.entity.relations).toEqual([ - { target: mockRelations[0].target, type: 'child' }, - ]); - - const returnedEntity2 = await db.transaction(tx => - db.entityByName(tx, mockRelations[0].source), - ); - expect(returnedEntity2?.entity.relations).toEqual([ - { target: mockRelations[0].target, type: 'child' }, - ]); - - const { entities } = await db.transaction(tx => db.entities(tx)); - const [returnedEntity3] = entities; - expect(returnedEntity3?.entity.relations).toEqual([ - { target: mockRelations[0].target, type: 'child' }, - ]); - }); - - function makeRelation(source: string, type: string, target: string) { - return { - source: parseEntityRef(source, { - defaultKind: 'x', - defaultNamespace: 'x', - }), - type, - target: parseEntityRef(target, { - defaultKind: 'x', - defaultNamespace: 'x', - }), - }; - } - - it('should not allow setting relations on nonexistent entities', async () => { - await expect( - db.transaction(async tx => { - await db.setRelations(tx, 'nonexistent', [ - makeRelation('a:b/c', 'rel1', 'x:y/z'), - ]); - }), - ).rejects.toThrow(/constraint failed/); - }); - - it('should allow setting relations on nonexistent entities without any relations', async () => { - await expect( - db.transaction(async tx => { - await db.setRelations(tx, 'nonexistent', []); - }), - ).resolves.toBeUndefined(); - }); - - it('adds multiple relations for entities', async () => { - const entity1 = { - apiVersion: 'v1', - kind: 'a', - metadata: { - name: 'c', - namespace: 'b', - }, - }; - const entity2 = { - apiVersion: 'v1', - kind: 'x', - metadata: { - name: 'z', - namespace: 'y', - }, - }; - const fromEntity1 = [ - makeRelation('a:b/c', 'rel1', 'x:y/z'), - makeRelation('x:y/z', 'rel2', 'a:b/c'), - makeRelation('a:b/c', 'rel2', 'x:y/z'), - ]; - const fromEntity2 = [ - makeRelation('a:b/c', 'rel4', 'x:y/z'), - makeRelation('a:b/c', 'rel5', 'x:y/z'), - makeRelation('x:y/z', 'rel6', 'a:b/c'), - // relations don't have to reference the originating entity, so this should be fine, but not show up - makeRelation('g:h/i', 'rel8', 'd:e/f'), - ]; - - const { id2: secondEntityId } = await db.transaction(async tx => { - const [{ entity: e1 }, { entity: e2 }] = await db.addEntities(tx, [ - { entity: entity1, relations: [] }, - { entity: entity2, relations: [] }, - ]); - const id1 = e1?.metadata?.uid!; - const id2 = e2?.metadata?.uid!; - - await db.setRelations(tx, id1, fromEntity1); - await db.setRelations(tx, id2, fromEntity2); - - return { id1, id2 }; - }); - - const res = await db.transaction(tx => db.entities(tx)); - expect( - res.entities.map(r => ({ - name: r.entity.metadata.name, - relations: r.entity.relations, - })), - ).toEqual([ - { - name: 'c', - relations: [ - { - type: 'rel1', - target: { kind: 'x', namespace: 'y', name: 'z' }, - }, - { - type: 'rel2', - target: { kind: 'x', namespace: 'y', name: 'z' }, - }, - { - type: 'rel4', - target: { kind: 'x', namespace: 'y', name: 'z' }, - }, - { - type: 'rel5', - target: { kind: 'x', namespace: 'y', name: 'z' }, - }, - ], - }, - { - name: 'z', - relations: [ - { - type: 'rel2', - target: { kind: 'a', namespace: 'b', name: 'c' }, - }, - { - type: 'rel6', - target: { kind: 'a', namespace: 'b', name: 'c' }, - }, - ], - }, - ]); - - await db.transaction(tx => db.removeEntityByUid(tx, secondEntityId)); - - const res2 = await db.transaction(tx => db.entities(tx)); - expect( - res2.entities.map(r => ({ - name: r.entity.metadata.name, - relations: r.entity.relations, - })), - ).toEqual([ - { - name: 'c', - relations: [ - { - type: 'rel1', - target: { kind: 'x', namespace: 'y', name: 'z' }, - }, - { - type: 'rel2', - target: { kind: 'x', namespace: 'y', name: 'z' }, - }, - ], - }, - ]); - }); - }); - - describe('entityByName', () => { - it('can get entities case insensitively', async () => { - const entities: Entity[] = [ - { - apiVersion: 'a', - kind: 'k1', - metadata: { name: 'n' }, - }, - { - apiVersion: 'B', - kind: 'K2', - metadata: { name: 'N', namespace: 'NS' }, - }, - ]; - - await db.transaction(async tx => { - await db.addEntities( - tx, - entities.map(entity => ({ entity, relations: [] })), - ); - }); - - const e1 = await db.transaction(async tx => - db.entityByName(tx, { kind: 'k1', namespace: 'default', name: 'n' }), - ); - expect(e1!.entity.metadata.name).toEqual('n'); - const e2 = await db.transaction(async tx => - db.entityByName(tx, { kind: 'k2', namespace: 'nS', name: 'n' }), - ); - expect(e2!.entity.metadata.name).toEqual('N'); - const e3 = await db.transaction(async tx => - db.entityByName(tx, { kind: 'unknown', namespace: 'nS', name: 'n' }), - ); - expect(e3).toBeUndefined(); - }); - }); -}); diff --git a/plugins/catalog-backend/src/legacy/database/CommonDatabase.ts b/plugins/catalog-backend/src/legacy/database/CommonDatabase.ts deleted file mode 100644 index 3ea8696437..0000000000 --- a/plugins/catalog-backend/src/legacy/database/CommonDatabase.ts +++ /dev/null @@ -1,624 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ConflictError, InputError, NotFoundError } from '@backstage/errors'; -import { - Entity, - EntityName, - EntityRelationSpec, - ENTITY_DEFAULT_NAMESPACE, - ENTITY_META_GENERATED_FIELDS, - generateEntityEtag, - generateEntityUid, - Location, - parseEntityName, -} from '@backstage/catalog-model'; -import { Knex } from 'knex'; -import lodash from 'lodash'; -import type { Logger } from 'winston'; -import { buildEntitySearch } from './search'; -import { - Database, - DatabaseLocationUpdateLogEvent, - DatabaseLocationUpdateLogStatus, - DbEntitiesRelationsRow, - DbEntitiesRequest, - DbEntitiesResponse, - DbEntitiesRow, - DbEntitiesSearchRow, - DbEntityRequest, - DbEntityResponse, - DbLocationsRow, - DbLocationsRowWithStatus, - DbPageInfo, - Transaction, -} from './types'; -import { EntityPagination, EntitiesSearchFilter } from '../../catalog/types'; - -type LegacyEntityFilter = { - anyOf: { allOf: EntitiesSearchFilter[] }[]; -}; - -// The number of items that are sent per batch to the database layer, when -// doing .batchInsert calls to knex. This needs to be low enough to not cause -// errors in the underlying engine due to exceeding query limits, but large -// enough to get the speed benefits. -const BATCH_SIZE = 50; - -/** - * The core database implementation.. - * @deprecated This was part of the legacy catalog engin - */ -export class CommonDatabase implements Database { - constructor( - private readonly database: Knex, - private readonly logger: Logger, - ) {} - - async transaction(fn: (tx: Transaction) => Promise): Promise { - try { - let result: T | undefined = undefined; - - await this.database.transaction( - async tx => { - // We can't return here, as knex swallows the return type in case the transaction is rolled back: - // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 - result = await fn(tx); - }, - { - // If we explicitly trigger a rollback, don't fail. - doNotRejectOnRollback: true, - }, - ); - - return result!; - } catch (e) { - this.logger.debug(`Error during transaction, ${e}`); - - if ( - /SQLITE_CONSTRAINT: UNIQUE/.test(e.message) || - /unique constraint/.test(e.message) - ) { - throw new ConflictError(`Rejected due to a conflicting entity`, e); - } - - throw e; - } - } - - async addEntities( - txOpaque: Transaction, - request: DbEntityRequest[], - ): Promise { - const tx = txOpaque as Knex.Transaction; - - const result: DbEntityResponse[] = []; - const entityRows: DbEntitiesRow[] = []; - const relationRows: DbEntitiesRelationsRow[] = []; - const searchRows: DbEntitiesSearchRow[] = []; - - for (const { entity, relations, locationId } of request) { - if (entity.metadata.uid !== undefined) { - throw new InputError('May not specify uid for new entities'); - } else if (entity.metadata.etag !== undefined) { - throw new InputError('May not specify etag for new entities'); - } else if (entity.metadata.generation !== undefined) { - throw new InputError('May not specify generation for new entities'); - } else if (entity.relations !== undefined) { - throw new InputError('May not specify relations for new entities'); - } - - const uid = generateEntityUid(); - const etag = generateEntityEtag(); - const generation = 1; - const newEntity = { - ...entity, - metadata: { - ...entity.metadata, - uid, - etag, - generation, - }, - }; - - result.push({ entity: newEntity, locationId }); - entityRows.push(this.toEntityRow(locationId, newEntity)); - relationRows.push(...this.toRelationRows(uid, relations)); - searchRows.push(...buildEntitySearch(uid, newEntity)); - } - - await tx.batchInsert('entities', entityRows, BATCH_SIZE); - await tx.batchInsert('entities_relations', relationRows, BATCH_SIZE); - await tx.batchInsert('entities_search', searchRows, BATCH_SIZE); - - return result; - } - - async updateEntity( - txOpaque: Transaction, - request: DbEntityRequest, - matchingEtag?: string, - matchingGeneration?: number, - ): Promise { - const tx = txOpaque as Knex.Transaction; - - const { uid } = request.entity.metadata; - if (!uid) { - throw new InputError('Must specify uid when updating entities'); - } - - // Find existing entity - const oldRows = await tx('entities') - .where({ id: uid }) - .select(); - if (oldRows.length !== 1) { - throw new NotFoundError('No matching entity found'); - } - const etag = oldRows[0].etag; - const generation = Number(oldRows[0].generation); - - // Validate the old entity. The Number cast is here because sqlite reads it - // as a string, no matter what the table actually says. - if (matchingEtag && matchingEtag !== etag) { - throw new ConflictError( - `Etag mismatch, expected="${matchingEtag}" found="${etag}"`, - ); - } - if (matchingGeneration && matchingGeneration !== generation) { - throw new ConflictError( - `Generation mismatch, expected="${matchingGeneration}" found="${generation}"`, - ); - } - - // Store the updated entity; select on the old etag to ensure that we do - // not lose to another writer - const newRow = this.toEntityRow(request.locationId, request.entity); - const updatedRows = await tx('entities') - .where({ id: uid, etag }) - .update(newRow); - if (updatedRows !== 1) { - throw new ConflictError(`Failed to update entity`); - } - - const relationRows = this.toRelationRows(uid, request.relations); - await tx('entities_relations') - .where({ originating_entity_id: uid }) - .del(); - await tx.batchInsert('entities_relations', relationRows, BATCH_SIZE); - - try { - const entries = buildEntitySearch(uid, request.entity); - await tx('entities_search') - .where({ entity_id: uid }) - .del(); - await tx.batchInsert('entities_search', entries, BATCH_SIZE); - } catch { - // ignore intentionally - if this happens, the entity was deleted before - // we got around to writing the entries - } - - return request; - } - - async entities( - txOpaque: Transaction, - request?: DbEntitiesRequest, - ): Promise { - const tx = txOpaque as Knex.Transaction; - - let entitiesQuery = tx('entities'); - - if ( - request?.filter && - (request.filter.hasOwnProperty('key') || - request.filter.hasOwnProperty('allOf') || - request.filter.hasOwnProperty('not')) - ) { - throw new Error( - 'Filters for the legacy CommonDatabase must obey the { anyOf: [{ allOf: [] }] } format.', - ); - } - for (const singleFilter of (request?.filter as LegacyEntityFilter)?.anyOf ?? - []) { - entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() { - for (const filter of singleFilter.allOf) { - if ( - filter.hasOwnProperty('anyOf') || - filter.hasOwnProperty('allOf') || - filter.hasOwnProperty('not') - ) { - throw new Error( - 'Nested filters are not supported in the legacy CommonDatabase', - ); - } - const { key, values } = filter; - // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to - // make a lot of sense. However, it had abysmal performance on sqlite - // when datasets grew large, so we're using IN instead. - const matchQuery = tx('entities_search') - .select('entity_id') - .where(function keyFilter() { - this.andWhere({ key: key.toLowerCase() }); - if (values) { - if (values.length === 1) { - this.andWhere({ value: values[0].toLowerCase() }); - } else if (values.length > 1) { - this.andWhere( - 'value', - 'in', - values.map(v => v.toLowerCase()), - ); - } - } - }); - this.andWhere('id', 'in', matchQuery); - } - }); - } - - entitiesQuery = entitiesQuery - .select('entities.*') - .orderBy('full_name', 'asc'); - - const { limit, offset } = parsePagination(request?.pagination); - if (limit !== undefined) { - entitiesQuery = entitiesQuery.limit(limit + 1); - } - if (offset !== undefined) { - entitiesQuery = entitiesQuery.offset(offset); - } - - let rows = await entitiesQuery; - - let pageInfo: DbPageInfo; - if (limit === undefined || rows.length <= limit) { - pageInfo = { hasNextPage: false }; - } else { - rows = rows.slice(0, -1); - pageInfo = { - hasNextPage: true, - endCursor: stringifyPagination({ - limit, - offset: (offset ?? 0) + limit, - }), - }; - } - - return { - entities: await this.toEntityResponses(tx, rows), - pageInfo, - }; - } - - async entityByName( - txOpaque: Transaction, - name: EntityName, - ): Promise { - const tx = txOpaque as Knex.Transaction; - - const rows = await tx('entities') - .where({ - full_name: `${name.kind}:${name.namespace}/${name.name}`.toLowerCase(), - }) - .select(); - - if (rows.length !== 1) { - return undefined; - } - - return this.toEntityResponses(tx, rows).then(r => r[0]); - } - - async entityByUid( - txOpaque: Transaction, - uid: string, - ): Promise { - const tx = txOpaque as Knex.Transaction; - - const rows = await tx('entities') - .where({ id: uid }) - .select(); - - if (rows.length !== 1) { - return undefined; - } - - return this.toEntityResponses(tx, rows).then(r => r[0]); - } - - async removeEntityByUid(txOpaque: Transaction, uid: string): Promise { - const tx = txOpaque as Knex.Transaction; - - const result = await tx('entities').where({ id: uid }).del(); - if (!result) { - throw new NotFoundError(`Found no entity with ID ${uid}`); - } - } - - async setRelations( - txOpaque: Transaction, - originatingEntityId: string, - relations: EntityRelationSpec[], - ): Promise { - const tx = txOpaque as Knex.Transaction; - const relationRows = this.toRelationRows(originatingEntityId, relations); - - await tx('entities_relations') - .where({ originating_entity_id: originatingEntityId }) - .del(); - await tx.batchInsert('entities_relations', relationRows, BATCH_SIZE); - } - - async addLocation( - txOpaque: Transaction, - location: Location, - ): Promise { - const tx = txOpaque as Knex.Transaction; - - const row: DbLocationsRow = { - id: location.id, - type: location.type, - target: location.target, - }; - await tx('locations').insert(row); - return row; - } - - async removeLocation(txOpaque: Transaction, id: string): Promise { - const tx = txOpaque as Knex.Transaction; - - const locations = await tx('locations') - .where({ id }) - .select(); - if (!locations.length) { - throw new NotFoundError(`Found no location with ID ${id}`); - } - - if (locations[0].type === 'bootstrap') { - throw new ConflictError('You may not delete the bootstrap location.'); - } - - await tx('entities') - .where({ location_id: id }) - .update({ location_id: null }); - await tx('locations').where({ id }).del(); - } - - async location(id: string): Promise { - const items = await this.database('locations') - .where('locations.id', id) - .leftOuterJoin( - 'location_update_log_latest', - 'locations.id', - 'location_update_log_latest.location_id', - ) - .select('locations.*', { - status: 'location_update_log_latest.status', - timestamp: 'location_update_log_latest.created_at', - message: 'location_update_log_latest.message', - }); - - if (!items.length) { - throw new NotFoundError(`Found no location with ID ${id}`); - } - return items[0]; - } - - async locations(): Promise { - const locations = await this.database('locations') - .leftOuterJoin( - 'location_update_log_latest', - 'locations.id', - 'location_update_log_latest.location_id', - ) - .select('locations.*', { - status: 'location_update_log_latest.status', - timestamp: 'location_update_log_latest.created_at', - message: 'location_update_log_latest.message', - }); - - return locations; - } - - async locationHistory(id: string): Promise { - const result = await this.database( - 'location_update_log', - ) - .where('location_id', id) - .orderBy('created_at', 'desc') - .limit(10) - .select(); - - return result; - } - - async addLocationUpdateLogEvent( - locationId: string, - status: DatabaseLocationUpdateLogStatus, - entityName?: string | string[], - message?: string, - ): Promise { - // Remove log entries older than a day - const cutoff = new Date(); - cutoff.setDate(cutoff.getDate() - 1); - await this.database('location_update_log') - .where('created_at', '<', cutoff.toISOString()) - .del(); - - const items: Partial[] = [entityName] - .flat() - .map(n => ({ - status, - location_id: locationId, - entity_name: n, - message, - })); - - for (const chunk of lodash.chunk(items, BATCH_SIZE)) { - await this.database( - 'location_update_log', - ).insert(chunk); - } - } - - private toEntityRow( - locationId: string | undefined, - entity: Entity, - ): DbEntitiesRow { - const lowerKind = entity.kind.toLowerCase(); - const lowerNamespace = ( - entity.metadata.namespace || ENTITY_DEFAULT_NAMESPACE - ).toLowerCase(); - const lowerName = entity.metadata.name.toLowerCase(); - - const data = { - ...entity, - metadata: lodash.omit(entity.metadata, ...ENTITY_META_GENERATED_FIELDS), - }; - - return { - id: entity.metadata.uid!, - location_id: locationId || null, - etag: entity.metadata.etag!, - generation: entity.metadata.generation!, - full_name: `${lowerKind}:${lowerNamespace}/${lowerName}`, - data: JSON.stringify(data), - }; - } - - private toRelationRows( - originatingEntityId: string, - relations: EntityRelationSpec[], - ): DbEntitiesRelationsRow[] { - const serializeName = (e: EntityName) => - `${e.kind}:${e.namespace}/${e.name}`.toLowerCase(); - - const rows = relations.map(({ source, target, type }) => ({ - originating_entity_id: originatingEntityId, - source_full_name: serializeName(source), - target_full_name: serializeName(target), - type, - })); - - return deduplicateRelations(rows); - } - - private async toEntityResponses( - tx: Knex.Transaction, - rows: DbEntitiesRow[], - ): Promise { - // TODO(Rugvip): This is here because it's simple for now, but we likely - // need to refactor this to be more efficient or introduce pagination. - const relations = await this.getRelationsPerFullName( - tx, - rows.map(r => r.full_name), - ); - - const result = new Array(); - for (const row of rows) { - const entity = JSON.parse(row.data) as Entity; - entity.metadata.uid = row.id; - entity.metadata.etag = row.etag; - entity.metadata.generation = Number(row.generation); // cast due to sqlite - - entity.relations = (relations[row.full_name] ?? []).map(r => ({ - target: parseEntityName(r.target_full_name), - type: r.type, - })); - - result.push({ - locationId: row.location_id || undefined, - entity, - }); - } - - return result; - } - - // Returns a mapping from e.g. component:default/foo to the relations whose - // source_full_name matches that. - private async getRelationsPerFullName( - tx: Knex.Transaction, - sourceFullNames: string[], - ): Promise> { - const batches = lodash.chunk(lodash.uniq(sourceFullNames), 500); - - const relations = new Array(); - for (const batch of batches) { - relations.push( - ...(await tx('entities_relations') - .whereIn('source_full_name', batch) - .orderBy(['type', 'target_full_name']) - .select()), - ); - } - - return lodash.groupBy( - deduplicateRelations(relations), - r => r.source_full_name, - ); - } -} - -function parsePagination(input?: EntityPagination): { - limit?: number; - offset?: number; -} { - if (!input) { - return {}; - } - - let { limit, offset } = input; - - if (input.after !== undefined) { - let cursor; - try { - const json = Buffer.from(input.after, 'base64').toString('utf8'); - cursor = JSON.parse(json); - } catch { - throw new InputError('Malformed after cursor, could not be parsed'); - } - if (cursor.limit !== undefined) { - if (!Number.isInteger(cursor.limit)) { - throw new InputError('Malformed after cursor, limit was not an number'); - } - limit = cursor.limit; - } - if (cursor.offset !== undefined) { - if (!Number.isInteger(cursor.offset)) { - throw new InputError('Malformed after cursor, offset was not a number'); - } - offset = cursor.offset; - } - } - - return { limit, offset }; -} - -function stringifyPagination(input: { limit: number; offset: number }) { - const json = JSON.stringify({ limit: input.limit, offset: input.offset }); - const base64 = Buffer.from(json, 'utf8').toString('base64'); - return base64; -} - -function deduplicateRelations( - rows: DbEntitiesRelationsRow[], -): DbEntitiesRelationsRow[] { - return lodash.uniqBy( - rows, - r => `${r.source_full_name}:${r.target_full_name}:${r.type}`, - ); -} diff --git a/plugins/catalog-backend/src/legacy/database/DatabaseManager.ts b/plugins/catalog-backend/src/legacy/database/DatabaseManager.ts deleted file mode 100644 index 9bb75a1043..0000000000 --- a/plugins/catalog-backend/src/legacy/database/DatabaseManager.ts +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; -import knexFactory, { Knex } from 'knex'; -import { v4 as uuidv4 } from 'uuid'; -import { Logger } from 'winston'; -import { CommonDatabase } from './CommonDatabase'; -import { Database } from './types'; - -const migrationsDir = resolvePackagePath( - '@backstage/plugin-catalog-backend', - 'migrations', -); - -/** @deprecated This was part of the legacy catalog engine */ -export type CreateDatabaseOptions = { - logger: Logger; -}; - -const defaultOptions: CreateDatabaseOptions = { - logger: getVoidLogger(), -}; - -/** @deprecated This was part of the legacy catalog engine */ -export class DatabaseManager { - public static async createDatabase( - knex: Knex, - options: Partial = {}, - ): Promise { - await knex.migrate.latest({ - directory: migrationsDir, - }); - const { logger } = { ...defaultOptions, ...options }; - return new CommonDatabase(knex, logger); - } - - public static async createInMemoryDatabase(): Promise { - const knex = await this.createInMemoryDatabaseConnection(); - return await this.createDatabase(knex); - } - - public static async createInMemoryDatabaseConnection(): Promise { - const knex = knexFactory({ - client: 'sqlite3', - connection: ':memory:', - useNullAsDefault: true, - }); - - knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { - resource.run('PRAGMA foreign_keys = ON', () => {}); - }); - - return knex; - } - - public static async createTestDatabase(): Promise { - const knex = await this.createTestDatabaseConnection(); - return await this.createDatabase(knex); - } - - public static async createTestDatabaseConnection(): Promise { - const config: Knex.Config = { - /* - client: 'pg', - connection: { - host: 'localhost', - user: 'postgres', - password: 'postgres', - }, - */ - client: 'sqlite3', - connection: ':memory:', - useNullAsDefault: true, - }; - - let knex = knexFactory(config); - if (typeof config.connection !== 'string') { - const tempDbName = `d${uuidv4().replace(/-/g, '')}`; - await knex.raw(`CREATE DATABASE ${tempDbName};`); - knex = knexFactory({ - ...config, - connection: { - ...config.connection, - database: tempDbName, - }, - }); - } - - knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { - resource.run('PRAGMA foreign_keys = ON', () => {}); - }); - - return knex; - } -} diff --git a/plugins/catalog-backend/src/legacy/database/index.ts b/plugins/catalog-backend/src/legacy/database/index.ts deleted file mode 100644 index 909c1de26c..0000000000 --- a/plugins/catalog-backend/src/legacy/database/index.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { CommonDatabase } from './CommonDatabase'; -export { DatabaseManager } from './DatabaseManager'; -export type { CreateDatabaseOptions } from './DatabaseManager'; -export type { - Database, - DbEntityRequest, - DbEntityResponse, - Transaction, - DbEntitiesRequest, - DbEntitiesResponse, - DbLocationsRowWithStatus, - DatabaseLocationUpdateLogEvent, - DbLocationsRow, - DatabaseLocationUpdateLogStatus, - DbPageInfo, -} from './types'; diff --git a/plugins/catalog-backend/src/legacy/database/search.test.ts b/plugins/catalog-backend/src/legacy/database/search.test.ts deleted file mode 100644 index c060553bf8..0000000000 --- a/plugins/catalog-backend/src/legacy/database/search.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; -import { buildEntitySearch, mapToRows, traverse } from './search'; - -describe('search', () => { - describe('traverse', () => { - it('expands lists of strings to several rows', () => { - const input = { a: ['b', 'c', 'd'] }; - const output = traverse(input); - expect(output).toEqual([ - { key: 'a', value: 'b' }, - { key: 'a.b', value: true }, - { key: 'a', value: 'c' }, - { key: 'a.c', value: true }, - { key: 'a', value: 'd' }, - { key: 'a.d', value: true }, - ]); - }); - - it('expands objects', () => { - const input = { a: { b: { c: 'd' }, e: 'f' } }; - const output = traverse(input); - expect(output).toEqual([ - { key: 'a.b.c', value: 'd' }, - { key: 'a.e', value: 'f' }, - ]); - }); - - it('expands list of objects', () => { - const input = { root: { list: [{ a: 1 }, { a: 2 }] } }; - const output = traverse(input); - expect(output).toEqual([ - { key: 'root.list.a', value: 1 }, - { key: 'root.list.a', value: 2 }, - ]); - }); - - it('skips over special keys', () => { - const input = { - a: 'a', - metadata: { - b: 'b', - name: 'name', - namespace: 'namespace', - uid: 'uid', - etag: 'etag', - generation: 'generation', - c: 'c', - }, - d: 'd', - }; - const output = traverse(input); - expect(output).toEqual([ - { key: 'a', value: 'a' }, - { key: 'metadata.b', value: 'b' }, - { key: 'metadata.c', value: 'c' }, - { key: 'd', value: 'd' }, - ]); - }); - }); - - describe('mapToRows', () => { - it('converts base types to strings or null', () => { - const input = [ - { key: 'a', value: true }, - { key: 'b', value: false }, - { key: 'c', value: 7 }, - { key: 'd', value: 'string' }, - { key: 'e', value: null }, - { key: 'f', value: undefined }, - ]; - const output = mapToRows(input, 'eid'); - expect(output).toEqual([ - { entity_id: 'eid', key: 'a', value: 'true' }, - { entity_id: 'eid', key: 'b', value: 'false' }, - { entity_id: 'eid', key: 'c', value: '7' }, - { entity_id: 'eid', key: 'd', value: 'string' }, - { entity_id: 'eid', key: 'e', value: null }, - { entity_id: 'eid', key: 'f', value: null }, - ]); - }); - - it('emits lowercase version of keys and values', () => { - const input = [{ key: 'fOo', value: 'BaR' }]; - const output = mapToRows(input, 'eid'); - expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: 'bar' }]); - }); - - it('skips very large keys', () => { - const input = [{ key: 'a'.repeat(10000), value: 'foo' }]; - const output = mapToRows(input, 'eid'); - expect(output).toEqual([]); - }); - - it('skips very large values', () => { - const input = [{ key: 'foo', value: 'a'.repeat(10000) }]; - const output = mapToRows(input, 'eid'); - expect(output).toEqual([]); - }); - }); - - describe('buildEntitySearch', () => { - it('adds special keys even if missing', () => { - const input: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { name: 'n' }, - }; - expect(buildEntitySearch('eid', input)).toEqual([ - { entity_id: 'eid', key: 'apiversion', value: 'a' }, - { entity_id: 'eid', key: 'kind', value: 'b' }, - { entity_id: 'eid', key: 'metadata.name', value: 'n' }, - { entity_id: 'eid', key: 'metadata.namespace', value: null }, - { entity_id: 'eid', key: 'metadata.uid', value: null }, - { - entity_id: 'eid', - key: 'metadata.namespace', - value: ENTITY_DEFAULT_NAMESPACE, - }, - ]); - }); - }); -}); diff --git a/plugins/catalog-backend/src/legacy/database/search.ts b/plugins/catalog-backend/src/legacy/database/search.ts deleted file mode 100644 index f9366322a4..0000000000 --- a/plugins/catalog-backend/src/legacy/database/search.ts +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; -import type { DbEntitiesSearchRow } from './types'; - -// These are excluded in the generic loop, either because they do not make sense -// to index, or because they are special-case always inserted whether they are -// null or not -const SPECIAL_KEYS = [ - 'metadata.name', - 'metadata.namespace', - 'metadata.uid', - 'metadata.etag', - 'metadata.generation', -]; - -// The maximum length allowed for search values. These columns are indexed, and -// database engines do not like to index on massive values. For example, -// postgres will balk after 8191 byte line sizes. -const MAX_KEY_LENGTH = 200; -const MAX_VALUE_LENGTH = 200; - -type Kv = { - key: string; - value: unknown; -}; - -// Helper for traversing through a nested structure and outputting a list of -// path->value entries of the leaves. -// -// For example, this yaml structure -// -// a: 1 -// b: -// c: null -// e: [f, g] -// h: -// - i: 1 -// j: k -// - i: 2 -// j: l -// -// will result in -// -// "a", 1 -// "b.c", null -// "b.e": "f" -// "b.e.f": true -// "b.e": "g" -// "b.e.g": true -// "h.i": 1 -// "h.j": "k" -// "h.i": 2 -// "h.j": "l" -export function traverse(root: unknown): Kv[] { - const output: Kv[] = []; - - function visit(path: string, current: unknown) { - if (SPECIAL_KEYS.includes(path)) { - return; - } - - // empty or scalar - if ( - current === undefined || - current === null || - ['string', 'number', 'boolean'].includes(typeof current) - ) { - output.push({ key: path, value: current }); - return; - } - - // unknown - if (typeof current !== 'object') { - return; - } - - // array - if (Array.isArray(current)) { - for (const item of current) { - // NOTE(freben): The reason that these are output in two different ways, - // is to support use cases where you want to express that MORE than one - // tag is present in a list. Since the EntityFilters structure is a - // record, you can't have several entries of the same key. Therefore - // you will have to match on - // - // { "a.b": ["true"], "a.c": ["true"] } - // - // rather than - // - // { "a": ["b", "c"] } - // - // because the latter means EITHER b or c has to be present. - visit(path, item); - if (typeof item === 'string') { - output.push({ key: `${path}.${item}`, value: true }); - } - } - return; - } - - // object - for (const [key, value] of Object.entries(current!)) { - visit(path ? `${path}.${key}` : key, value); - } - } - - visit('', root); - - return output; -} - -// Translates a number of raw data rows to search table rows -export function mapToRows( - input: Kv[], - entityId: string, -): DbEntitiesSearchRow[] { - const result: DbEntitiesSearchRow[] = []; - - for (const { key: rawKey, value: rawValue } of input) { - const key = rawKey.toLowerCase(); - if (rawValue === undefined || rawValue === null) { - result.push({ entity_id: entityId, key, value: null }); - } else { - const value = String(rawValue).toLowerCase(); - if (key.length <= MAX_KEY_LENGTH && value.length <= MAX_VALUE_LENGTH) { - result.push({ entity_id: entityId, key, value }); - } - } - } - - return result; -} - -/** - * Generates all of the search rows that are relevant for this entity. - * - * @param entityId - The uid of the entity - * @param entity - The entity - * @returns A list of entity search rows - */ -export function buildEntitySearch( - entityId: string, - entity: Entity, -): DbEntitiesSearchRow[] { - // Visit the entire structure recursively - const raw = traverse(entity); - - // Start with some special keys that are always present because you want to - // be able to easily search for null specifically - raw.push({ key: 'metadata.name', value: entity.metadata.name }); - raw.push({ key: 'metadata.namespace', value: entity.metadata.namespace }); - raw.push({ key: 'metadata.uid', value: entity.metadata.uid }); - - // Namespace not specified has the default value "default", so we want to - // match on that as well - if (!entity.metadata.namespace) { - raw.push({ key: 'metadata.namespace', value: ENTITY_DEFAULT_NAMESPACE }); - } - - return mapToRows(raw, entityId); -} diff --git a/plugins/catalog-backend/src/legacy/database/types.ts b/plugins/catalog-backend/src/legacy/database/types.ts deleted file mode 100644 index c2454df261..0000000000 --- a/plugins/catalog-backend/src/legacy/database/types.ts +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { - Entity, - EntityName, - EntityRelationSpec, - Location, -} from '@backstage/catalog-model'; -import { EntityFilter, EntityPagination } from '../../catalog/types'; - -/** @deprecated This was part of the legacy catalog engine */ -export type DbEntitiesRow = { - id: string; - location_id: string | null; - etag: string; - generation: number; - full_name: string; - data: string; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type DbEntityRequest = { - locationId?: string; - entity: Entity; - relations: EntityRelationSpec[]; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type DbEntitiesRequest = { - filter?: EntityFilter; - pagination?: EntityPagination; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type DbEntitiesResponse = { - entities: DbEntityResponse[]; - pageInfo: DbPageInfo; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type DbPageInfo = - | { - hasNextPage: false; - } - | { - hasNextPage: true; - endCursor: string; - }; - -/** @deprecated This was part of the legacy catalog engine */ -export type DbEntityResponse = { - locationId?: string; - entity: Entity; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type DbEntitiesRelationsRow = { - originating_entity_id: string; - source_full_name: string; - type: string; - target_full_name: string; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type DbEntitiesSearchRow = { - entity_id: string; - key: string; - value: string | null; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type DbLocationsRow = { - id: string; - type: string; - target: string; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type DbLocationsRowWithStatus = DbLocationsRow & { - status: string | null; - timestamp: string | null; - message: string | null; -}; - -export enum DatabaseLocationUpdateLogStatus { - FAIL = 'fail', - SUCCESS = 'success', -} - -/** @deprecated This was part of the legacy catalog engine */ -export type DatabaseLocationUpdateLogEvent = { - id: string; - status: DatabaseLocationUpdateLogStatus; - location_id: string; - entity_name: string; - created_at?: string; - message?: string; -}; - -/** - * An abstraction for transactions of the underlying database technology. - * - * @deprecated This was part of the legacy catalog engine - */ -export type Transaction = { - rollback(): Promise; -}; - -/** - * An abstraction on top of the underlying database, wrapping the basic CRUD - * needs. - * @deprecated This was part of the legacy catalog engine - */ -export type Database = { - /** - * Runs a transaction. - * - * The callback is expected to make calls back into this class. When it - * completes, the transaction is closed. - * - * @param fn - The callback that implements the transaction - */ - transaction(fn: (tx: Transaction) => Promise): Promise; - - /** - * Adds a set of new entities to the catalog. - * - * @param tx - An ongoing transaction - * @param request - The entities being added - */ - addEntities( - tx: Transaction, - request: DbEntityRequest[], - ): Promise; - - /** - * Updates an existing entity in the catalog. - * - * The given entity must contain an uid to identify an already stored entity - * in the catalog. If it is missing or if no matching entity is found, the - * operation fails. - * - * If matchingEtag or matchingGeneration are given, they are taken into - * account. Attempts to update a matching entity, but where the etag and/or - * generation are not equal to the passed values, will fail. - * - * @param tx - An ongoing transaction - * @param request - The entity being updated - * @param matchingEtag - If specified, reject with ConflictError if not - * matching the entry in the database - * @param matchingGeneration - If specified, reject with ConflictError if not - * matching the entry in the database - * @returns The updated entity - */ - updateEntity( - tx: Transaction, - request: DbEntityRequest, - matchingEtag?: string, - matchingGeneration?: number, - ): Promise; - - entities( - tx: Transaction, - request?: DbEntitiesRequest, - ): Promise; - - entityByName( - tx: Transaction, - name: EntityName, - ): Promise; - - entityByUid( - tx: Transaction, - uid: string, - ): Promise; - - removeEntityByUid(tx: Transaction, uid: string): Promise; - - /** - * Remove current relations for the entity and replace them with the new - * relations array. - * - * @param tx - An ongoing transaction - * @param entityUid - The entity uid - * @param relations - The relationships to be set - */ - setRelations( - tx: Transaction, - entityUid: string, - relations: EntityRelationSpec[], - ): Promise; - - addLocation(tx: Transaction, location: Location): Promise; - - removeLocation(tx: Transaction, id: string): Promise; - - location(id: string): Promise; - - locations(): Promise; - - locationHistory(id: string): Promise; - - addLocationUpdateLogEvent( - locationId: string, - status: DatabaseLocationUpdateLogStatus, - entityName?: string | string[], - message?: string, - ): Promise; -}; diff --git a/plugins/catalog-backend/src/legacy/index.ts b/plugins/catalog-backend/src/legacy/index.ts deleted file mode 100644 index be8c7b8d63..0000000000 --- a/plugins/catalog-backend/src/legacy/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export * from './catalog'; -export * from './ingestion'; -export * from './service'; -export * from './database'; diff --git a/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts b/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts deleted file mode 100644 index e6c042421b..0000000000 --- a/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts +++ /dev/null @@ -1,415 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getVoidLogger } from '@backstage/backend-common'; -import { Entity, Location, LocationSpec } from '@backstage/catalog-model'; -import { EntitiesCatalog } from '../../catalog'; -import { LocationsCatalog } from '../catalog'; -import { LocationUpdateStatus } from '../catalog/types'; -import { DatabaseLocationUpdateLogStatus } from '../database/types'; -import { HigherOrderOperations } from './HigherOrderOperations'; -import { LocationReader } from './types'; - -describe('HigherOrderOperations', () => { - let entitiesCatalog: jest.Mocked>; - let locationsCatalog: jest.Mocked; - let locationReader: jest.Mocked; - let higherOrderOperation: HigherOrderOperations; - - beforeAll(() => { - entitiesCatalog = { - entities: jest.fn(), - removeEntityByUid: jest.fn(), - batchAddOrUpdateEntities: jest.fn(), - entityAncestry: jest.fn(), - }; - locationsCatalog = { - addLocation: jest.fn(), - removeLocation: jest.fn(), - locations: jest.fn(), - location: jest.fn(), - locationHistory: jest.fn(), - logUpdateSuccess: jest.fn(), - logUpdateFailure: jest.fn(), - }; - locationReader = { - read: jest.fn(), - }; - higherOrderOperation = new HigherOrderOperations( - entitiesCatalog, - locationsCatalog, - locationReader, - getVoidLogger(), - ); - }); - - beforeEach(() => { - jest.resetAllMocks(); - }); - - describe('addLocation', () => { - it('just inserts the location when there are no entities to read', async () => { - const spec = { - type: 'a', - target: 'b', - }; - locationsCatalog.addLocation.mockImplementation(x => Promise.resolve(x)); - locationsCatalog.locations.mockResolvedValue([]); - locationReader.read.mockResolvedValue({ - entities: [], - errors: [], - }); - - const result = await higherOrderOperation.addLocation(spec); - - expect(result.location).toEqual( - expect.objectContaining({ - id: expect.anything(), - ...spec, - }), - ); - expect(result.entities).toEqual([]); - expect(locationsCatalog.locations).toBeCalledTimes(1); - expect(locationReader.read).toBeCalledTimes(1); - expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' }); - expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled(); - expect(locationsCatalog.addLocation).toBeCalledTimes(1); - expect(locationsCatalog.addLocation).toBeCalledWith( - expect.objectContaining({ - id: expect.anything(), - ...spec, - }), - ); - }); - - it('insert the location and its entities', async () => { - const spec = { - type: 'a', - target: 'b', - }; - const location: LocationSpec = { type: '', target: '' }; - const entity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { name: 'n' }, - }; - locationsCatalog.addLocation.mockImplementation(x => Promise.resolve(x)); - locationsCatalog.locations.mockResolvedValue([]); - locationsCatalog.locations.mockResolvedValue([]); - entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([ - { - entityId: 'id', - entity, - }, - ]); - - locationReader.read.mockResolvedValue({ - entities: [ - { - location, - entity, - relations: [], - }, - ], - errors: [], - }); - - const result = await higherOrderOperation.addLocation(spec); - - expect(result.location).toEqual( - expect.objectContaining({ - id: expect.anything(), - ...spec, - }), - ); - expect(result.entities).toEqual([entity]); - expect(locationsCatalog.locations).toBeCalledTimes(1); - expect(locationsCatalog.addLocation).toBeCalledTimes(1); - expect(locationsCatalog.addLocation).toBeCalledWith( - expect.objectContaining({ - id: expect.anything(), - ...spec, - }), - ); - expect(locationReader.read).toBeCalledTimes(1); - expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' }); - expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledTimes(1); - expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledWith( - expect.anything(), - expect.objectContaining({ - locationId: expect.anything(), - dryRun: false, - outputEntities: true, - }), - ); - }); - - it('reuses the location if a match already existed', async () => { - const spec = { - type: 'a', - target: 'b', - }; - const location = { - id: 'dd12620d-0436-422f-93bd-929aa0788123', - ...spec, - }; - - locationsCatalog.locations.mockResolvedValue([ - { - currentStatus: { timestamp: '', status: '', message: '' }, - data: location, - }, - ]); - locationReader.read.mockResolvedValue({ - entities: [], - errors: [], - }); - - const result = await higherOrderOperation.addLocation(spec); - - expect(result.location).toEqual(location); - expect(result.entities).toEqual([]); - expect(locationsCatalog.locations).toBeCalledTimes(1); - expect(locationReader.read).toBeCalledTimes(1); - expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' }); - expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled(); - expect(locationsCatalog.addLocation).not.toBeCalled(); - }); - - it('rejects the whole operation if any entity could not be read', async () => { - const spec = { - type: 'a', - target: 'b', - }; - const location: LocationSpec = { type: '', target: '' }; - const entity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { name: 'n' }, - }; - - locationsCatalog.locations.mockResolvedValue([]); - locationReader.read.mockResolvedValue({ - entities: [{ entity, location, relations: [] }], - errors: [{ error: new Error('abcd'), location }], - }); - - await expect(higherOrderOperation.addLocation(spec)).rejects.toThrow( - /abcd/, - ); - expect(locationsCatalog.locations).toBeCalledTimes(1); - expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled(); - expect(locationsCatalog.addLocation).not.toBeCalled(); - }); - - it('rollback everything after a dry run', async () => { - const spec = { - type: 'a', - target: 'b', - }; - const location: LocationSpec = { type: '', target: '' }; - const entity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { name: 'n' }, - }; - locationsCatalog.locations.mockResolvedValue([]); - locationsCatalog.locations.mockResolvedValue([]); - entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([ - { - entityId: 'id', - entity, - }, - ]); - locationReader.read.mockResolvedValue({ - entities: [ - { - location, - entity, - relations: [], - }, - ], - errors: [], - }); - - const result = await higherOrderOperation.addLocation(spec, { - dryRun: true, - }); - - expect(result.location).toEqual( - expect.objectContaining({ - id: expect.anything(), - ...spec, - }), - ); - expect(result.entities).toEqual([entity]); - expect(locationsCatalog.locations).toBeCalledTimes(1); - expect(locationReader.read).toBeCalledTimes(1); - expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' }); - expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledTimes(1); - expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledWith( - expect.anything(), - expect.objectContaining({ - dryRun: true, - outputEntities: true, - }), - ); - }); - }); - - describe('refreshLocations', () => { - it('works with no locations added', async () => { - locationsCatalog.locations.mockResolvedValue([]); - - await expect( - higherOrderOperation.refreshAllLocations(), - ).resolves.toBeUndefined(); - - expect(locationsCatalog.locations).toHaveBeenCalledTimes(1); - expect(locationReader.read).not.toHaveBeenCalled(); - expect(entitiesCatalog.batchAddOrUpdateEntities).not.toHaveBeenCalled(); - }); - - it('can update a single location where a matching entity did not exist', async () => { - const locationStatus: LocationUpdateStatus = { - message: '', - status: DatabaseLocationUpdateLogStatus.SUCCESS, - timestamp: new Date(314159265).toISOString(), - }; - const location: Location = { - id: '123', - type: 'some', - target: 'thing', - }; - const desc: Entity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { name: 'c1' }, - spec: { type: 'service' }, - }; - const entityId = 'xyz123'; - - locationsCatalog.locations.mockResolvedValue([ - { currentStatus: locationStatus, data: location }, - ]); - locationReader.read.mockResolvedValue({ - entities: [{ entity: desc, location, relations: [] }], - errors: [], - }); - entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([ - { entityId }, - ]); - - await expect( - higherOrderOperation.refreshAllLocations(), - ).resolves.toBeUndefined(); - - expect(locationsCatalog.locations).toHaveBeenCalledTimes(1); - expect(locationReader.read).toHaveBeenCalledTimes(1); - expect(locationReader.read).toHaveBeenNthCalledWith(1, { - type: 'some', - target: 'thing', - }); - expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledWith( - [ - expect.objectContaining({ - entity: expect.objectContaining({ metadata: { name: 'c1' } }), - relations: [], - }), - ], - { - locationId: '123', - }, - ); - }); - - it('logs successful updates', async () => { - const locationStatus: LocationUpdateStatus = { - message: '', - status: DatabaseLocationUpdateLogStatus.SUCCESS, - timestamp: new Date(314159265).toISOString(), - }; - const location: Location = { - id: '123', - type: 'some', - target: 'thing', - }; - const desc: Entity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { name: 'c1' }, - spec: { type: 'service' }, - }; - - locationsCatalog.locations.mockResolvedValue([ - { currentStatus: locationStatus, data: location }, - ]); - locationReader.read.mockResolvedValue({ - entities: [{ entity: desc, location, relations: [] }], - errors: [], - }); - entitiesCatalog.entities.mockResolvedValue({ - entities: [], - pageInfo: { hasNextPage: false }, - }); - entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([]); - - await expect( - higherOrderOperation.refreshAllLocations(), - ).resolves.toBeUndefined(); - - expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledTimes(2); - expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith( - '123', - undefined, - ); - expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith('123', [ - 'c1', - ]); - }); - - it('logs unsuccessful updates when reader fails', async () => { - const locationStatus: LocationUpdateStatus = { - message: '', - status: DatabaseLocationUpdateLogStatus.SUCCESS, - timestamp: new Date(314159265).toISOString(), - }; - const location: Location = { - id: '123', - type: 'some', - target: 'thing', - }; - - locationsCatalog.locations.mockResolvedValue([ - { currentStatus: locationStatus, data: location }, - ]); - locationReader.read.mockRejectedValue(new Error('reader error message')); - - await expect( - higherOrderOperation.refreshAllLocations(), - ).resolves.toBeUndefined(); - - expect(locationReader.read).toHaveBeenCalledTimes(1); - expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledTimes(1); - expect(locationsCatalog.logUpdateSuccess).not.toHaveBeenCalled(); - expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledWith( - '123', - expect.objectContaining({ message: 'reader error message' }), - ); - }); - }); -}); diff --git a/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.ts b/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.ts deleted file mode 100644 index e57f149485..0000000000 --- a/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.ts +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Location, - LocationSpec, - stringifyLocationReference, -} from '@backstage/catalog-model'; -import { v4 as uuidv4 } from 'uuid'; -import { Logger } from 'winston'; -import { EntitiesCatalog } from '../../catalog'; -import { LocationsCatalog } from '../catalog'; -import { durationText } from '../../util'; -import { - AddLocationResult, - HigherOrderOperation, - LocationReader, -} from './types'; - -/** - * Placeholder for operations that span several catalogs and/or stretches out - * in time. - * - * @deprecated This was part of the legacy catalog engine - */ -export class HigherOrderOperations implements HigherOrderOperation { - constructor( - private readonly entitiesCatalog: EntitiesCatalog, - private readonly locationsCatalog: LocationsCatalog, - private readonly locationReader: LocationReader, - private readonly logger: Logger, - ) {} - - /** - * Adds a single location to the catalog. - * - * The location is inspected and fetched, and all of the resulting data is - * validated. If everything goes well, the location and entities are stored - * in the catalog. - * - * If the location already existed, the old location is returned instead and - * the catalog is left unchanged. - * - * @param spec - The location to add - */ - async addLocation( - spec: LocationSpec, - options?: { dryRun?: boolean }, - ): Promise { - const dryRun = options?.dryRun || false; - - // Attempt to find a previous location matching the spec - const previousLocations = await this.locationsCatalog.locations(); - const previousLocation = previousLocations.find( - l => spec.type === l.data.type && spec.target === l.data.target, - ); - const location: Location = previousLocation - ? previousLocation.data - : { - id: uuidv4(), - type: spec.type, - target: spec.target, - }; - - // Read the location fully, bailing on any errors - const readerOutput = await this.locationReader.read(spec); - if (!(spec.presence === 'optional') && readerOutput.errors.length) { - const item = readerOutput.errors[0]; - throw item.error; - } - - // TODO(freben): At this point, we could detect orphaned entities, by way - // of having a location annotation pointing to the location but not being - // in the entities list. But we aren't sure what to do about those yet. - - // Write - if (!previousLocation && !dryRun) { - // TODO: We do not include location operations in the dryRun. We might perform - // this operation as a separate dry run. - await this.locationsCatalog.addLocation(location); - } - if (readerOutput.entities.length === 0) { - return { location, entities: [] }; - } - - const writtenEntities = await this.entitiesCatalog - .batchAddOrUpdateEntities!(readerOutput.entities, { - locationId: dryRun ? undefined : location.id, - dryRun, - outputEntities: true, - }); - - const entities = writtenEntities.map(e => e.entity!); - - return { location, entities }; - } - - /** - * Goes through all registered locations, and performs a refresh of each one. - * - * Entities are read from their respective sources, are parsed and validated - * according to the entity policy, and get inserted or updated in the catalog. - * Entities that have disappeared from their location are left orphaned, - * without changes. - */ - async refreshAllLocations(): Promise { - const startTimestamp = process.hrtime(); - const logger = this.logger.child({ - component: 'catalog-all-locations-refresh', - }); - - logger.info('Locations Refresh: Beginning locations refresh'); - - const locations = await this.locationsCatalog.locations(); - logger.info(`Locations Refresh: Visiting ${locations.length} locations`); - - for (const { data: location } of locations) { - logger.info( - `Locations Refresh: Refreshing location ${stringifyLocationReference( - location, - )}`, - ); - try { - await this.refreshSingleLocation(location, logger); - await this.locationsCatalog.logUpdateSuccess(location.id, undefined); - } catch (e) { - logger.warn( - `Locations Refresh: Failed to refresh location ${stringifyLocationReference( - location, - )}, ${e.stack}`, - ); - await this.locationsCatalog.logUpdateFailure(location.id, e); - } - } - - logger.info( - `Locations Refresh: Completed locations refresh in ${durationText( - startTimestamp, - )}`, - ); - } - - // Performs a full refresh of a single location - private async refreshSingleLocation( - location: Location, - optionalLogger?: Logger, - ) { - let startTimestamp = process.hrtime(); - const logger = optionalLogger || this.logger; - - const readerOutput = await this.locationReader.read({ - type: location.type, - target: location.target, - }); - - for (const item of readerOutput.errors) { - logger.warn( - `Failed item in location ${stringifyLocationReference( - item.location, - )}, ${item.error.stack}`, - ); - } - - logger.info( - `Read ${ - readerOutput.entities.length - } entities from location ${stringifyLocationReference( - location, - )} in ${durationText(startTimestamp)}`, - ); - - startTimestamp = process.hrtime(); - - try { - await this.entitiesCatalog.batchAddOrUpdateEntities!( - readerOutput.entities, - { locationId: location.id }, - ); - } catch (e) { - for (const entity of readerOutput.entities) { - await this.locationsCatalog.logUpdateFailure( - location.id, - e, - entity.entity.metadata.name, - ); - } - throw e; - } - - logger.debug(`Posting update success markers`); - - await this.locationsCatalog.logUpdateSuccess( - location.id, - readerOutput.entities.map(e => e.entity.metadata.name), - ); - - logger.info( - `Wrote ${ - readerOutput.entities.length - } entities from location ${stringifyLocationReference( - location, - )} in ${durationText(startTimestamp)}`, - ); - } -} diff --git a/plugins/catalog-backend/src/legacy/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/legacy/ingestion/LocationReaders.ts deleted file mode 100644 index 04347615e5..0000000000 --- a/plugins/catalog-backend/src/legacy/ingestion/LocationReaders.ts +++ /dev/null @@ -1,353 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { NotAllowedError } from '@backstage/errors'; -import { UrlReader } from '@backstage/backend-common'; -import { - Entity, - EntityPolicy, - EntityRelationSpec, - ENTITY_DEFAULT_NAMESPACE, - LocationSpec, - stringifyLocationReference, -} from '@backstage/catalog-model'; -import { Config } from '@backstage/config'; -import { Logger } from 'winston'; -import { CatalogRulesEnforcer } from '../../ingestion/CatalogRules'; -import * as result from '../../ingestion/processors/results'; -import { - CatalogProcessor, - CatalogProcessorEmit, - CatalogProcessorEntityResult, - CatalogProcessorErrorResult, - CatalogProcessorLocationResult, - CatalogProcessorParser, - CatalogProcessorResult, -} from '../../ingestion/processors/types'; -import { LocationReader, ReadLocationResult } from './types'; - -// The max amount of nesting depth of generated work items -const MAX_DEPTH = 10; - -type Options = { - reader: UrlReader; - parser: CatalogProcessorParser; - logger: Logger; - config: Config; - processors: CatalogProcessor[]; - rulesEnforcer: CatalogRulesEnforcer; - policy: EntityPolicy; -}; - -const noopCache = { - async get() { - return undefined; - }, - async set() {}, -}; - -/** - * Implements the reading of a location through a series of processor tasks. - * - * @deprecated This was part of the legacy catalog engine - */ -export class LocationReaders implements LocationReader { - private readonly options: Options; - - constructor(options: Options) { - this.options = options; - } - - async read(location: LocationSpec): Promise { - const { rulesEnforcer, logger } = this.options; - - const output: ReadLocationResult = { - entities: [], - errors: [], - }; - let items: CatalogProcessorResult[] = [result.location(location, false)]; - - for (let depth = 0; depth < MAX_DEPTH; ++depth) { - const newItems: CatalogProcessorResult[] = []; - const emit: CatalogProcessorEmit = i => newItems.push(i); - - for (const item of items) { - if (item.type === 'location') { - await this.handleLocation(item, emit); - } else if (item.type === 'entity') { - if (rulesEnforcer.isAllowed(item.entity, item.location)) { - const relations = Array(); - - const entity = await this.handleEntity( - item, - emitResult => { - if (emitResult.type === 'relation') { - relations.push(emitResult.relation); - return; - } - emit(emitResult); - }, - location, - ); - - if (entity) { - output.entities.push({ - entity, - location: item.location, - relations, - }); - } - } else { - output.errors.push({ - location: item.location, - error: new NotAllowedError( - `Entity of kind ${ - item.entity.kind - } is not allowed from location ${stringifyLocationReference( - item.location, - )}`, - ), - }); - } - } else if (item.type === 'error') { - await this.handleError(item, emit); - output.errors.push({ - location: item.location, - error: item.error, - }); - } - } - - if (newItems.length === 0) { - return output; - } - - items = newItems; - } - - const message = `Max recursion depth ${MAX_DEPTH} reached for location ${location.type} ${location.target}`; - logger.warn(message); - output.errors.push({ location, error: new Error(message) }); - return output; - } - - private async handleLocation( - item: CatalogProcessorLocationResult, - emit: CatalogProcessorEmit, - ) { - const { processors, logger } = this.options; - - const validatedEmit: CatalogProcessorEmit = emitResult => { - if (emitResult.type === 'relation') { - throw new Error('readLocation may not emit entity relations'); - } - if ( - emitResult.type === 'location' && - emitResult.location.type === item.location.type && - emitResult.location.target === item.location.target - ) { - // Ignore self-referential locations silently (this can happen for - // example if you use a glob target like "**/*.yaml" in a Location - // entity) - return; - } - emit(emitResult); - }; - - for (const processor of processors) { - if (processor.readLocation) { - try { - if ( - await processor.readLocation( - item.location, - item.optional, - validatedEmit, - this.options.parser, - noopCache, - ) - ) { - return; - } - } catch (e) { - const message = `Processor ${ - processor.constructor.name - } threw an error while reading location ${stringifyLocationReference( - item.location, - )}, ${e}`; - emit(result.generalError(item.location, message)); - logger.warn(message); - } - } - } - - const message = `No processor was able to read location ${stringifyLocationReference( - item.location, - )}`; - emit(result.inputError(item.location, message)); - logger.warn(message); - } - - private async handleEntity( - item: CatalogProcessorEntityResult, - emit: CatalogProcessorEmit, - originLocation: LocationSpec, - ): Promise { - const { processors, logger } = this.options; - - let current = item.entity; - - // Construct the name carefully, this happens before validation below - // so we do not want to crash here due to missing metadata or so - const kind = current.kind || ''; - const namespace = !current.metadata - ? '' - : current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE; - const name = !current.metadata ? '' : current.metadata.name; - - for (const processor of processors) { - if (processor.preProcessEntity) { - try { - current = await processor.preProcessEntity( - current, - item.location, - emit, - originLocation, - noopCache, - ); - } catch (e) { - const message = `Processor ${ - processor.constructor.name - } threw an error while preprocessing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference( - item.location, - )}, ${e}`; - emit(result.generalError(item.location, e.message)); - logger.warn(message); - return undefined; - } - } - } - - try { - const next = await this.options.policy.enforce(current); - if (!next) { - const message = `Policy unexpectedly returned no data while analyzing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference( - item.location, - )}`; - emit(result.generalError(item.location, message)); - logger.warn(message); - return undefined; - } - current = next; - } catch (e) { - const message = `Policy check failed while analyzing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference( - item.location, - )}, ${e}`; - emit(result.inputError(item.location, message)); - logger.warn(message); - return undefined; - } - - let handled = false; - for (const processor of processors) { - if (processor.validateEntityKind) { - try { - handled = await processor.validateEntityKind(current); - if (handled) { - break; - } - } catch (e) { - const message = `Processor ${ - processor.constructor.name - } threw an error while validating the entity ${kind}:${namespace}/${name} at ${stringifyLocationReference( - item.location, - )}, ${e}`; - emit(result.inputError(item.location, message)); - logger.warn(message); - return undefined; - } - } - } - if (!handled) { - const message = `No processor recognized the entity ${kind}:${namespace}/${name} at ${stringifyLocationReference( - item.location, - )}`; - emit(result.inputError(item.location, message)); - logger.warn(message); - return undefined; - } - - for (const processor of processors) { - if (processor.postProcessEntity) { - try { - current = await processor.postProcessEntity( - current, - item.location, - emit, - noopCache, - ); - } catch (e) { - const message = `Processor ${ - processor.constructor.name - } threw an error while postprocessing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference( - item.location, - )}, ${e}`; - emit(result.generalError(item.location, message)); - logger.warn(message); - return undefined; - } - } - } - - return current; - } - - private async handleError( - item: CatalogProcessorErrorResult, - emit: CatalogProcessorEmit, - ) { - const { processors, logger } = this.options; - - logger.debug( - `Encountered error at location ${stringifyLocationReference( - item.location, - )}, ${item.error}`, - ); - - const validatedEmit: CatalogProcessorEmit = emitResult => { - if (emitResult.type === 'relation') { - throw new Error('handleError may not emit entity relations'); - } - - emit(emitResult); - }; - - for (const processor of processors) { - if (processor.handleError) { - try { - await processor.handleError(item.error, item.location, validatedEmit); - } catch (e) { - const message = `Processor ${ - processor.constructor.name - } threw an error while handling another error at ${stringifyLocationReference( - item.location, - )}, ${e}`; - emit(result.generalError(item.location, message)); - logger.warn(message); - } - } - } - } -} diff --git a/plugins/catalog-backend/src/legacy/ingestion/index.ts b/plugins/catalog-backend/src/legacy/ingestion/index.ts deleted file mode 100644 index fa63f076d6..0000000000 --- a/plugins/catalog-backend/src/legacy/ingestion/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { HigherOrderOperations } from './HigherOrderOperations'; -export { LocationReaders } from './LocationReaders'; -export type { - AddLocationResult, - HigherOrderOperation, - LocationReader, - ReadLocationEntity, - ReadLocationError, - ReadLocationResult, -} from './types'; diff --git a/plugins/catalog-backend/src/legacy/ingestion/types.ts b/plugins/catalog-backend/src/legacy/ingestion/types.ts deleted file mode 100644 index 0a02a5e17e..0000000000 --- a/plugins/catalog-backend/src/legacy/ingestion/types.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Entity, - EntityRelationSpec, - Location, - LocationSpec, -} from '@backstage/catalog-model'; - -// -// LocationReader -// - -/** @deprecated This was part of the legacy catalog engine */ -export type HigherOrderOperation = { - addLocation( - spec: LocationSpec, - options?: { dryRun?: boolean }, - ): Promise; - refreshAllLocations(): Promise; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type AddLocationResult = { - location: Location; - entities: Entity[]; -}; - -// -// LocationReader -// - -/** @deprecated This was part of the legacy catalog engine */ -export type LocationReader = { - /** - * Reads the contents of a location. - * - * @param location - The location to read - * @throws An error if the location was handled by this reader, but could not - * be read - */ - read(location: LocationSpec): Promise; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type ReadLocationResult = { - entities: ReadLocationEntity[]; - errors: ReadLocationError[]; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type ReadLocationEntity = { - location: LocationSpec; - entity: Entity; - relations: EntityRelationSpec[]; -}; - -/** @deprecated This was part of the legacy catalog engine */ -export type ReadLocationError = { - location: LocationSpec; - error: Error; -}; diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts deleted file mode 100644 index 728d35ea3d..0000000000 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - getVoidLogger, - PluginEndpointDiscovery, - ServerTokenManager, - UrlReader, -} from '@backstage/backend-common'; -import { Entity } from '@backstage/catalog-model'; -import { ConfigReader } from '@backstage/config'; -import { Knex } from 'knex'; -import yaml from 'yaml'; -import { DatabaseManager } from '../database'; -import { CatalogProcessorParser } from '../../ingestion'; -import * as result from '../../ingestion/processors/results'; -import { CatalogBuilder } from './CatalogBuilder'; -import { CatalogEnvironment } from '../../service'; -import { ServerPermissionClient } from '@backstage/plugin-permission-node'; - -const dummyEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'n', - }, - spec: { - type: 't', - owner: 'o', - lifecycle: 'l', - }, -}; - -const dummyEntityYaml = yaml.stringify(dummyEntity); - -describe('CatalogBuilder', () => { - let db: Knex; - const reader: jest.Mocked = { - read: jest.fn(), - readTree: jest.fn(), - search: jest.fn(), - }; - const config = new ConfigReader({}); - const mockBaseUrl = 'http://backstage:9191/i-am-a-mock-base'; - const discovery: PluginEndpointDiscovery = { - async getBaseUrl() { - return mockBaseUrl; - }, - async getExternalBaseUrl() { - return mockBaseUrl; - }, - }; - const env: CatalogEnvironment = { - logger: getVoidLogger(), - database: { getClient: async () => db }, - config, - reader, - permissions: ServerPermissionClient.fromConfig(config, { - discovery, - tokenManager: ServerTokenManager.noop(), - }), - }; - - beforeEach(async () => { - db = await DatabaseManager.createTestDatabaseConnection(); - jest.resetAllMocks(); - }); - - it('works with no changes', async () => { - const builder = new CatalogBuilder(env); - const built = await builder.build(); - await expect(built.entitiesCatalog.entities()).resolves.toEqual({ - entities: [], - pageInfo: { hasNextPage: false }, - }); - await expect(built.locationsCatalog.locations()).resolves.toEqual([ - expect.objectContaining({ - data: expect.objectContaining({ type: 'bootstrap' }), - }), - ]); - }); - - it('works with everything replaced', async () => { - reader.read.mockResolvedValueOnce(Buffer.from('junk')); - - const builder = new CatalogBuilder(env) - .replaceEntityPolicies([ - { - async enforce(entity: Entity) { - expect(entity.metadata.namespace).toBe('ns'); - return entity; - }, - }, - ]) - .setPlaceholderResolver('t', async ({ value }) => { - expect(value).toBe('tt'); - return 'tt2'; - }) - .setFieldFormatValidators({ - isValidEntityName: n => { - expect(n).toBe('n'); - return true; - }, - }) - .replaceProcessors([ - { - async readLocation(location, _optional, emit) { - expect(location.type).toBe('test'); - emit( - result.entity(location, { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { name: 'n', replaced: { $t: 'tt' } }, - spec: { type: 't', owner: 'o', lifecycle: 'l' }, - }), - ); - return true; - }, - async preProcessEntity(entity) { - expect(entity.apiVersion).toBe('backstage.io/v1alpha1'); - return { - ...entity, - metadata: { ...entity.metadata, namespace: 'ns' }, - }; - }, - async postProcessEntity(entity) { - expect(entity.metadata.namespace).toBe('ns'); - return { - ...entity, - metadata: { ...entity.metadata, post: 'p' }, - }; - }, - }, - ]); - const out = await builder.build(); - - const added = await out.higherOrderOperation.addLocation({ - type: 'test', - target: '', - }); - expect.assertions(6); - expect(added.entities).toEqual([ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'n', - namespace: 'ns', - post: 'p', - replaced: 'tt2', - uid: expect.any(String), - etag: expect.any(String), - generation: expect.any(Number), - }, - spec: { - type: 't', - owner: 'o', - lifecycle: 'l', - }, - relations: expect.anything(), - }, - ]); - }); - - it('addProcessor works', async () => { - reader.read.mockResolvedValueOnce(Buffer.from(dummyEntityYaml)); - - const builder = new CatalogBuilder(env); - builder.addProcessor({ - async preProcessEntity(e) { - return { ...e, metadata: { ...e.metadata, foo: 7 } }; - }, - }); - - const { entitiesCatalog, higherOrderOperation } = await builder.build(); - await higherOrderOperation.addLocation({ - type: 'url', - target: 'https://github.com/a/b/x.yaml', - }); - const { entities } = await entitiesCatalog.entities(); - - expect(entities).toEqual([ - expect.objectContaining({ - metadata: expect.objectContaining({ - foo: 7, - }), - }), - ]); - }); - - it('replaceProcessors works', async () => { - reader.read.mockResolvedValueOnce(Buffer.from(dummyEntityYaml)); - - const builder = new CatalogBuilder(env); - builder.replaceProcessors([ - { - async readLocation(location, _optional, emit) { - expect(location.type).toBe('x'); - emit(result.entity(location, dummyEntity)); - return true; - }, - async preProcessEntity(e) { - expect(e.metadata.name).toBe('n'); - return { ...e, metadata: { ...e.metadata, foo: 7 } }; - }, - }, - ]); - - const { entitiesCatalog, higherOrderOperation } = await builder.build(); - await higherOrderOperation.addLocation({ - type: 'x', - target: 'y', - }); - const { entities } = await entitiesCatalog.entities(); - - expect.assertions(3); - expect(entities).toEqual([ - expect.objectContaining({ - metadata: expect.objectContaining({ - foo: 7, - }), - }), - ]); - }); - - it('setEntityDataParser works', async () => { - const mockParser: CatalogProcessorParser = jest - .fn() - .mockImplementation(() => {}); - - const builder = new CatalogBuilder(env) - .setEntityDataParser(mockParser) - .replaceProcessors([ - { - async readLocation(_location, _optional, _emit, parser) { - expect(parser).toBe(mockParser); - return true; - }, - }, - ]); - - const { higherOrderOperation } = await builder.build(); - await higherOrderOperation.addLocation({ type: 'x', target: 'y' }); - - expect.assertions(1); - }); -}); diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts deleted file mode 100644 index a94fd32484..0000000000 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts +++ /dev/null @@ -1,373 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - DefaultNamespaceEntityPolicy, - EntityPolicies, - EntityPolicy, - FieldFormatEntityPolicy, - makeValidator, - NoForeignRootFieldsEntityPolicy, - SchemaValidEntityPolicy, - Validators, -} from '@backstage/catalog-model'; -import { - ScmIntegrations, - DefaultGithubCredentialsProvider, -} from '@backstage/integration'; -import lodash from 'lodash'; -import { EntitiesCatalog } from '../../catalog'; -import { - DatabaseEntitiesCatalog, - DatabaseLocationsCatalog, - LocationsCatalog, -} from '../catalog'; -import { DatabaseManager } from '../database'; -import { - AnnotateLocationEntityProcessor, - BitbucketDiscoveryProcessor, - BuiltinKindsEntityProcessor, - CatalogProcessor, - CatalogProcessorParser, - CodeOwnersProcessor, - FileReaderProcessor, - GithubDiscoveryProcessor, - AzureDevOpsDiscoveryProcessor, - GithubOrgReaderProcessor, - GitLabDiscoveryProcessor, - LocationEntityProcessor, - PlaceholderProcessor, - PlaceholderResolver, - StaticLocationProcessor, - UrlReaderProcessor, -} from '../../ingestion'; -import { - HigherOrderOperation, - HigherOrderOperations, - LocationReaders, -} from '../ingestion'; -import { DefaultCatalogRulesEnforcer } from '../../ingestion/CatalogRules'; -import { RepoLocationAnalyzer } from '../../ingestion/LocationAnalyzer'; -import { - jsonPlaceholderResolver, - textPlaceholderResolver, - yamlPlaceholderResolver, -} from '../../ingestion/processors/PlaceholderProcessor'; -import { defaultEntityDataParser } from '../../ingestion/processors/util/parse'; -import { LocationAnalyzer } from '../../ingestion/types'; -import { CatalogEnvironment } from '../../service'; - -/** - * A builder that helps wire up all of the component parts of the catalog. - * - * The touch points where you can replace or extend behavior are as follows: - * - * - Entity policies can be added or replaced. These are automatically run - * after the processors' pre-processing steps. All policies are given the - * chance to inspect the entity, and all of them have to pass in order for - * the entity to be considered valid from an overall point of view. - * - Placeholder resolvers can be replaced or added. These run on the raw - * structured data between the parsing and pre-processing steps, to replace - * dollar-prefixed entries with their actual values (like $file). - * - Field format validators can be replaced. These check the format of - * individual core fields such as metadata.name, to ensure that they adhere - * to certain rules. - * - Processors can be added or replaced. These implement the functionality of - * reading, parsing, validating, and processing the entity data before it is - * persisted in the catalog. - * - * NOTE(freben): Not actually marking the class as deprecated formally, since - * it would appear to end users that even using `create` is deprecated. We will - * instead hot-swap the entire exported class when we are ready. - */ -export class CatalogBuilder { - private readonly env: CatalogEnvironment; - private entityPolicies: EntityPolicy[]; - private entityPoliciesReplace: boolean; - private placeholderResolvers: Record; - private fieldFormatValidators: Partial; - private processors: CatalogProcessor[]; - private processorsReplace: boolean; - private parser: CatalogProcessorParser | undefined; - - static async create(env: CatalogEnvironment): Promise { - return new CatalogBuilder(env); - } - - /** @deprecated Please use CatalogBuilder.create() instead */ - constructor(env: CatalogEnvironment) { - this.env = env; - this.entityPolicies = []; - this.entityPoliciesReplace = false; - this.placeholderResolvers = {}; - this.fieldFormatValidators = {}; - this.processors = []; - this.processorsReplace = false; - this.parser = undefined; - - env.logger.warn( - "Creating the catalog with 'new CatalogBuilder(env)' is deprecated! Use CatalogBuilder.create(env) instead", - ); - } - - /** - * Adds policies that are used to validate entities between the pre- - * processing and post-processing stages. All such policies must pass for the - * entity to be considered valid. - * - * If what you want to do is to replace the rules for what format is allowed - * in various core entity fields (such as metadata.name), you may want to use - * {@link CatalogBuilder#setFieldFormatValidators} instead. - * - * @param policies - One or more policies - */ - addEntityPolicy(...policies: EntityPolicy[]): CatalogBuilder { - this.entityPolicies.push(...policies); - return this; - } - - /** - * Sets what policies to use for validation of entities between the pre- - * processing and post-processing stages. All such policies must pass for the - * entity to be considered valid. - * - * If what you want to do is to replace the rules for what format is allowed - * in various core entity fields (such as metadata.name), you may want to use - * {@link CatalogBuilder#setFieldFormatValidators} instead. - * - * This function replaces the default set of policies; use with care. - * - * @param policies - One or more policies - */ - replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder { - this.entityPolicies = [...policies]; - this.entityPoliciesReplace = true; - return this; - } - - /** - * Adds, or overwrites, a handler for placeholders (e.g. $file) in entity - * definition files. - * - * @param key - The key that identifies the placeholder, e.g. "file" - * @param resolver - The resolver that gets values for this placeholder - */ - setPlaceholderResolver( - key: string, - resolver: PlaceholderResolver, - ): CatalogBuilder { - this.placeholderResolvers[key] = resolver; - return this; - } - - /** - * Sets the validator function to use for one or more special fields of an - * entity. This is useful if the default rules for formatting of fields are - * not sufficient. - * - * This function has no effect if used together with - * {@link CatalogBuilder#replaceEntityPolicies}. - * - * @param validators - The (subset of) validators to set - */ - setFieldFormatValidators(validators: Partial): CatalogBuilder { - lodash.merge(this.fieldFormatValidators, validators); - return this; - } - - /** - * Adds entity processors. These are responsible for reading, parsing, and - * processing entities before they are persisted in the catalog. - * - * @param processors - One or more processors - */ - addProcessor(...processors: CatalogProcessor[]): CatalogBuilder { - this.processors.push(...processors); - return this; - } - - /** - * Sets what entity processors to use. These are responsible for reading, - * parsing, and processing entities before they are persisted in the catalog. - * - * This function replaces the default set of processors; use with care. - * - * @param processors - One or more processors - */ - replaceProcessors(processors: CatalogProcessor[]): CatalogBuilder { - this.processors = [...processors]; - this.processorsReplace = true; - return this; - } - - /** - * Sets up the catalog to use a custom parser for entity data. - * - * This is the function that gets called immediately after some raw entity - * specification data has been read from a remote source, and needs to be - * parsed and emitted as structured data. - * - * @param parser - The custom parser - */ - setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder { - this.parser = parser; - return this; - } - - /** - * Wires up and returns all of the component parts of the catalog - */ - async build(): Promise<{ - entitiesCatalog: EntitiesCatalog; - locationsCatalog: LocationsCatalog; - higherOrderOperation: HigherOrderOperation; - locationAnalyzer: LocationAnalyzer; - }> { - const { config, database, logger } = this.env; - const integrations = ScmIntegrations.fromConfig(config); - - const policy = this.buildEntityPolicy(); - const processors = this.buildProcessors(); - const rulesEnforcer = DefaultCatalogRulesEnforcer.fromConfig(config); - const parser = this.parser || defaultEntityDataParser; - - const locationReader = new LocationReaders({ - ...this.env, - parser, - processors, - rulesEnforcer, - policy, - }); - - const db = await DatabaseManager.createDatabase( - await database.getClient(), - { logger }, - ); - - const entitiesCatalog = new DatabaseEntitiesCatalog(db, this.env.logger); - const locationsCatalog = new DatabaseLocationsCatalog(db); - const higherOrderOperation = new HigherOrderOperations( - entitiesCatalog, - locationsCatalog, - locationReader, - logger, - ); - const locationAnalyzer = new RepoLocationAnalyzer(logger, integrations); - - return { - entitiesCatalog, - locationsCatalog, - higherOrderOperation, - locationAnalyzer, - }; - } - - private buildEntityPolicy(): EntityPolicy { - const entityPolicies: EntityPolicy[] = this.entityPoliciesReplace - ? [new SchemaValidEntityPolicy(), ...this.entityPolicies] - : [ - new SchemaValidEntityPolicy(), - new DefaultNamespaceEntityPolicy(), - new NoForeignRootFieldsEntityPolicy(), - new FieldFormatEntityPolicy( - makeValidator(this.fieldFormatValidators), - ), - ...this.entityPolicies, - ]; - - return EntityPolicies.allOf(entityPolicies); - } - - private buildProcessors(): CatalogProcessor[] { - const { config, logger, reader } = this.env; - const integrations = ScmIntegrations.fromConfig(config); - const githubCredentialsProvider = - DefaultGithubCredentialsProvider.fromIntegrations(integrations); - - this.checkDeprecatedReaderProcessors(); - - const placeholderResolvers: Record = { - json: jsonPlaceholderResolver, - yaml: yamlPlaceholderResolver, - text: textPlaceholderResolver, - ...this.placeholderResolvers, - }; - - // These are always there no matter what - const processors: CatalogProcessor[] = [ - StaticLocationProcessor.fromConfig(config), - new PlaceholderProcessor({ - resolvers: placeholderResolvers, - reader, - integrations, - }), - new BuiltinKindsEntityProcessor(), - ]; - - // These are only added unless the user replaced them all - if (!this.processorsReplace) { - processors.push( - new FileReaderProcessor(), - BitbucketDiscoveryProcessor.fromConfig(config, { logger }), - GithubDiscoveryProcessor.fromConfig(config, { - logger, - githubCredentialsProvider, - }), - AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }), - GithubOrgReaderProcessor.fromConfig(config, { - logger, - githubCredentialsProvider, - }), - GitLabDiscoveryProcessor.fromConfig(config, { logger }), - new UrlReaderProcessor({ reader, logger }), - CodeOwnersProcessor.fromConfig(config, { logger, reader }), - new LocationEntityProcessor({ integrations }), - new AnnotateLocationEntityProcessor({ integrations }), - ); - } - - // Add the ones (if any) that the user added - processors.push(...this.processors); - - return processors; - } - - // TODO(Rugvip): These old processors are removed, for a while we'll be throwing - // errors here to make sure people know where to move the config - private checkDeprecatedReaderProcessors() { - const pc = this.env.config.getOptionalConfig('catalog.processors'); - if (pc?.has('github')) { - throw new Error( - `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`, - ); - } - if (pc?.has('gitlabApi')) { - throw new Error( - `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`, - ); - } - if (pc?.has('bitbucketApi')) { - throw new Error( - `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`, - ); - } - if (pc?.has('azureApi')) { - throw new Error( - `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`, - ); - } - } -} diff --git a/plugins/catalog-backend/src/legacy/service/index.ts b/plugins/catalog-backend/src/legacy/service/index.ts deleted file mode 100644 index 18d3355004..0000000000 --- a/plugins/catalog-backend/src/legacy/service/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { CatalogBuilder } from './CatalogBuilder'; -export { createRouter } from './router'; -export type { RouterOptions } from './router'; diff --git a/plugins/catalog-backend/src/legacy/service/router.test.ts b/plugins/catalog-backend/src/legacy/service/router.test.ts deleted file mode 100644 index 0ff5a44ed0..0000000000 --- a/plugins/catalog-backend/src/legacy/service/router.test.ts +++ /dev/null @@ -1,529 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getVoidLogger } from '@backstage/backend-common'; -import { ConfigReader } from '@backstage/config'; -import { NotFoundError } from '@backstage/errors'; -import type { Entity, LocationSpec } from '@backstage/catalog-model'; -import express from 'express'; -import request from 'supertest'; -import { EntitiesCatalog } from '../../catalog'; -import { LocationResponse, LocationsCatalog } from '../catalog/types'; -import { HigherOrderOperation } from '../ingestion/types'; -import { createRouter } from './router'; -import { basicEntityFilter } from '../../service/request'; -import { RefreshService } from '../../service'; - -describe('createRouter readonly disabled', () => { - let entitiesCatalog: jest.Mocked>; - let locationsCatalog: jest.Mocked; - let higherOrderOperation: jest.Mocked; - let app: express.Express; - let refreshService: RefreshService; - - beforeAll(async () => { - entitiesCatalog = { - entities: jest.fn(), - removeEntityByUid: jest.fn(), - batchAddOrUpdateEntities: jest.fn(), - entityAncestry: jest.fn(), - }; - locationsCatalog = { - addLocation: jest.fn(), - removeLocation: jest.fn(), - locations: jest.fn(), - location: jest.fn(), - locationHistory: jest.fn(), - logUpdateSuccess: jest.fn(), - logUpdateFailure: jest.fn(), - }; - higherOrderOperation = { - addLocation: jest.fn(), - refreshAllLocations: jest.fn(), - }; - refreshService = { refresh: jest.fn() }; - const router = await createRouter({ - entitiesCatalog, - locationsCatalog, - higherOrderOperation, - logger: getVoidLogger(), - refreshService, - config: new ConfigReader(undefined), - }); - app = express().use(router); - }); - - beforeEach(() => { - jest.resetAllMocks(); - }); - - describe('POST /refresh', () => { - it('refreshes an entity using the refresh service', async () => { - const response = await request(app) - .post('/refresh') - .set('Content-Type', 'application/json') - .send({ entityRef: 'Component/default:foo' }); - expect(response.status).toBe(200); - expect(refreshService.refresh).toHaveBeenCalledWith({ - entityRef: 'Component/default:foo', - }); - }); - }); - describe('GET /entities', () => { - it('happy path: lists entities', async () => { - const entities: Entity[] = [ - { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, - ]; - - entitiesCatalog.entities.mockResolvedValueOnce({ - entities: [entities[0]], - pageInfo: { hasNextPage: false }, - }); - - const response = await request(app).get('/entities'); - - expect(response.status).toEqual(200); - expect(response.body).toEqual(entities); - }); - - it('parses single and multiple request parameters and passes them down', async () => { - entitiesCatalog.entities.mockResolvedValueOnce({ - entities: [], - pageInfo: { hasNextPage: false }, - }); - const response = await request(app).get( - '/entities?filter=a=1,a=2,b=3&filter=c=4', - ); - - expect(response.status).toEqual(200); - expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith({ - filter: { - anyOf: [ - { - allOf: [ - { key: 'a', values: ['1', '2'] }, - { key: 'b', values: ['3'] }, - ], - }, - { allOf: [{ key: 'c', values: ['4'] }] }, - ], - }, - }); - }); - }); - - describe('GET /entities/by-uid/:uid', () => { - it('can fetch entity by uid', async () => { - const entity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { - name: 'c', - }, - }; - entitiesCatalog.entities.mockResolvedValue({ - entities: [entity], - pageInfo: { hasNextPage: false }, - }); - - const response = await request(app).get('/entities/by-uid/zzz'); - - expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith({ - filter: basicEntityFilter({ 'metadata.uid': 'zzz' }), - }); - expect(response.status).toEqual(200); - expect(response.body).toEqual(expect.objectContaining(entity)); - }); - - it('responds with a 404 for missing entities', async () => { - entitiesCatalog.entities.mockResolvedValue({ - entities: [], - pageInfo: { hasNextPage: false }, - }); - - const response = await request(app).get('/entities/by-uid/zzz'); - - expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith({ - filter: basicEntityFilter({ 'metadata.uid': 'zzz' }), - }); - expect(response.status).toEqual(404); - expect(response.text).toMatch(/uid/); - }); - }); - - describe('GET /entities/by-name/:kind/:namespace/:name', () => { - it('can fetch entity by name', async () => { - const entity: Entity = { - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'n', - namespace: 'ns', - }, - }; - entitiesCatalog.entities.mockResolvedValue({ - entities: [entity], - pageInfo: { hasNextPage: false }, - }); - - const response = await request(app).get('/entities/by-name/k/ns/n'); - - expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith({ - filter: basicEntityFilter({ - kind: 'k', - 'metadata.namespace': 'ns', - 'metadata.name': 'n', - }), - }); - expect(response.status).toEqual(200); - expect(response.body).toEqual(expect.objectContaining(entity)); - }); - - it('responds with a 404 for missing entities', async () => { - entitiesCatalog.entities.mockResolvedValue({ - entities: [], - pageInfo: { hasNextPage: false }, - }); - - const response = await request(app).get('/entities/by-name/b/d/c'); - - expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith({ - filter: basicEntityFilter({ - kind: 'b', - 'metadata.namespace': 'd', - 'metadata.name': 'c', - }), - }); - expect(response.status).toEqual(404); - expect(response.text).toMatch(/name/); - }); - }); - - describe('POST /entities', () => { - it('requires a body', async () => { - const response = await request(app) - .post('/entities') - .set('Content-Type', 'application/json') - .send(); - - expect(entitiesCatalog.batchAddOrUpdateEntities).not.toHaveBeenCalled(); - expect(response.status).toEqual(400); - expect(response.text).toMatch(/body/); - }); - - it('passes the body down', async () => { - const entity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { - name: 'c', - namespace: 'd', - }, - }; - - entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([ - { entityId: 'u' }, - ]); - entitiesCatalog.entities.mockResolvedValue({ - entities: [entity], - pageInfo: { hasNextPage: false }, - }); - - const response = await request(app) - .post('/entities') - .send(entity) - .set('Content-Type', 'application/json'); - - expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledWith([ - { entity, relations: [] }, - ]); - expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith({ - filter: basicEntityFilter({ 'metadata.uid': 'u' }), - }); - expect(response.status).toEqual(200); - expect(response.body).toEqual(entity); - }); - }); - - describe('DELETE /entities/by-uid/:uid', () => { - it('can remove', async () => { - entitiesCatalog.removeEntityByUid.mockResolvedValue(undefined); - - const response = await request(app).delete('/entities/by-uid/apa'); - - expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledWith('apa'); - expect(response.status).toEqual(204); - }); - - it('responds with a 404 for missing entities', async () => { - entitiesCatalog.removeEntityByUid.mockRejectedValue( - new NotFoundError('nope'), - ); - - const response = await request(app).delete('/entities/by-uid/apa'); - - expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledWith('apa'); - expect(response.status).toEqual(404); - }); - }); - - describe('GET /locations', () => { - it('happy path: lists locations', async () => { - const locations: LocationResponse[] = [ - { - currentStatus: { timestamp: '', status: '', message: '' }, - data: { id: 'a', type: 'b', target: 'c' }, - }, - ]; - locationsCatalog.locations.mockResolvedValueOnce(locations); - - const response = await request(app).get('/locations'); - - expect(response.status).toEqual(200); - expect(response.body).toEqual(locations); - }); - }); - - describe('POST /locations', () => { - it('rejects malformed locations', async () => { - const spec = { - typez: 'b', - target: 'c', - } as unknown as LocationSpec; - - const response = await request(app).post('/locations').send(spec); - - expect(higherOrderOperation.addLocation).not.toHaveBeenCalled(); - expect(response.status).toEqual(400); - }); - - it('passes the body down', async () => { - const spec: LocationSpec = { - type: 'b', - target: 'c', - }; - - higherOrderOperation.addLocation.mockResolvedValue({ - location: { id: 'a', ...spec }, - entities: [], - }); - - const response = await request(app).post('/locations').send(spec); - - expect(higherOrderOperation.addLocation).toHaveBeenCalledTimes(1); - expect(higherOrderOperation.addLocation).toHaveBeenCalledWith(spec, { - dryRun: false, - }); - expect(response.status).toEqual(201); - expect(response.body).toEqual( - expect.objectContaining({ - location: { id: 'a', ...spec }, - }), - ); - }); - - it('supports dry run', async () => { - const spec: LocationSpec = { - type: 'b', - target: 'c', - }; - - higherOrderOperation.addLocation.mockResolvedValue({ - location: { id: 'a', ...spec }, - entities: [], - }); - - const response = await request(app) - .post('/locations?dryRun=true') - .send(spec); - - expect(higherOrderOperation.addLocation).toHaveBeenCalledTimes(1); - expect(higherOrderOperation.addLocation).toHaveBeenCalledWith(spec, { - dryRun: true, - }); - expect(response.status).toEqual(201); - expect(response.body).toEqual( - expect.objectContaining({ - location: { id: 'a', ...spec }, - }), - ); - }); - }); -}); - -describe('createRouter readonly enabled', () => { - let entitiesCatalog: jest.Mocked; - let locationsCatalog: jest.Mocked; - let higherOrderOperation: jest.Mocked; - let app: express.Express; - - beforeAll(async () => { - entitiesCatalog = { - entities: jest.fn(), - removeEntityByUid: jest.fn(), - batchAddOrUpdateEntities: jest.fn(), - entityAncestry: jest.fn(), - }; - locationsCatalog = { - addLocation: jest.fn(), - removeLocation: jest.fn(), - locations: jest.fn(), - location: jest.fn(), - locationHistory: jest.fn(), - logUpdateSuccess: jest.fn(), - logUpdateFailure: jest.fn(), - }; - higherOrderOperation = { - addLocation: jest.fn(), - refreshAllLocations: jest.fn(), - }; - const router = await createRouter({ - entitiesCatalog, - locationsCatalog, - higherOrderOperation, - logger: getVoidLogger(), - config: new ConfigReader({ - catalog: { - readonly: true, - }, - }), - }); - app = express().use(router); - }); - - beforeEach(() => { - jest.resetAllMocks(); - }); - - describe('GET /entities', () => { - it('happy path: lists entities', async () => { - const entities: Entity[] = [ - { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, - ]; - - entitiesCatalog.entities.mockResolvedValueOnce({ - entities: [entities[0]], - pageInfo: { hasNextPage: false }, - }); - - const response = await request(app).get('/entities'); - - expect(response.status).toEqual(200); - expect(response.body).toEqual(entities); - }); - }); - - describe('POST /entities', () => { - it('is not allowed', async () => { - const entity: Entity = { - apiVersion: 'a', - kind: 'b', - metadata: { - name: 'c', - namespace: 'd', - }, - }; - - const response = await request(app) - .post('/entities') - .set('Content-Type', 'application/json') - .send(entity); - - expect(entitiesCatalog.batchAddOrUpdateEntities).not.toHaveBeenCalled(); - expect(response.status).toEqual(403); - expect(response.text).toMatch(/not allowed in readonly/); - }); - }); - - describe('DELETE /entities/by-uid/:uid', () => { - // this delete is allowed as there is no other way to remove entities - it('is allowed', async () => { - const response = await request(app).delete('/entities/by-uid/apa'); - - expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledWith('apa'); - expect(response.status).toEqual(204); - }); - }); - - describe('GET /locations', () => { - it('happy path: lists locations', async () => { - const locations: LocationResponse[] = [ - { - currentStatus: { timestamp: '', status: '', message: '' }, - data: { id: 'a', type: 'b', target: 'c' }, - }, - ]; - locationsCatalog.locations.mockResolvedValueOnce(locations); - - const response = await request(app).get('/locations'); - - expect(response.status).toEqual(200); - expect(response.body).toEqual(locations); - }); - }); - - describe('POST /locations', () => { - it('is not allowed', async () => { - const spec: LocationSpec = { - type: 'b', - target: 'c', - }; - - const response = await request(app).post('/locations').send(spec); - - expect(higherOrderOperation.addLocation).not.toHaveBeenCalled(); - expect(response.status).toEqual(403); - expect(response.text).toMatch(/not allowed in readonly/); - }); - - it('supports dry run', async () => { - const spec: LocationSpec = { - type: 'b', - target: 'c', - }; - - higherOrderOperation.addLocation.mockResolvedValue({ - location: { id: 'a', ...spec }, - entities: [], - }); - - const response = await request(app) - .post('/locations?dryRun=true') - .send(spec); - - expect(higherOrderOperation.addLocation).toHaveBeenCalledTimes(1); - expect(higherOrderOperation.addLocation).toHaveBeenCalledWith(spec, { - dryRun: true, - }); - expect(response.status).toEqual(201); - expect(response.body).toEqual( - expect.objectContaining({ - location: { id: 'a', ...spec }, - }), - ); - }); - }); -}); diff --git a/plugins/catalog-backend/src/legacy/service/router.ts b/plugins/catalog-backend/src/legacy/service/router.ts deleted file mode 100644 index e7d097a9c7..0000000000 --- a/plugins/catalog-backend/src/legacy/service/router.ts +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { errorHandler } from '@backstage/backend-common'; -import type { Entity } from '@backstage/catalog-model'; -import { - analyzeLocationSchema, - locationSpecSchema, -} from '@backstage/catalog-model'; -import { Config } from '@backstage/config'; -import { NotFoundError } from '@backstage/errors'; -import express from 'express'; -import Router from 'express-promise-router'; -import { Logger } from 'winston'; -import yn from 'yn'; -import { EntitiesCatalog } from '../../catalog'; -import { LocationsCatalog } from '../catalog'; -import { LocationAnalyzer } from '../../ingestion/types'; -import { HigherOrderOperation } from '../ingestion/types'; -import { - RefreshService, - LocationService, - RefreshOptions, -} from '../../service/types'; -import { - basicEntityFilter, - parseEntityFilterParams, - parseEntityPaginationParams, - parseEntityTransformParams, -} from '../../service/request'; -import { - disallowReadonlyMode, - requireRequestBody, - validateRequestBody, -} from '../../service/util'; - -/** @deprecated This was part of the legacy catalog engine */ -export interface RouterOptions { - entitiesCatalog?: EntitiesCatalog; - locationsCatalog?: LocationsCatalog; - higherOrderOperation?: HigherOrderOperation; - locationAnalyzer?: LocationAnalyzer; - locationService?: LocationService; - refreshService?: RefreshService; - logger: Logger; - config: Config; -} - -/** @deprecated This was part of the legacy catalog engine */ -export async function createRouter( - options: RouterOptions, -): Promise { - const { - entitiesCatalog, - locationsCatalog, - higherOrderOperation, - locationAnalyzer, - locationService, - refreshService, - config, - logger, - } = options; - - const router = Router(); - router.use(express.json()); - - const readonlyEnabled = - config.getOptionalBoolean('catalog.readonly') || false; - if (readonlyEnabled) { - logger.info('Catalog is running in readonly mode'); - } - - if (refreshService) { - router.post('/refresh', async (req, res) => { - const refreshOptions: RefreshOptions = req.body; - await refreshService.refresh(refreshOptions); - res.status(200).send(); - }); - } - - if (entitiesCatalog) { - router - .get('/entities', async (req, res) => { - const { entities, pageInfo } = await entitiesCatalog.entities({ - filter: parseEntityFilterParams(req.query), - fields: parseEntityTransformParams(req.query), - pagination: parseEntityPaginationParams(req.query), - }); - - // Add a Link header to the next page - if (pageInfo.hasNextPage) { - const url = new URL(`http://ignored${req.url}`); - url.searchParams.delete('offset'); - url.searchParams.set('after', pageInfo.endCursor); - res.setHeader('link', `<${url.pathname}${url.search}>; rel="next"`); - } - - // TODO(freben): encode the pageInfo in the response - res.json(entities); - }) - .post('/entities', async (req, res) => { - /* - * NOTE: THIS METHOD IS DEPRECATED AND NOT RECOMMENDED TO USE - * - * Posting entities to this method has unclear semantics and will not - * properly subject them to limitations, processing, or resolution of - * relations. - * - * It stays around in the service for the time being, but may be - * removed or change semantics at any time without prior notice. - */ - disallowReadonlyMode(readonlyEnabled); - - const body = await requireRequestBody(req); - const [result] = await entitiesCatalog.batchAddOrUpdateEntities!([ - { entity: body as Entity, relations: [] }, - ]); - const response = await entitiesCatalog.entities({ - filter: basicEntityFilter({ 'metadata.uid': result.entityId }), - }); - res.status(200).json(response.entities[0]); - }) - .get('/entities/by-uid/:uid', async (req, res) => { - const { uid } = req.params; - const { entities } = await entitiesCatalog.entities({ - filter: basicEntityFilter({ 'metadata.uid': uid }), - }); - if (!entities.length) { - throw new NotFoundError(`No entity with uid ${uid}`); - } - res.status(200).json(entities[0]); - }) - .delete('/entities/by-uid/:uid', async (req, res) => { - const { uid } = req.params; - await entitiesCatalog.removeEntityByUid(uid); - res.status(204).end(); - }) - .get('/entities/by-name/:kind/:namespace/:name', async (req, res) => { - const { kind, namespace, name } = req.params; - const { entities } = await entitiesCatalog.entities({ - filter: basicEntityFilter({ - kind: kind, - 'metadata.namespace': namespace, - 'metadata.name': name, - }), - }); - if (!entities.length) { - throw new NotFoundError( - `No entity named '${name}' found, with kind '${kind}' in namespace '${namespace}'`, - ); - } - res.status(200).json(entities[0]); - }); - } - - if (locationService) { - router - .post('/locations', async (req, res) => { - const input = await validateRequestBody(req, locationSpecSchema); - const dryRun = yn(req.query.dryRun, { default: false }); - - // when in dryRun addLocation is effectively a read operation so we don't - // need to disallow readonly - if (!dryRun) { - disallowReadonlyMode(readonlyEnabled); - } - - const output = await locationService.createLocation(input, dryRun); - res.status(201).json(output); - }) - .get('/locations', async (_req, res) => { - const locations = await locationService.listLocations(); - res.status(200).json(locations.map(l => ({ data: l }))); - }) - - .get('/locations/:id', async (req, res) => { - const { id } = req.params; - const output = await locationService.getLocation(id); - res.status(200).json(output); - }) - .delete('/locations/:id', async (req, res) => { - disallowReadonlyMode(readonlyEnabled); - - const { id } = req.params; - await locationService.deleteLocation(id); - res.status(204).end(); - }); - } - - if (higherOrderOperation) { - router.post('/locations', async (req, res) => { - const input = await validateRequestBody(req, locationSpecSchema); - const dryRun = yn(req.query.dryRun, { default: false }); - - // when in dryRun addLocation is effectively a read operation so we don't - // need to disallow readonly - if (!dryRun) { - disallowReadonlyMode(readonlyEnabled); - } - - const output = await higherOrderOperation.addLocation(input, { dryRun }); - res.status(201).json(output); - }); - } - - if (locationsCatalog) { - router - .get('/locations', async (_req, res) => { - const output = await locationsCatalog.locations(); - res.status(200).json(output); - }) - .get('/locations/:id/history', async (req, res) => { - const { id } = req.params; - const output = await locationsCatalog.locationHistory(id); - res.status(200).json(output); - }) - .get('/locations/:id', async (req, res) => { - const { id } = req.params; - const output = await locationsCatalog.location(id); - res.status(200).json(output); - }) - .delete('/locations/:id', async (req, res) => { - disallowReadonlyMode(readonlyEnabled); - - const { id } = req.params; - await locationsCatalog.removeLocation(id); - res.status(204).end(); - }); - } - - if (locationAnalyzer) { - router.post('/analyze-location', async (req, res) => { - const input = await validateRequestBody(req, analyzeLocationSchema); - const output = await locationAnalyzer.analyzeLocation(input); - res.status(200).json(output); - }); - } - - router.use(errorHandler()); - return router; -}