remove the legacy folder
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
@@ -1,426 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { Entity, LOCATION_ANNOTATION } from '@backstage/catalog-model';
|
||||
import { Database, DatabaseManager, Transaction } from '../database';
|
||||
import { basicEntityFilter } from '../../service/request';
|
||||
import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
|
||||
import { EntityUpsertRequest } from '../../catalog/types';
|
||||
|
||||
describe('DatabaseEntitiesCatalog', () => {
|
||||
let db: jest.Mocked<Database>;
|
||||
let transaction: jest.Mocked<Transaction>;
|
||||
|
||||
beforeAll(() => {
|
||||
db = {
|
||||
transaction: jest.fn(),
|
||||
addEntities: jest.fn(),
|
||||
updateEntity: jest.fn(),
|
||||
entities: jest.fn(),
|
||||
entityByName: jest.fn(),
|
||||
entityByUid: jest.fn(),
|
||||
removeEntityByUid: jest.fn(),
|
||||
setRelations: jest.fn(),
|
||||
addLocation: jest.fn(),
|
||||
removeLocation: jest.fn(),
|
||||
location: jest.fn(),
|
||||
locations: jest.fn(),
|
||||
locationHistory: jest.fn(),
|
||||
addLocationUpdateLogEvent: jest.fn(),
|
||||
};
|
||||
transaction = {
|
||||
rollback: jest.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
db.transaction.mockImplementation(async f => f(transaction));
|
||||
});
|
||||
|
||||
describe('batchAddOrUpdateEntities', () => {
|
||||
it('adds when no given uid and no matching by name', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
};
|
||||
|
||||
db.entities.mockResolvedValue({
|
||||
entities: [],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
db.addEntities.mockResolvedValue([
|
||||
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
|
||||
]);
|
||||
|
||||
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
|
||||
const result = await catalog.batchAddOrUpdateEntities([
|
||||
{ entity, relations: [] },
|
||||
]);
|
||||
|
||||
expect(db.entities).toHaveBeenCalledTimes(1);
|
||||
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
|
||||
filter: basicEntityFilter({
|
||||
kind: 'b',
|
||||
'metadata.namespace': 'd',
|
||||
'metadata.name': 'c',
|
||||
}),
|
||||
});
|
||||
expect(db.addEntities).toHaveBeenCalledTimes(1);
|
||||
expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [
|
||||
{ entity: expect.anything(), relations: [] },
|
||||
]);
|
||||
expect(result).toEqual([{ entityId: 'u' }]);
|
||||
});
|
||||
|
||||
it('dry run of add operation', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
};
|
||||
db.entities.mockResolvedValue({
|
||||
entities: [],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
db.addEntities.mockResolvedValue([
|
||||
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
|
||||
]);
|
||||
|
||||
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
|
||||
const result = await catalog.batchAddOrUpdateEntities(
|
||||
[{ entity, relations: [] }],
|
||||
{ dryRun: true },
|
||||
);
|
||||
|
||||
expect(db.entities).toHaveBeenCalledTimes(1);
|
||||
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
|
||||
filter: basicEntityFilter({
|
||||
kind: 'b',
|
||||
'metadata.namespace': 'd',
|
||||
'metadata.name': 'c',
|
||||
}),
|
||||
});
|
||||
expect(db.addEntities).toHaveBeenCalledTimes(1);
|
||||
expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [
|
||||
{ entity: expect.anything(), relations: [] },
|
||||
]);
|
||||
expect(transaction.rollback).toBeCalledTimes(1);
|
||||
expect(result).toEqual([{ entityId: 'u' }]);
|
||||
});
|
||||
|
||||
it('output modified entities', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
annotations: {
|
||||
[LOCATION_ANNOTATION]: 'mock',
|
||||
},
|
||||
},
|
||||
};
|
||||
const dbEntity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
description: 'changes',
|
||||
uid: 'u',
|
||||
annotations: {
|
||||
[LOCATION_ANNOTATION]: 'mock',
|
||||
},
|
||||
},
|
||||
};
|
||||
db.entities.mockResolvedValue({
|
||||
entities: [{ entity: dbEntity }],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
db.addEntities.mockResolvedValue([
|
||||
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
|
||||
]);
|
||||
|
||||
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
|
||||
const result = await catalog.batchAddOrUpdateEntities(
|
||||
[{ entity, relations: [] }],
|
||||
{ outputEntities: true },
|
||||
);
|
||||
|
||||
expect(db.entities).toHaveBeenCalledTimes(2);
|
||||
expect(db.addEntities).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual([
|
||||
{
|
||||
entityId: 'u',
|
||||
entity: dbEntity,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('updates when given uid', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
uid: 'u',
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
spec: {
|
||||
x: 'b',
|
||||
},
|
||||
};
|
||||
const existing = {
|
||||
entity: {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
uid: 'u',
|
||||
etag: 'e',
|
||||
generation: 1,
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
spec: {
|
||||
x: 'a',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
db.entities.mockResolvedValue({
|
||||
entities: [existing],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
db.entityByUid.mockResolvedValue(existing);
|
||||
db.updateEntity.mockResolvedValue({ entity });
|
||||
|
||||
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
|
||||
const result = await catalog.batchAddOrUpdateEntities([
|
||||
{ entity, relations: [] },
|
||||
]);
|
||||
|
||||
expect(db.entities).toHaveBeenCalledTimes(1);
|
||||
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
|
||||
filter: basicEntityFilter({
|
||||
kind: 'b',
|
||||
'metadata.namespace': 'd',
|
||||
'metadata.name': 'c',
|
||||
}),
|
||||
});
|
||||
expect(db.entityByName).not.toHaveBeenCalled();
|
||||
expect(db.entityByUid).toHaveBeenCalledTimes(1);
|
||||
expect(db.entityByUid).toHaveBeenCalledWith(transaction, 'u');
|
||||
expect(db.updateEntity).toHaveBeenCalledTimes(1);
|
||||
expect(db.updateEntity).toHaveBeenCalledWith(
|
||||
transaction,
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
uid: 'u',
|
||||
etag: expect.any(String),
|
||||
generation: 2,
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
spec: {
|
||||
x: 'b',
|
||||
},
|
||||
},
|
||||
relations: [],
|
||||
},
|
||||
'e',
|
||||
1,
|
||||
);
|
||||
expect(result).toEqual([{ entityId: 'u' }]);
|
||||
});
|
||||
|
||||
it('update when no given uid and matching by name', async () => {
|
||||
const added: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
spec: {
|
||||
x: 'b',
|
||||
},
|
||||
};
|
||||
const existing = {
|
||||
entity: {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
uid: 'u',
|
||||
etag: 'e',
|
||||
generation: 1,
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
spec: {
|
||||
x: 'a',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
db.entities.mockResolvedValue({
|
||||
entities: [existing],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
db.entityByName.mockResolvedValue(existing);
|
||||
db.updateEntity.mockResolvedValue(existing);
|
||||
|
||||
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
|
||||
const result = await catalog.batchAddOrUpdateEntities([
|
||||
{ entity: added, relations: [] },
|
||||
]);
|
||||
|
||||
expect(db.entities).toHaveBeenCalledTimes(1);
|
||||
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
|
||||
filter: basicEntityFilter({
|
||||
kind: 'b',
|
||||
'metadata.namespace': 'd',
|
||||
'metadata.name': 'c',
|
||||
}),
|
||||
});
|
||||
expect(db.entityByName).toHaveBeenCalledTimes(1);
|
||||
expect(db.entityByName).toHaveBeenCalledWith(transaction, {
|
||||
kind: 'b',
|
||||
namespace: 'd',
|
||||
name: 'c',
|
||||
});
|
||||
expect(db.updateEntity).toHaveBeenCalledTimes(1);
|
||||
expect(db.updateEntity).toHaveBeenCalledWith(
|
||||
transaction,
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
uid: 'u',
|
||||
etag: expect.any(String),
|
||||
generation: 2,
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
spec: {
|
||||
x: 'b',
|
||||
},
|
||||
},
|
||||
relations: [],
|
||||
},
|
||||
'e',
|
||||
1,
|
||||
);
|
||||
expect(result).toEqual([{ entityId: 'u' }]);
|
||||
});
|
||||
|
||||
it('should not update if entity is unchanged', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
uid: 'u',
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
spec: {
|
||||
x: 'a',
|
||||
},
|
||||
};
|
||||
|
||||
db.entities.mockResolvedValue({
|
||||
entities: [{ entity }],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
db.entityByUid.mockResolvedValue({ entity });
|
||||
db.updateEntity.mockResolvedValue({ entity });
|
||||
|
||||
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
|
||||
const result = await catalog.batchAddOrUpdateEntities([
|
||||
{ entity, relations: [] },
|
||||
]);
|
||||
|
||||
expect(db.entities).toHaveBeenCalledTimes(1);
|
||||
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
|
||||
filter: basicEntityFilter({
|
||||
kind: 'b',
|
||||
'metadata.namespace': 'd',
|
||||
'metadata.name': 'c',
|
||||
}),
|
||||
});
|
||||
expect(db.entityByName).not.toHaveBeenCalled();
|
||||
expect(db.entityByUid).not.toHaveBeenCalled();
|
||||
expect(db.updateEntity).not.toHaveBeenCalled();
|
||||
expect(db.setRelations).toHaveBeenCalledTimes(1);
|
||||
expect(db.setRelations).toHaveBeenCalledWith(expect.anything(), 'u', []);
|
||||
expect(result).toEqual([{ entityId: 'u' }]);
|
||||
});
|
||||
|
||||
it('both adds and updates', async () => {
|
||||
const catalog = new DatabaseEntitiesCatalog(
|
||||
await DatabaseManager.createTestDatabase(),
|
||||
getVoidLogger(),
|
||||
);
|
||||
const entities: EntityUpsertRequest[] = [];
|
||||
for (let i = 0; i < 300; ++i) {
|
||||
entities.push({
|
||||
entity: {
|
||||
apiVersion: 'a',
|
||||
kind: 'k',
|
||||
metadata: { name: `n${i}` },
|
||||
},
|
||||
relations: [],
|
||||
});
|
||||
}
|
||||
|
||||
await catalog.batchAddOrUpdateEntities(entities);
|
||||
const afterFirst = await catalog.entities();
|
||||
expect(afterFirst.entities.length).toBe(300);
|
||||
|
||||
entities[40].entity.metadata.op = 'changed';
|
||||
entities.push({
|
||||
entity: {
|
||||
apiVersion: 'a',
|
||||
kind: 'k',
|
||||
metadata: { name: `n300`, op: 'added' },
|
||||
},
|
||||
relations: [],
|
||||
});
|
||||
|
||||
await catalog.batchAddOrUpdateEntities(entities);
|
||||
const afterSecond = await catalog.entities();
|
||||
expect(afterSecond.entities.length).toBe(301);
|
||||
expect(
|
||||
afterSecond.entities.find(e => e.metadata.op === 'changed'),
|
||||
).toBeDefined();
|
||||
expect(
|
||||
afterSecond.entities.find(e => e.metadata.op === 'added'),
|
||||
).toBeDefined();
|
||||
}, 10000);
|
||||
});
|
||||
});
|
||||
@@ -1,377 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
Entity,
|
||||
entityHasChanges,
|
||||
generateUpdatedEntity,
|
||||
getEntityName,
|
||||
LOCATION_ANNOTATION,
|
||||
serializeEntityRef,
|
||||
} from '@backstage/catalog-model';
|
||||
import { ConflictError } from '@backstage/errors';
|
||||
import { chunk, groupBy } from 'lodash';
|
||||
import limiterFactory from 'p-limit';
|
||||
import { Logger } from 'winston';
|
||||
import type { Database, DbEntityResponse, Transaction } from '../database';
|
||||
import { DbEntitiesRequest } from '../database/types';
|
||||
import { basicEntityFilter } from '../../service/request';
|
||||
import { durationText } from '../../util/timing';
|
||||
import type {
|
||||
EntitiesCatalog,
|
||||
EntitiesRequest,
|
||||
EntitiesResponse,
|
||||
EntityUpsertRequest,
|
||||
EntityUpsertResponse,
|
||||
} from '../../catalog/types';
|
||||
|
||||
type BatchContext = {
|
||||
kind: string;
|
||||
namespace: string;
|
||||
locationId?: string;
|
||||
};
|
||||
|
||||
// Some locations return tens or hundreds of thousands of entities. To make
|
||||
// those payloads more manageable, we break work apart in batches of this
|
||||
// many entities and write them to storage per batch.
|
||||
const BATCH_SIZE = 100;
|
||||
|
||||
// When writing large batches, there's an increasing chance of contention in
|
||||
// the form of conflicts where we compete with other writes. Each batch gets
|
||||
// this many attempts at being written before giving up.
|
||||
const BATCH_ATTEMPTS = 3;
|
||||
|
||||
// The number of batches that may be ongoing at the same time.
|
||||
const BATCH_CONCURRENCY = 3;
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export class DatabaseEntitiesCatalog implements EntitiesCatalog {
|
||||
constructor(
|
||||
private readonly database: Database,
|
||||
private readonly logger: Logger,
|
||||
) {}
|
||||
|
||||
async entities(request?: EntitiesRequest): Promise<EntitiesResponse> {
|
||||
const dbRequest: DbEntitiesRequest = {
|
||||
filter: request?.filter,
|
||||
pagination: request?.pagination,
|
||||
};
|
||||
|
||||
const dbResponse = await this.database.transaction(tx =>
|
||||
this.database.entities(tx, dbRequest),
|
||||
);
|
||||
|
||||
const entities = dbResponse.entities.map(e =>
|
||||
request?.fields ? request.fields(e.entity) : e.entity,
|
||||
);
|
||||
|
||||
return {
|
||||
entities,
|
||||
pageInfo: dbResponse.pageInfo,
|
||||
};
|
||||
}
|
||||
|
||||
async removeEntityByUid(uid: string): Promise<void> {
|
||||
await this.database.transaction(async tx => {
|
||||
await this.database.removeEntityByUid(tx, uid);
|
||||
});
|
||||
}
|
||||
|
||||
async batchAddOrUpdateEntities(
|
||||
requests: EntityUpsertRequest[],
|
||||
options?: {
|
||||
locationId?: string;
|
||||
dryRun?: boolean;
|
||||
outputEntities?: boolean;
|
||||
},
|
||||
): Promise<EntityUpsertResponse[]> {
|
||||
// Group the requests by unique kind+namespace combinations. The reason for
|
||||
// this is that the change detection and merging logic requires finding
|
||||
// pre-existing versions of the entities in the database. Those queries are
|
||||
// easier and faster to make if every batch revolves around a single kind-
|
||||
// namespace pair.
|
||||
const requestsByKindAndNamespace = groupBy(requests, ({ entity }) => {
|
||||
const name = getEntityName(entity);
|
||||
return `${name.kind}:${name.namespace}`.toLowerCase();
|
||||
});
|
||||
|
||||
// Go through the requests in reasonable batch sizes. Sometimes, sources
|
||||
// produce tens of thousands of entities, and those are too large batch
|
||||
// sizes to reasonably send to the database.
|
||||
const batches = Object.values(requestsByKindAndNamespace)
|
||||
.map(request => chunk(request, BATCH_SIZE))
|
||||
.flat();
|
||||
|
||||
// Bound the number of concurrent batches. We want a bit of concurrency for
|
||||
// performance reasons, but not so much that we starve the connection pool
|
||||
// or start thrashing.
|
||||
const limiter = limiterFactory(BATCH_CONCURRENCY);
|
||||
const tasks = batches.map(batch =>
|
||||
limiter(async () => {
|
||||
// Retry the batch write a few times to deal with contention
|
||||
for (let attempt = 1; ; ++attempt) {
|
||||
try {
|
||||
return this.batchAddOrUpdateEntitiesSingleBatch(batch, options);
|
||||
} catch (e) {
|
||||
if (e instanceof ConflictError && attempt < BATCH_ATTEMPTS) {
|
||||
this.logger.warn(
|
||||
`Failed to write batch at attempt ${attempt}/${BATCH_ATTEMPTS}, ${e}`,
|
||||
);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
const responses = await Promise.all(tasks);
|
||||
return responses.flat();
|
||||
}
|
||||
|
||||
// Defines the actual logic of running a single batch. All of these share a
|
||||
// common kind and namespace.
|
||||
private async batchAddOrUpdateEntitiesSingleBatch(
|
||||
batch: EntityUpsertRequest[],
|
||||
options?: {
|
||||
locationId?: string;
|
||||
dryRun?: boolean;
|
||||
outputEntities?: boolean;
|
||||
},
|
||||
) {
|
||||
const { kind, namespace } = getEntityName(batch[0].entity);
|
||||
const context = {
|
||||
kind,
|
||||
namespace,
|
||||
locationId: options?.locationId,
|
||||
};
|
||||
|
||||
this.logger.debug(
|
||||
`Considering batch ${serializeEntityRef(
|
||||
batch[0].entity,
|
||||
)}-${serializeEntityRef(batch[batch.length - 1].entity)} (${
|
||||
batch.length
|
||||
} entries)`,
|
||||
);
|
||||
|
||||
return this.database.transaction(async tx => {
|
||||
const { toAdd, toUpdate, toIgnore } = await this.analyzeBatch(
|
||||
batch,
|
||||
context,
|
||||
tx,
|
||||
);
|
||||
|
||||
let responses = new Array<EntityUpsertResponse>();
|
||||
if (toAdd.length) {
|
||||
const items = await this.batchAdd(toAdd, context, tx);
|
||||
responses.push(...items);
|
||||
}
|
||||
if (toUpdate.length) {
|
||||
const items = await this.batchUpdate(toUpdate, context, tx);
|
||||
responses.push(...items);
|
||||
}
|
||||
for (const { entity, relations } of toIgnore) {
|
||||
// TODO(Rugvip): We currently always update relations, but we
|
||||
// likely want to figure out a way to avoid that
|
||||
const entityId = entity.metadata.uid;
|
||||
if (entityId) {
|
||||
await this.database.setRelations(tx, entityId, relations);
|
||||
responses.push({ entityId });
|
||||
}
|
||||
}
|
||||
|
||||
if (options?.outputEntities && responses.length > 0) {
|
||||
const writtenEntities = await this.database.entities(tx, {
|
||||
filter: basicEntityFilter({
|
||||
'metadata.uid': responses.map(e => e.entityId),
|
||||
}),
|
||||
});
|
||||
responses = writtenEntities.entities.map(e => ({
|
||||
entityId: e.entity.metadata.uid!,
|
||||
entity: e.entity,
|
||||
}));
|
||||
}
|
||||
|
||||
// If this is only a dry run, cancel the database transaction even if it
|
||||
// was successful.
|
||||
if (options?.dryRun) {
|
||||
await tx.rollback();
|
||||
this.logger.debug(`Performed successful dry run of adding entities`);
|
||||
}
|
||||
|
||||
return responses;
|
||||
});
|
||||
}
|
||||
|
||||
// Given a batch of entities that were just read from a location, take them
|
||||
// into consideration by comparing against the existing catalog entities and
|
||||
// produce the list of entities to be added, and the list of entities to be
|
||||
// updated
|
||||
private async analyzeBatch(
|
||||
requests: EntityUpsertRequest[],
|
||||
{ kind, namespace }: BatchContext,
|
||||
tx: Transaction,
|
||||
): Promise<{
|
||||
toAdd: EntityUpsertRequest[];
|
||||
toUpdate: EntityUpsertRequest[];
|
||||
toIgnore: EntityUpsertRequest[];
|
||||
}> {
|
||||
const markTimestamp = process.hrtime();
|
||||
|
||||
// Here we make use of the fact that all of the entities share kind and
|
||||
// namespace within a batch
|
||||
const names = requests.map(({ entity }) => entity.metadata.name);
|
||||
const oldEntitiesResponse = await this.database.entities(tx, {
|
||||
filter: basicEntityFilter({
|
||||
kind: kind,
|
||||
'metadata.namespace': namespace,
|
||||
'metadata.name': names,
|
||||
}),
|
||||
});
|
||||
|
||||
const oldEntitiesByName = new Map(
|
||||
oldEntitiesResponse.entities.map(e => [e.entity.metadata.name, e.entity]),
|
||||
);
|
||||
|
||||
const toAdd: EntityUpsertRequest[] = [];
|
||||
const toUpdate: EntityUpsertRequest[] = [];
|
||||
const toIgnore: EntityUpsertRequest[] = [];
|
||||
|
||||
for (const request of requests) {
|
||||
const newEntity = request.entity;
|
||||
const oldEntity = oldEntitiesByName.get(newEntity.metadata.name);
|
||||
const newLocation = newEntity.metadata.annotations?.[LOCATION_ANNOTATION];
|
||||
const oldLocation =
|
||||
oldEntity?.metadata.annotations?.[LOCATION_ANNOTATION];
|
||||
if (!oldEntity) {
|
||||
toAdd.push(request);
|
||||
} else if (oldLocation !== newLocation) {
|
||||
this.logger.warn(
|
||||
`Rejecting write of entity ${serializeEntityRef(
|
||||
newEntity,
|
||||
)} from ${newLocation} because entity existed from ${oldLocation}`,
|
||||
);
|
||||
toIgnore.push(request);
|
||||
} else if (entityHasChanges(oldEntity, newEntity)) {
|
||||
// TODO(freben): This currently uses addOrUpdateEntity under the hood,
|
||||
// but should probably calculate the end result entity right here
|
||||
// instead and call a dedicated batch update database method
|
||||
toUpdate.push(request);
|
||||
} else {
|
||||
// Use the existing entity to ensure that we're able to read it back by uid if needed
|
||||
toIgnore.push({ ...request, entity: oldEntity });
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.debug(
|
||||
`Found ${toAdd.length} entities to add, ${
|
||||
toUpdate.length
|
||||
} entities to update in ${durationText(markTimestamp)}`,
|
||||
);
|
||||
|
||||
return { toAdd, toUpdate, toIgnore };
|
||||
}
|
||||
|
||||
// Efficiently adds the given entities to storage, under the assumption that
|
||||
// they do not conflict with any existing entities
|
||||
private async batchAdd(
|
||||
requests: EntityUpsertRequest[],
|
||||
{ locationId }: BatchContext,
|
||||
tx: Transaction,
|
||||
): Promise<EntityUpsertResponse[]> {
|
||||
const markTimestamp = process.hrtime();
|
||||
|
||||
const res = await this.database.addEntities(
|
||||
tx,
|
||||
requests.map(({ entity, relations }) => ({
|
||||
locationId,
|
||||
entity,
|
||||
relations,
|
||||
})),
|
||||
);
|
||||
|
||||
const responses = res.map(({ entity }) => ({
|
||||
entityId: entity.metadata.uid!,
|
||||
}));
|
||||
|
||||
this.logger.debug(
|
||||
`Added ${requests.length} entities in ${durationText(markTimestamp)}`,
|
||||
);
|
||||
|
||||
return responses;
|
||||
}
|
||||
|
||||
// Efficiently updates the given entities into storage, under the assumption
|
||||
// that there already exist entities with the same names
|
||||
private async batchUpdate(
|
||||
requests: EntityUpsertRequest[],
|
||||
{ locationId }: BatchContext,
|
||||
tx: Transaction,
|
||||
): Promise<EntityUpsertResponse[]> {
|
||||
const markTimestamp = process.hrtime();
|
||||
const responses: EntityUpsertResponse[] = [];
|
||||
|
||||
// TODO(freben): Still not batched
|
||||
for (const request of requests) {
|
||||
const res = await this.addOrUpdateEntity(tx, request, locationId);
|
||||
const entityId = res.metadata.uid!;
|
||||
responses.push({ entityId });
|
||||
}
|
||||
|
||||
this.logger.debug(
|
||||
`Updated ${requests.length} entities in ${durationText(markTimestamp)}`,
|
||||
);
|
||||
|
||||
return responses;
|
||||
}
|
||||
|
||||
// TODO(freben): Incorporate this into batchUpdate which is the only caller
|
||||
private async addOrUpdateEntity(
|
||||
tx: Transaction,
|
||||
{ entity, relations }: EntityUpsertRequest,
|
||||
locationId?: string,
|
||||
): Promise<Entity> {
|
||||
// Find a matching (by uid, or by compound name, depending on the given
|
||||
// entity) existing entity, to know whether to update or add
|
||||
const existing = entity.metadata.uid
|
||||
? await this.database.entityByUid(tx, entity.metadata.uid)
|
||||
: await this.database.entityByName(tx, getEntityName(entity));
|
||||
|
||||
// If it's an update, run the algorithm for annotation merging, updating
|
||||
// etag/generation, etc.
|
||||
let response: DbEntityResponse;
|
||||
if (existing) {
|
||||
const updated = generateUpdatedEntity(existing.entity, entity);
|
||||
response = await this.database.updateEntity(
|
||||
tx,
|
||||
{ locationId, entity: updated, relations },
|
||||
existing.entity.metadata.etag,
|
||||
existing.entity.metadata.generation,
|
||||
);
|
||||
} else {
|
||||
const added = await this.database.addEntities(tx, [
|
||||
{ locationId, entity, relations },
|
||||
]);
|
||||
response = added[0];
|
||||
}
|
||||
|
||||
return response.entity;
|
||||
}
|
||||
|
||||
async entityAncestry(): Promise<never> {
|
||||
throw new Error('Not implemented');
|
||||
}
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { DatabaseManager } from '../database';
|
||||
import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
|
||||
|
||||
const bootstrapLocation = {
|
||||
id: expect.any(String),
|
||||
type: 'bootstrap',
|
||||
target: 'bootstrap',
|
||||
};
|
||||
|
||||
describe('DatabaseLocationsCatalog', () => {
|
||||
let catalog: DatabaseLocationsCatalog;
|
||||
|
||||
beforeEach(async () => {
|
||||
const db = await DatabaseManager.createTestDatabase();
|
||||
catalog = new DatabaseLocationsCatalog(db);
|
||||
});
|
||||
|
||||
it('can add a location', async () => {
|
||||
const location = {
|
||||
id: 'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
type: 'valid_type',
|
||||
target: 'valid_target',
|
||||
};
|
||||
await expect(catalog.addLocation(location)).resolves.toEqual(location);
|
||||
await expect(
|
||||
catalog.location('dd12620d-0436-422f-93bd-929aa0788123'),
|
||||
).resolves.toEqual(expect.objectContaining({ data: location }));
|
||||
await expect(catalog.locations()).resolves.toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ data: location }),
|
||||
expect.objectContaining({ data: bootstrapLocation }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('does not return duplicates of rows because of logs', async () => {
|
||||
const location1 = {
|
||||
id: 'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
type: 'valid_type',
|
||||
target: 'valid_target1',
|
||||
};
|
||||
const location2 = {
|
||||
id: '1a89c479-1a33-4f27-8927-6090ba488c42',
|
||||
type: 'valid_type',
|
||||
target: 'valid_target2',
|
||||
};
|
||||
await expect(catalog.addLocation(location1)).resolves.toEqual(location1);
|
||||
await expect(catalog.addLocation(location2)).resolves.toEqual(location2);
|
||||
await expect(
|
||||
catalog.logUpdateSuccess(location1.id),
|
||||
).resolves.toBeUndefined();
|
||||
await expect(
|
||||
catalog.logUpdateSuccess(location1.id),
|
||||
).resolves.toBeUndefined();
|
||||
const locations = await catalog.locations();
|
||||
expect(locations.length).toBe(3);
|
||||
expect(locations).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ data: location1 }),
|
||||
expect.objectContaining({ data: location2 }),
|
||||
expect.objectContaining({ data: bootstrapLocation }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,91 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Location } from '@backstage/catalog-model';
|
||||
import type { Database } from '../database';
|
||||
import {
|
||||
DatabaseLocationUpdateLogEvent,
|
||||
DatabaseLocationUpdateLogStatus,
|
||||
} from '../database/types';
|
||||
import { LocationResponse, LocationsCatalog } from './types';
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export class DatabaseLocationsCatalog implements LocationsCatalog {
|
||||
constructor(private readonly database: Database) {}
|
||||
|
||||
async addLocation(location: Location): Promise<Location> {
|
||||
return await this.database.transaction(
|
||||
async tx => await this.database.addLocation(tx, location),
|
||||
);
|
||||
}
|
||||
|
||||
async removeLocation(id: string): Promise<void> {
|
||||
await this.database.transaction(tx => this.database.removeLocation(tx, id));
|
||||
}
|
||||
|
||||
async locations(): Promise<LocationResponse[]> {
|
||||
const items = await this.database.locations();
|
||||
return items.map(({ message, status, timestamp, ...data }) => ({
|
||||
currentStatus: {
|
||||
message,
|
||||
status,
|
||||
timestamp,
|
||||
},
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
async locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]> {
|
||||
return this.database.locationHistory(id);
|
||||
}
|
||||
|
||||
async location(id: string): Promise<LocationResponse> {
|
||||
const { message, status, timestamp, ...data } =
|
||||
await this.database.location(id);
|
||||
return {
|
||||
currentStatus: {
|
||||
message,
|
||||
status,
|
||||
timestamp,
|
||||
},
|
||||
data,
|
||||
};
|
||||
}
|
||||
|
||||
async logUpdateSuccess(
|
||||
locationId: string,
|
||||
entityName?: string | string[],
|
||||
): Promise<void> {
|
||||
await this.database.addLocationUpdateLogEvent(
|
||||
locationId,
|
||||
DatabaseLocationUpdateLogStatus.SUCCESS,
|
||||
entityName,
|
||||
);
|
||||
}
|
||||
|
||||
async logUpdateFailure(
|
||||
locationId: string,
|
||||
error?: Error,
|
||||
entityName?: string,
|
||||
): Promise<void> {
|
||||
await this.database.addLocationUpdateLogEvent(
|
||||
locationId,
|
||||
DatabaseLocationUpdateLogStatus.FAIL,
|
||||
entityName,
|
||||
error?.message,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
|
||||
export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
|
||||
export type {
|
||||
LocationResponse,
|
||||
LocationsCatalog,
|
||||
LocationUpdateLogEvent,
|
||||
LocationUpdateStatus,
|
||||
} from './types';
|
||||
@@ -1,62 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Location } from '@backstage/catalog-model';
|
||||
|
||||
//
|
||||
// Locations
|
||||
//
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type LocationUpdateStatus = {
|
||||
timestamp: string | null;
|
||||
status: string | null;
|
||||
message: string | null;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type LocationUpdateLogEvent = {
|
||||
id: string;
|
||||
status: 'fail' | 'success';
|
||||
location_id: string;
|
||||
entity_name: string;
|
||||
created_at?: string;
|
||||
message?: string;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type LocationResponse = {
|
||||
data: Location;
|
||||
currentStatus: LocationUpdateStatus;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type LocationsCatalog = {
|
||||
addLocation(location: Location): Promise<Location>;
|
||||
removeLocation(id: string): Promise<void>;
|
||||
locations(): Promise<LocationResponse[]>;
|
||||
location(id: string): Promise<LocationResponse>;
|
||||
locationHistory(id: string): Promise<LocationUpdateLogEvent[]>;
|
||||
logUpdateSuccess(
|
||||
locationId: string,
|
||||
entityName?: string | string[],
|
||||
): Promise<void>;
|
||||
logUpdateFailure(
|
||||
locationId: string,
|
||||
error?: Error,
|
||||
entityName?: string,
|
||||
): Promise<void>;
|
||||
};
|
||||
@@ -1,789 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Entity, Location, parseEntityRef } from '@backstage/catalog-model';
|
||||
import { ConflictError } from '@backstage/errors';
|
||||
import { basicEntityFilter } from '../../service/request';
|
||||
import { DatabaseManager } from './DatabaseManager';
|
||||
import type {
|
||||
DbEntityRequest,
|
||||
DbEntityResponse,
|
||||
DbLocationsRowWithStatus,
|
||||
} from './types';
|
||||
import { Database, DatabaseLocationUpdateLogStatus } from './types';
|
||||
|
||||
const bootstrapLocation = {
|
||||
id: expect.any(String),
|
||||
type: 'bootstrap',
|
||||
target: 'bootstrap',
|
||||
message: null,
|
||||
status: null,
|
||||
timestamp: null,
|
||||
};
|
||||
|
||||
describe('CommonDatabase', () => {
|
||||
let db: Database;
|
||||
let entityRequest: DbEntityRequest;
|
||||
let entityResponse: DbEntityResponse;
|
||||
|
||||
beforeEach(async () => {
|
||||
db = await DatabaseManager.createTestDatabase();
|
||||
|
||||
entityRequest = {
|
||||
entity: {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
labels: { e: 'f' },
|
||||
annotations: { g: 'h' },
|
||||
},
|
||||
spec: { i: 'j' },
|
||||
},
|
||||
relations: [],
|
||||
};
|
||||
|
||||
entityResponse = {
|
||||
locationId: undefined,
|
||||
entity: {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
uid: expect.anything(),
|
||||
etag: expect.anything(),
|
||||
generation: expect.anything(),
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
labels: { e: 'f' },
|
||||
annotations: {
|
||||
g: 'h',
|
||||
},
|
||||
},
|
||||
spec: { i: 'j' },
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
it('manages locations', async () => {
|
||||
const input: Location = {
|
||||
id: 'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
type: 'a',
|
||||
target: 'b',
|
||||
};
|
||||
const output: DbLocationsRowWithStatus = {
|
||||
id: 'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
type: 'a',
|
||||
target: 'b',
|
||||
message: null,
|
||||
status: null,
|
||||
timestamp: null,
|
||||
};
|
||||
|
||||
await db.transaction(async tx => await db.addLocation(tx, input));
|
||||
|
||||
const locations = await db.locations();
|
||||
expect(locations).toEqual(
|
||||
expect.arrayContaining([output, bootstrapLocation]),
|
||||
);
|
||||
const location = await db.location(
|
||||
locations.find(l => l.type !== 'bootstrap')!.id,
|
||||
);
|
||||
expect(location).toEqual(output);
|
||||
|
||||
// If we add 2 new update log events,
|
||||
// this should not result in location duplication
|
||||
// due to incorrect join in DB
|
||||
await db.addLocationUpdateLogEvent(
|
||||
'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
DatabaseLocationUpdateLogStatus.SUCCESS,
|
||||
);
|
||||
|
||||
// Have a second in-between
|
||||
// To avoid having same timestamp on event
|
||||
await new Promise(res => setTimeout(res, 1000));
|
||||
await db.addLocationUpdateLogEvent(
|
||||
'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
DatabaseLocationUpdateLogStatus.FAIL,
|
||||
);
|
||||
|
||||
await expect(db.locations()).resolves.toEqual(
|
||||
expect.arrayContaining([
|
||||
bootstrapLocation,
|
||||
{
|
||||
...output,
|
||||
status: DatabaseLocationUpdateLogStatus.FAIL,
|
||||
timestamp: expect.anything(),
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
await db.transaction(tx => db.removeLocation(tx, location.id));
|
||||
|
||||
await expect(db.locations()).resolves.toEqual([bootstrapLocation]);
|
||||
await expect(db.location(location.id)).rejects.toThrow(/Found no location/);
|
||||
});
|
||||
|
||||
it('refuses to remove the bootstrap location', async () => {
|
||||
const input: Location = {
|
||||
id: 'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
type: 'bootstrap',
|
||||
target: 'bootstrap',
|
||||
};
|
||||
|
||||
const output = await db.transaction(
|
||||
async tx => await db.addLocation(tx, input),
|
||||
);
|
||||
|
||||
await expect(
|
||||
db.transaction(async tx => await db.removeLocation(tx, output.id)),
|
||||
).rejects.toThrow(ConflictError);
|
||||
});
|
||||
|
||||
describe('addEntities', () => {
|
||||
it('happy path: adds entities to empty database', async () => {
|
||||
const result = await db.transaction(tx =>
|
||||
db.addEntities(tx, [entityRequest]),
|
||||
);
|
||||
expect(result).toEqual([entityResponse]);
|
||||
});
|
||||
|
||||
it('rejects adding the same-named entity twice', async () => {
|
||||
const req: DbEntityRequest[] = [
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'av1',
|
||||
kind: 'k1',
|
||||
metadata: { name: 'n1', namespace: 'ns1' },
|
||||
},
|
||||
relations: [],
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'av1',
|
||||
kind: 'k1',
|
||||
metadata: { name: 'n1', namespace: 'ns1' },
|
||||
},
|
||||
relations: [],
|
||||
},
|
||||
];
|
||||
await expect(
|
||||
db.transaction(tx => db.addEntities(tx, req)),
|
||||
).rejects.toThrow(ConflictError);
|
||||
});
|
||||
|
||||
it('rejects adding the almost-same-namespace entity twice', async () => {
|
||||
const req: DbEntityRequest[] = [
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'av1',
|
||||
kind: 'k1',
|
||||
metadata: { name: 'n1', namespace: 'ns1' },
|
||||
},
|
||||
relations: [],
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'av1',
|
||||
kind: 'k1',
|
||||
metadata: { name: 'n1', namespace: 'nS1' },
|
||||
},
|
||||
relations: [],
|
||||
},
|
||||
];
|
||||
await expect(
|
||||
db.transaction(tx => db.addEntities(tx, req)),
|
||||
).rejects.toThrow(ConflictError);
|
||||
});
|
||||
|
||||
it('accepts adding the same-named entity twice if on different namespaces', async () => {
|
||||
const req: DbEntityRequest[] = [
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'av1',
|
||||
kind: 'k1',
|
||||
metadata: { name: 'n1', namespace: 'ns1' },
|
||||
},
|
||||
relations: [],
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'av1',
|
||||
kind: 'k1',
|
||||
metadata: { name: 'n1', namespace: 'ns2' },
|
||||
},
|
||||
relations: [],
|
||||
},
|
||||
];
|
||||
await expect(
|
||||
db.transaction(tx => db.addEntities(tx, req)),
|
||||
).resolves.toEqual([
|
||||
{
|
||||
entity: expect.objectContaining({
|
||||
metadata: expect.objectContaining({
|
||||
namespace: 'ns1',
|
||||
uid: expect.any(String),
|
||||
etag: expect.any(String),
|
||||
generation: expect.any(Number),
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
entity: expect.objectContaining({
|
||||
metadata: expect.objectContaining({
|
||||
namespace: 'ns2',
|
||||
uid: expect.any(String),
|
||||
etag: expect.any(String),
|
||||
generation: expect.any(Number),
|
||||
}),
|
||||
}),
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('locationHistory', () => {
|
||||
it('outputs the history correctly', async () => {
|
||||
const location: Location = {
|
||||
id: 'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
type: 'a',
|
||||
target: 'b',
|
||||
};
|
||||
|
||||
await db.transaction(async tx => await db.addLocation(tx, location));
|
||||
|
||||
await db.addLocationUpdateLogEvent(
|
||||
'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
DatabaseLocationUpdateLogStatus.SUCCESS,
|
||||
);
|
||||
await db.addLocationUpdateLogEvent(
|
||||
'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
DatabaseLocationUpdateLogStatus.FAIL,
|
||||
undefined,
|
||||
'Something went wrong',
|
||||
);
|
||||
|
||||
const result = await db.locationHistory(
|
||||
'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
);
|
||||
expect(result).toEqual(
|
||||
expect.arrayContaining([
|
||||
{
|
||||
created_at: expect.anything(),
|
||||
entity_name: null,
|
||||
id: expect.anything(),
|
||||
location_id: 'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
message: null,
|
||||
status: DatabaseLocationUpdateLogStatus.SUCCESS,
|
||||
},
|
||||
{
|
||||
created_at: expect.anything(),
|
||||
entity_name: null,
|
||||
id: expect.anything(),
|
||||
location_id: 'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
message: 'Something went wrong',
|
||||
status: DatabaseLocationUpdateLogStatus.FAIL,
|
||||
},
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('updateEntity', () => {
|
||||
it('can read and no-op-update an entity', async () => {
|
||||
const [added] = await db.transaction(tx =>
|
||||
db.addEntities(tx, [entityRequest]),
|
||||
);
|
||||
const updated = await db.transaction(tx =>
|
||||
db.updateEntity(tx, { entity: added.entity, relations: [] }),
|
||||
);
|
||||
expect(updated.entity.apiVersion).toEqual(added.entity.apiVersion);
|
||||
expect(updated.entity.kind).toEqual(added.entity.kind);
|
||||
expect(updated.entity.metadata.etag).toEqual(added.entity.metadata.etag);
|
||||
expect(updated.entity.metadata.generation).toEqual(
|
||||
added.entity.metadata.generation,
|
||||
);
|
||||
expect(updated.entity.metadata.name).toEqual(added.entity.metadata.name);
|
||||
expect(updated.entity.metadata.namespace).toEqual(
|
||||
added.entity.metadata.namespace,
|
||||
);
|
||||
});
|
||||
|
||||
it('can update name if uid matches', async () => {
|
||||
const [added] = await db.transaction(tx =>
|
||||
db.addEntities(tx, [entityRequest]),
|
||||
);
|
||||
added.entity.metadata.name! = 'new!';
|
||||
const updated = await db.transaction(tx =>
|
||||
db.updateEntity(tx, { entity: added.entity, relations: [] }),
|
||||
);
|
||||
expect(updated.entity.metadata.name).toEqual('new!');
|
||||
});
|
||||
|
||||
it('fails to update an entity if etag does not match', async () => {
|
||||
const [added] = await db.transaction(tx =>
|
||||
db.addEntities(tx, [entityRequest]),
|
||||
);
|
||||
await expect(
|
||||
db.transaction(tx =>
|
||||
db.updateEntity(
|
||||
tx,
|
||||
{ entity: added.entity, relations: [] },
|
||||
'garbage',
|
||||
),
|
||||
),
|
||||
).rejects.toThrow(ConflictError);
|
||||
});
|
||||
|
||||
it('fails to update an entity if generation does not match', async () => {
|
||||
const [added] = await db.transaction(tx =>
|
||||
db.addEntities(tx, [entityRequest]),
|
||||
);
|
||||
await expect(
|
||||
db.transaction(tx =>
|
||||
db.updateEntity(
|
||||
tx,
|
||||
{ entity: added.entity, relations: [] },
|
||||
undefined,
|
||||
1e20,
|
||||
),
|
||||
),
|
||||
).rejects.toThrow(ConflictError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('entities', () => {
|
||||
it('can get all entities with empty filters list', async () => {
|
||||
const e1: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'k1',
|
||||
metadata: { name: 'n' },
|
||||
};
|
||||
const e2: Entity = {
|
||||
apiVersion: 'c',
|
||||
kind: 'k2',
|
||||
metadata: { name: 'n' },
|
||||
spec: { c: null },
|
||||
};
|
||||
await db.transaction(async tx => {
|
||||
await db.addEntities(tx, [
|
||||
{ entity: e1, relations: [] },
|
||||
{ entity: e2, relations: [] },
|
||||
]);
|
||||
});
|
||||
const result = await db.transaction(async tx => db.entities(tx));
|
||||
expect(result.entities.length).toEqual(2);
|
||||
expect(result.entities).toEqual(
|
||||
expect.arrayContaining([
|
||||
{
|
||||
locationId: undefined,
|
||||
entity: expect.objectContaining({ kind: 'k1' }),
|
||||
},
|
||||
{
|
||||
locationId: undefined,
|
||||
entity: expect.objectContaining({ kind: 'k2' }),
|
||||
},
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('can get all specific entities for matching filters (naive case)', async () => {
|
||||
const entities: Entity[] = [
|
||||
{ apiVersion: 'a', kind: 'k1', metadata: { name: 'n' } },
|
||||
{
|
||||
apiVersion: 'a',
|
||||
kind: 'k2',
|
||||
metadata: { name: 'n' },
|
||||
spec: { c: 'some' },
|
||||
},
|
||||
{
|
||||
apiVersion: 'a',
|
||||
kind: 'k3',
|
||||
metadata: { name: 'n' },
|
||||
spec: { c: null },
|
||||
},
|
||||
];
|
||||
|
||||
await db.transaction(async tx => {
|
||||
await db.addEntities(
|
||||
tx,
|
||||
entities.map(entity => ({ entity, relations: [] })),
|
||||
);
|
||||
});
|
||||
|
||||
const response = await db.transaction(async tx =>
|
||||
db.entities(tx, {
|
||||
filter: basicEntityFilter({ kind: 'k2', 'spec.c': 'some' }),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.entities).toEqual([
|
||||
{
|
||||
locationId: undefined,
|
||||
entity: expect.objectContaining({ kind: 'k2' }),
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('can get all specific entities for matching filters case insensitively', async () => {
|
||||
const entities: Entity[] = [
|
||||
{
|
||||
apiVersion: 'A',
|
||||
kind: 'K1',
|
||||
metadata: { name: 'N' },
|
||||
spec: { c: 'SOME' },
|
||||
},
|
||||
{
|
||||
apiVersion: 'a',
|
||||
kind: 'k2',
|
||||
metadata: { name: 'n' },
|
||||
spec: { c: 'Some' },
|
||||
},
|
||||
{
|
||||
apiVersion: 'a',
|
||||
kind: 'k3',
|
||||
metadata: { name: 'n' },
|
||||
spec: { c: 'somE' },
|
||||
},
|
||||
];
|
||||
|
||||
await db.transaction(async tx => {
|
||||
await db.addEntities(
|
||||
tx,
|
||||
entities.map(entity => ({ entity, relations: [] })),
|
||||
);
|
||||
});
|
||||
|
||||
const rows = await db.transaction(async tx =>
|
||||
db.entities(tx, {
|
||||
filter: basicEntityFilter({ ApiVersioN: 'A', 'spEc.C': 'some' }),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(rows.entities.length).toEqual(3);
|
||||
expect(rows.entities).toEqual(
|
||||
expect.arrayContaining([
|
||||
{
|
||||
locationId: undefined,
|
||||
entity: expect.objectContaining({ kind: 'K1' }),
|
||||
},
|
||||
{
|
||||
locationId: undefined,
|
||||
entity: expect.objectContaining({ kind: 'k2' }),
|
||||
},
|
||||
{
|
||||
locationId: undefined,
|
||||
entity: expect.objectContaining({ kind: 'k3' }),
|
||||
},
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('can get all specific entities for matching existence filters', async () => {
|
||||
const entities: Entity[] = [
|
||||
{
|
||||
apiVersion: 'A',
|
||||
kind: 'K1',
|
||||
metadata: {
|
||||
name: 'N',
|
||||
annotations: {
|
||||
foo: 'bar',
|
||||
},
|
||||
},
|
||||
spec: { c: 'SOME' },
|
||||
},
|
||||
{
|
||||
apiVersion: 'a',
|
||||
kind: 'k2',
|
||||
metadata: {
|
||||
name: 'N',
|
||||
annotations: {
|
||||
foo: 'bar',
|
||||
},
|
||||
},
|
||||
spec: { c: 'Some' },
|
||||
},
|
||||
{
|
||||
apiVersion: 'a',
|
||||
kind: 'k3',
|
||||
metadata: { name: 'n' },
|
||||
spec: { c: 'somE' },
|
||||
},
|
||||
];
|
||||
|
||||
await db.transaction(async tx => {
|
||||
await db.addEntities(
|
||||
tx,
|
||||
entities.map(entity => ({ entity, relations: [] })),
|
||||
);
|
||||
});
|
||||
|
||||
const existRows = await db.transaction(async tx =>
|
||||
db.entities(tx, {
|
||||
filter: {
|
||||
anyOf: [
|
||||
{
|
||||
allOf: [{ key: 'metadata.annotations.foo' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(existRows.entities.length).toEqual(2);
|
||||
expect(existRows.entities).toEqual(
|
||||
expect.arrayContaining([
|
||||
{
|
||||
locationId: undefined,
|
||||
entity: expect.objectContaining({ kind: 'K1' }),
|
||||
},
|
||||
{
|
||||
locationId: undefined,
|
||||
entity: expect.objectContaining({ kind: 'k2' }),
|
||||
},
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('setRelations', () => {
|
||||
it('adds a relation for an entity', async () => {
|
||||
const mockRelations = [
|
||||
{
|
||||
source: {
|
||||
kind: entityRequest.entity.kind,
|
||||
namespace: entityRequest.entity.metadata.namespace!,
|
||||
name: entityRequest.entity.metadata.name,
|
||||
},
|
||||
target: {
|
||||
kind: 'component',
|
||||
namespace: 'asd',
|
||||
name: 'bleb',
|
||||
},
|
||||
type: 'child',
|
||||
},
|
||||
];
|
||||
|
||||
const entityId = await db.transaction(async tx => {
|
||||
const [{ entity }] = await db.addEntities(tx, [entityRequest]);
|
||||
|
||||
await db.setRelations(tx, entity?.metadata?.uid!, mockRelations);
|
||||
return entity.metadata.uid;
|
||||
});
|
||||
|
||||
const returnedEntity1 = await db.transaction(tx =>
|
||||
db.entityByUid(tx, entityId!),
|
||||
);
|
||||
expect(returnedEntity1?.entity.relations).toEqual([
|
||||
{ target: mockRelations[0].target, type: 'child' },
|
||||
]);
|
||||
|
||||
const returnedEntity2 = await db.transaction(tx =>
|
||||
db.entityByName(tx, mockRelations[0].source),
|
||||
);
|
||||
expect(returnedEntity2?.entity.relations).toEqual([
|
||||
{ target: mockRelations[0].target, type: 'child' },
|
||||
]);
|
||||
|
||||
const { entities } = await db.transaction(tx => db.entities(tx));
|
||||
const [returnedEntity3] = entities;
|
||||
expect(returnedEntity3?.entity.relations).toEqual([
|
||||
{ target: mockRelations[0].target, type: 'child' },
|
||||
]);
|
||||
});
|
||||
|
||||
function makeRelation(source: string, type: string, target: string) {
|
||||
return {
|
||||
source: parseEntityRef(source, {
|
||||
defaultKind: 'x',
|
||||
defaultNamespace: 'x',
|
||||
}),
|
||||
type,
|
||||
target: parseEntityRef(target, {
|
||||
defaultKind: 'x',
|
||||
defaultNamespace: 'x',
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
it('should not allow setting relations on nonexistent entities', async () => {
|
||||
await expect(
|
||||
db.transaction(async tx => {
|
||||
await db.setRelations(tx, 'nonexistent', [
|
||||
makeRelation('a:b/c', 'rel1', 'x:y/z'),
|
||||
]);
|
||||
}),
|
||||
).rejects.toThrow(/constraint failed/);
|
||||
});
|
||||
|
||||
it('should allow setting relations on nonexistent entities without any relations', async () => {
|
||||
await expect(
|
||||
db.transaction(async tx => {
|
||||
await db.setRelations(tx, 'nonexistent', []);
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('adds multiple relations for entities', async () => {
|
||||
const entity1 = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'a',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
namespace: 'b',
|
||||
},
|
||||
};
|
||||
const entity2 = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'x',
|
||||
metadata: {
|
||||
name: 'z',
|
||||
namespace: 'y',
|
||||
},
|
||||
};
|
||||
const fromEntity1 = [
|
||||
makeRelation('a:b/c', 'rel1', 'x:y/z'),
|
||||
makeRelation('x:y/z', 'rel2', 'a:b/c'),
|
||||
makeRelation('a:b/c', 'rel2', 'x:y/z'),
|
||||
];
|
||||
const fromEntity2 = [
|
||||
makeRelation('a:b/c', 'rel4', 'x:y/z'),
|
||||
makeRelation('a:b/c', 'rel5', 'x:y/z'),
|
||||
makeRelation('x:y/z', 'rel6', 'a:b/c'),
|
||||
// relations don't have to reference the originating entity, so this should be fine, but not show up
|
||||
makeRelation('g:h/i', 'rel8', 'd:e/f'),
|
||||
];
|
||||
|
||||
const { id2: secondEntityId } = await db.transaction(async tx => {
|
||||
const [{ entity: e1 }, { entity: e2 }] = await db.addEntities(tx, [
|
||||
{ entity: entity1, relations: [] },
|
||||
{ entity: entity2, relations: [] },
|
||||
]);
|
||||
const id1 = e1?.metadata?.uid!;
|
||||
const id2 = e2?.metadata?.uid!;
|
||||
|
||||
await db.setRelations(tx, id1, fromEntity1);
|
||||
await db.setRelations(tx, id2, fromEntity2);
|
||||
|
||||
return { id1, id2 };
|
||||
});
|
||||
|
||||
const res = await db.transaction(tx => db.entities(tx));
|
||||
expect(
|
||||
res.entities.map(r => ({
|
||||
name: r.entity.metadata.name,
|
||||
relations: r.entity.relations,
|
||||
})),
|
||||
).toEqual([
|
||||
{
|
||||
name: 'c',
|
||||
relations: [
|
||||
{
|
||||
type: 'rel1',
|
||||
target: { kind: 'x', namespace: 'y', name: 'z' },
|
||||
},
|
||||
{
|
||||
type: 'rel2',
|
||||
target: { kind: 'x', namespace: 'y', name: 'z' },
|
||||
},
|
||||
{
|
||||
type: 'rel4',
|
||||
target: { kind: 'x', namespace: 'y', name: 'z' },
|
||||
},
|
||||
{
|
||||
type: 'rel5',
|
||||
target: { kind: 'x', namespace: 'y', name: 'z' },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'z',
|
||||
relations: [
|
||||
{
|
||||
type: 'rel2',
|
||||
target: { kind: 'a', namespace: 'b', name: 'c' },
|
||||
},
|
||||
{
|
||||
type: 'rel6',
|
||||
target: { kind: 'a', namespace: 'b', name: 'c' },
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
await db.transaction(tx => db.removeEntityByUid(tx, secondEntityId));
|
||||
|
||||
const res2 = await db.transaction(tx => db.entities(tx));
|
||||
expect(
|
||||
res2.entities.map(r => ({
|
||||
name: r.entity.metadata.name,
|
||||
relations: r.entity.relations,
|
||||
})),
|
||||
).toEqual([
|
||||
{
|
||||
name: 'c',
|
||||
relations: [
|
||||
{
|
||||
type: 'rel1',
|
||||
target: { kind: 'x', namespace: 'y', name: 'z' },
|
||||
},
|
||||
{
|
||||
type: 'rel2',
|
||||
target: { kind: 'x', namespace: 'y', name: 'z' },
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('entityByName', () => {
|
||||
it('can get entities case insensitively', async () => {
|
||||
const entities: Entity[] = [
|
||||
{
|
||||
apiVersion: 'a',
|
||||
kind: 'k1',
|
||||
metadata: { name: 'n' },
|
||||
},
|
||||
{
|
||||
apiVersion: 'B',
|
||||
kind: 'K2',
|
||||
metadata: { name: 'N', namespace: 'NS' },
|
||||
},
|
||||
];
|
||||
|
||||
await db.transaction(async tx => {
|
||||
await db.addEntities(
|
||||
tx,
|
||||
entities.map(entity => ({ entity, relations: [] })),
|
||||
);
|
||||
});
|
||||
|
||||
const e1 = await db.transaction(async tx =>
|
||||
db.entityByName(tx, { kind: 'k1', namespace: 'default', name: 'n' }),
|
||||
);
|
||||
expect(e1!.entity.metadata.name).toEqual('n');
|
||||
const e2 = await db.transaction(async tx =>
|
||||
db.entityByName(tx, { kind: 'k2', namespace: 'nS', name: 'n' }),
|
||||
);
|
||||
expect(e2!.entity.metadata.name).toEqual('N');
|
||||
const e3 = await db.transaction(async tx =>
|
||||
db.entityByName(tx, { kind: 'unknown', namespace: 'nS', name: 'n' }),
|
||||
);
|
||||
expect(e3).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,624 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConflictError, InputError, NotFoundError } from '@backstage/errors';
|
||||
import {
|
||||
Entity,
|
||||
EntityName,
|
||||
EntityRelationSpec,
|
||||
ENTITY_DEFAULT_NAMESPACE,
|
||||
ENTITY_META_GENERATED_FIELDS,
|
||||
generateEntityEtag,
|
||||
generateEntityUid,
|
||||
Location,
|
||||
parseEntityName,
|
||||
} from '@backstage/catalog-model';
|
||||
import { Knex } from 'knex';
|
||||
import lodash from 'lodash';
|
||||
import type { Logger } from 'winston';
|
||||
import { buildEntitySearch } from './search';
|
||||
import {
|
||||
Database,
|
||||
DatabaseLocationUpdateLogEvent,
|
||||
DatabaseLocationUpdateLogStatus,
|
||||
DbEntitiesRelationsRow,
|
||||
DbEntitiesRequest,
|
||||
DbEntitiesResponse,
|
||||
DbEntitiesRow,
|
||||
DbEntitiesSearchRow,
|
||||
DbEntityRequest,
|
||||
DbEntityResponse,
|
||||
DbLocationsRow,
|
||||
DbLocationsRowWithStatus,
|
||||
DbPageInfo,
|
||||
Transaction,
|
||||
} from './types';
|
||||
import { EntityPagination, EntitiesSearchFilter } from '../../catalog/types';
|
||||
|
||||
type LegacyEntityFilter = {
|
||||
anyOf: { allOf: EntitiesSearchFilter[] }[];
|
||||
};
|
||||
|
||||
// The number of items that are sent per batch to the database layer, when
|
||||
// doing .batchInsert calls to knex. This needs to be low enough to not cause
|
||||
// errors in the underlying engine due to exceeding query limits, but large
|
||||
// enough to get the speed benefits.
|
||||
const BATCH_SIZE = 50;
|
||||
|
||||
/**
|
||||
* The core database implementation..
|
||||
* @deprecated This was part of the legacy catalog engin
|
||||
*/
|
||||
export class CommonDatabase implements Database {
|
||||
constructor(
|
||||
private readonly database: Knex,
|
||||
private readonly logger: Logger,
|
||||
) {}
|
||||
|
||||
async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {
|
||||
try {
|
||||
let result: T | undefined = undefined;
|
||||
|
||||
await this.database.transaction(
|
||||
async tx => {
|
||||
// We can't return here, as knex swallows the return type in case the transaction is rolled back:
|
||||
// https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136
|
||||
result = await fn(tx);
|
||||
},
|
||||
{
|
||||
// If we explicitly trigger a rollback, don't fail.
|
||||
doNotRejectOnRollback: true,
|
||||
},
|
||||
);
|
||||
|
||||
return result!;
|
||||
} catch (e) {
|
||||
this.logger.debug(`Error during transaction, ${e}`);
|
||||
|
||||
if (
|
||||
/SQLITE_CONSTRAINT: UNIQUE/.test(e.message) ||
|
||||
/unique constraint/.test(e.message)
|
||||
) {
|
||||
throw new ConflictError(`Rejected due to a conflicting entity`, e);
|
||||
}
|
||||
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async addEntities(
|
||||
txOpaque: Transaction,
|
||||
request: DbEntityRequest[],
|
||||
): Promise<DbEntityResponse[]> {
|
||||
const tx = txOpaque as Knex.Transaction;
|
||||
|
||||
const result: DbEntityResponse[] = [];
|
||||
const entityRows: DbEntitiesRow[] = [];
|
||||
const relationRows: DbEntitiesRelationsRow[] = [];
|
||||
const searchRows: DbEntitiesSearchRow[] = [];
|
||||
|
||||
for (const { entity, relations, locationId } of request) {
|
||||
if (entity.metadata.uid !== undefined) {
|
||||
throw new InputError('May not specify uid for new entities');
|
||||
} else if (entity.metadata.etag !== undefined) {
|
||||
throw new InputError('May not specify etag for new entities');
|
||||
} else if (entity.metadata.generation !== undefined) {
|
||||
throw new InputError('May not specify generation for new entities');
|
||||
} else if (entity.relations !== undefined) {
|
||||
throw new InputError('May not specify relations for new entities');
|
||||
}
|
||||
|
||||
const uid = generateEntityUid();
|
||||
const etag = generateEntityEtag();
|
||||
const generation = 1;
|
||||
const newEntity = {
|
||||
...entity,
|
||||
metadata: {
|
||||
...entity.metadata,
|
||||
uid,
|
||||
etag,
|
||||
generation,
|
||||
},
|
||||
};
|
||||
|
||||
result.push({ entity: newEntity, locationId });
|
||||
entityRows.push(this.toEntityRow(locationId, newEntity));
|
||||
relationRows.push(...this.toRelationRows(uid, relations));
|
||||
searchRows.push(...buildEntitySearch(uid, newEntity));
|
||||
}
|
||||
|
||||
await tx.batchInsert('entities', entityRows, BATCH_SIZE);
|
||||
await tx.batchInsert('entities_relations', relationRows, BATCH_SIZE);
|
||||
await tx.batchInsert('entities_search', searchRows, BATCH_SIZE);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async updateEntity(
|
||||
txOpaque: Transaction,
|
||||
request: DbEntityRequest,
|
||||
matchingEtag?: string,
|
||||
matchingGeneration?: number,
|
||||
): Promise<DbEntityResponse> {
|
||||
const tx = txOpaque as Knex.Transaction;
|
||||
|
||||
const { uid } = request.entity.metadata;
|
||||
if (!uid) {
|
||||
throw new InputError('Must specify uid when updating entities');
|
||||
}
|
||||
|
||||
// Find existing entity
|
||||
const oldRows = await tx<DbEntitiesRow>('entities')
|
||||
.where({ id: uid })
|
||||
.select();
|
||||
if (oldRows.length !== 1) {
|
||||
throw new NotFoundError('No matching entity found');
|
||||
}
|
||||
const etag = oldRows[0].etag;
|
||||
const generation = Number(oldRows[0].generation);
|
||||
|
||||
// Validate the old entity. The Number cast is here because sqlite reads it
|
||||
// as a string, no matter what the table actually says.
|
||||
if (matchingEtag && matchingEtag !== etag) {
|
||||
throw new ConflictError(
|
||||
`Etag mismatch, expected="${matchingEtag}" found="${etag}"`,
|
||||
);
|
||||
}
|
||||
if (matchingGeneration && matchingGeneration !== generation) {
|
||||
throw new ConflictError(
|
||||
`Generation mismatch, expected="${matchingGeneration}" found="${generation}"`,
|
||||
);
|
||||
}
|
||||
|
||||
// Store the updated entity; select on the old etag to ensure that we do
|
||||
// not lose to another writer
|
||||
const newRow = this.toEntityRow(request.locationId, request.entity);
|
||||
const updatedRows = await tx<DbEntitiesRow>('entities')
|
||||
.where({ id: uid, etag })
|
||||
.update(newRow);
|
||||
if (updatedRows !== 1) {
|
||||
throw new ConflictError(`Failed to update entity`);
|
||||
}
|
||||
|
||||
const relationRows = this.toRelationRows(uid, request.relations);
|
||||
await tx<DbEntitiesRelationsRow>('entities_relations')
|
||||
.where({ originating_entity_id: uid })
|
||||
.del();
|
||||
await tx.batchInsert('entities_relations', relationRows, BATCH_SIZE);
|
||||
|
||||
try {
|
||||
const entries = buildEntitySearch(uid, request.entity);
|
||||
await tx<DbEntitiesSearchRow>('entities_search')
|
||||
.where({ entity_id: uid })
|
||||
.del();
|
||||
await tx.batchInsert('entities_search', entries, BATCH_SIZE);
|
||||
} catch {
|
||||
// ignore intentionally - if this happens, the entity was deleted before
|
||||
// we got around to writing the entries
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
async entities(
|
||||
txOpaque: Transaction,
|
||||
request?: DbEntitiesRequest,
|
||||
): Promise<DbEntitiesResponse> {
|
||||
const tx = txOpaque as Knex.Transaction;
|
||||
|
||||
let entitiesQuery = tx<DbEntitiesRow>('entities');
|
||||
|
||||
if (
|
||||
request?.filter &&
|
||||
(request.filter.hasOwnProperty('key') ||
|
||||
request.filter.hasOwnProperty('allOf') ||
|
||||
request.filter.hasOwnProperty('not'))
|
||||
) {
|
||||
throw new Error(
|
||||
'Filters for the legacy CommonDatabase must obey the { anyOf: [{ allOf: [] }] } format.',
|
||||
);
|
||||
}
|
||||
for (const singleFilter of (request?.filter as LegacyEntityFilter)?.anyOf ??
|
||||
[]) {
|
||||
entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() {
|
||||
for (const filter of singleFilter.allOf) {
|
||||
if (
|
||||
filter.hasOwnProperty('anyOf') ||
|
||||
filter.hasOwnProperty('allOf') ||
|
||||
filter.hasOwnProperty('not')
|
||||
) {
|
||||
throw new Error(
|
||||
'Nested filters are not supported in the legacy CommonDatabase',
|
||||
);
|
||||
}
|
||||
const { key, values } = filter;
|
||||
// NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
|
||||
// make a lot of sense. However, it had abysmal performance on sqlite
|
||||
// when datasets grew large, so we're using IN instead.
|
||||
const matchQuery = tx<DbEntitiesSearchRow>('entities_search')
|
||||
.select('entity_id')
|
||||
.where(function keyFilter() {
|
||||
this.andWhere({ key: key.toLowerCase() });
|
||||
if (values) {
|
||||
if (values.length === 1) {
|
||||
this.andWhere({ value: values[0].toLowerCase() });
|
||||
} else if (values.length > 1) {
|
||||
this.andWhere(
|
||||
'value',
|
||||
'in',
|
||||
values.map(v => v.toLowerCase()),
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
this.andWhere('id', 'in', matchQuery);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
entitiesQuery = entitiesQuery
|
||||
.select('entities.*')
|
||||
.orderBy('full_name', 'asc');
|
||||
|
||||
const { limit, offset } = parsePagination(request?.pagination);
|
||||
if (limit !== undefined) {
|
||||
entitiesQuery = entitiesQuery.limit(limit + 1);
|
||||
}
|
||||
if (offset !== undefined) {
|
||||
entitiesQuery = entitiesQuery.offset(offset);
|
||||
}
|
||||
|
||||
let rows = await entitiesQuery;
|
||||
|
||||
let pageInfo: DbPageInfo;
|
||||
if (limit === undefined || rows.length <= limit) {
|
||||
pageInfo = { hasNextPage: false };
|
||||
} else {
|
||||
rows = rows.slice(0, -1);
|
||||
pageInfo = {
|
||||
hasNextPage: true,
|
||||
endCursor: stringifyPagination({
|
||||
limit,
|
||||
offset: (offset ?? 0) + limit,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
entities: await this.toEntityResponses(tx, rows),
|
||||
pageInfo,
|
||||
};
|
||||
}
|
||||
|
||||
async entityByName(
|
||||
txOpaque: Transaction,
|
||||
name: EntityName,
|
||||
): Promise<DbEntityResponse | undefined> {
|
||||
const tx = txOpaque as Knex.Transaction;
|
||||
|
||||
const rows = await tx<DbEntitiesRow>('entities')
|
||||
.where({
|
||||
full_name: `${name.kind}:${name.namespace}/${name.name}`.toLowerCase(),
|
||||
})
|
||||
.select();
|
||||
|
||||
if (rows.length !== 1) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return this.toEntityResponses(tx, rows).then(r => r[0]);
|
||||
}
|
||||
|
||||
async entityByUid(
|
||||
txOpaque: Transaction,
|
||||
uid: string,
|
||||
): Promise<DbEntityResponse | undefined> {
|
||||
const tx = txOpaque as Knex.Transaction;
|
||||
|
||||
const rows = await tx<DbEntitiesRow>('entities')
|
||||
.where({ id: uid })
|
||||
.select();
|
||||
|
||||
if (rows.length !== 1) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return this.toEntityResponses(tx, rows).then(r => r[0]);
|
||||
}
|
||||
|
||||
async removeEntityByUid(txOpaque: Transaction, uid: string): Promise<void> {
|
||||
const tx = txOpaque as Knex.Transaction;
|
||||
|
||||
const result = await tx<DbEntitiesRow>('entities').where({ id: uid }).del();
|
||||
if (!result) {
|
||||
throw new NotFoundError(`Found no entity with ID ${uid}`);
|
||||
}
|
||||
}
|
||||
|
||||
async setRelations(
|
||||
txOpaque: Transaction,
|
||||
originatingEntityId: string,
|
||||
relations: EntityRelationSpec[],
|
||||
): Promise<void> {
|
||||
const tx = txOpaque as Knex.Transaction;
|
||||
const relationRows = this.toRelationRows(originatingEntityId, relations);
|
||||
|
||||
await tx<DbEntitiesRelationsRow>('entities_relations')
|
||||
.where({ originating_entity_id: originatingEntityId })
|
||||
.del();
|
||||
await tx.batchInsert('entities_relations', relationRows, BATCH_SIZE);
|
||||
}
|
||||
|
||||
async addLocation(
|
||||
txOpaque: Transaction,
|
||||
location: Location,
|
||||
): Promise<DbLocationsRow> {
|
||||
const tx = txOpaque as Knex.Transaction;
|
||||
|
||||
const row: DbLocationsRow = {
|
||||
id: location.id,
|
||||
type: location.type,
|
||||
target: location.target,
|
||||
};
|
||||
await tx<DbLocationsRow>('locations').insert(row);
|
||||
return row;
|
||||
}
|
||||
|
||||
async removeLocation(txOpaque: Transaction, id: string): Promise<void> {
|
||||
const tx = txOpaque as Knex.Transaction;
|
||||
|
||||
const locations = await tx<DbLocationsRow>('locations')
|
||||
.where({ id })
|
||||
.select();
|
||||
if (!locations.length) {
|
||||
throw new NotFoundError(`Found no location with ID ${id}`);
|
||||
}
|
||||
|
||||
if (locations[0].type === 'bootstrap') {
|
||||
throw new ConflictError('You may not delete the bootstrap location.');
|
||||
}
|
||||
|
||||
await tx<DbEntitiesRow>('entities')
|
||||
.where({ location_id: id })
|
||||
.update({ location_id: null });
|
||||
await tx<DbLocationsRow>('locations').where({ id }).del();
|
||||
}
|
||||
|
||||
async location(id: string): Promise<DbLocationsRowWithStatus> {
|
||||
const items = await this.database<DbLocationsRowWithStatus>('locations')
|
||||
.where('locations.id', id)
|
||||
.leftOuterJoin(
|
||||
'location_update_log_latest',
|
||||
'locations.id',
|
||||
'location_update_log_latest.location_id',
|
||||
)
|
||||
.select('locations.*', {
|
||||
status: 'location_update_log_latest.status',
|
||||
timestamp: 'location_update_log_latest.created_at',
|
||||
message: 'location_update_log_latest.message',
|
||||
});
|
||||
|
||||
if (!items.length) {
|
||||
throw new NotFoundError(`Found no location with ID ${id}`);
|
||||
}
|
||||
return items[0];
|
||||
}
|
||||
|
||||
async locations(): Promise<DbLocationsRowWithStatus[]> {
|
||||
const locations = await this.database('locations')
|
||||
.leftOuterJoin(
|
||||
'location_update_log_latest',
|
||||
'locations.id',
|
||||
'location_update_log_latest.location_id',
|
||||
)
|
||||
.select('locations.*', {
|
||||
status: 'location_update_log_latest.status',
|
||||
timestamp: 'location_update_log_latest.created_at',
|
||||
message: 'location_update_log_latest.message',
|
||||
});
|
||||
|
||||
return locations;
|
||||
}
|
||||
|
||||
async locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]> {
|
||||
const result = await this.database<DatabaseLocationUpdateLogEvent>(
|
||||
'location_update_log',
|
||||
)
|
||||
.where('location_id', id)
|
||||
.orderBy('created_at', 'desc')
|
||||
.limit(10)
|
||||
.select();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async addLocationUpdateLogEvent(
|
||||
locationId: string,
|
||||
status: DatabaseLocationUpdateLogStatus,
|
||||
entityName?: string | string[],
|
||||
message?: string,
|
||||
): Promise<void> {
|
||||
// Remove log entries older than a day
|
||||
const cutoff = new Date();
|
||||
cutoff.setDate(cutoff.getDate() - 1);
|
||||
await this.database<DatabaseLocationUpdateLogEvent>('location_update_log')
|
||||
.where('created_at', '<', cutoff.toISOString())
|
||||
.del();
|
||||
|
||||
const items: Partial<DatabaseLocationUpdateLogEvent>[] = [entityName]
|
||||
.flat()
|
||||
.map(n => ({
|
||||
status,
|
||||
location_id: locationId,
|
||||
entity_name: n,
|
||||
message,
|
||||
}));
|
||||
|
||||
for (const chunk of lodash.chunk(items, BATCH_SIZE)) {
|
||||
await this.database<DatabaseLocationUpdateLogEvent>(
|
||||
'location_update_log',
|
||||
).insert(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
private toEntityRow(
|
||||
locationId: string | undefined,
|
||||
entity: Entity,
|
||||
): DbEntitiesRow {
|
||||
const lowerKind = entity.kind.toLowerCase();
|
||||
const lowerNamespace = (
|
||||
entity.metadata.namespace || ENTITY_DEFAULT_NAMESPACE
|
||||
).toLowerCase();
|
||||
const lowerName = entity.metadata.name.toLowerCase();
|
||||
|
||||
const data = {
|
||||
...entity,
|
||||
metadata: lodash.omit(entity.metadata, ...ENTITY_META_GENERATED_FIELDS),
|
||||
};
|
||||
|
||||
return {
|
||||
id: entity.metadata.uid!,
|
||||
location_id: locationId || null,
|
||||
etag: entity.metadata.etag!,
|
||||
generation: entity.metadata.generation!,
|
||||
full_name: `${lowerKind}:${lowerNamespace}/${lowerName}`,
|
||||
data: JSON.stringify(data),
|
||||
};
|
||||
}
|
||||
|
||||
private toRelationRows(
|
||||
originatingEntityId: string,
|
||||
relations: EntityRelationSpec[],
|
||||
): DbEntitiesRelationsRow[] {
|
||||
const serializeName = (e: EntityName) =>
|
||||
`${e.kind}:${e.namespace}/${e.name}`.toLowerCase();
|
||||
|
||||
const rows = relations.map(({ source, target, type }) => ({
|
||||
originating_entity_id: originatingEntityId,
|
||||
source_full_name: serializeName(source),
|
||||
target_full_name: serializeName(target),
|
||||
type,
|
||||
}));
|
||||
|
||||
return deduplicateRelations(rows);
|
||||
}
|
||||
|
||||
private async toEntityResponses(
|
||||
tx: Knex.Transaction,
|
||||
rows: DbEntitiesRow[],
|
||||
): Promise<DbEntityResponse[]> {
|
||||
// TODO(Rugvip): This is here because it's simple for now, but we likely
|
||||
// need to refactor this to be more efficient or introduce pagination.
|
||||
const relations = await this.getRelationsPerFullName(
|
||||
tx,
|
||||
rows.map(r => r.full_name),
|
||||
);
|
||||
|
||||
const result = new Array<DbEntityResponse>();
|
||||
for (const row of rows) {
|
||||
const entity = JSON.parse(row.data) as Entity;
|
||||
entity.metadata.uid = row.id;
|
||||
entity.metadata.etag = row.etag;
|
||||
entity.metadata.generation = Number(row.generation); // cast due to sqlite
|
||||
|
||||
entity.relations = (relations[row.full_name] ?? []).map(r => ({
|
||||
target: parseEntityName(r.target_full_name),
|
||||
type: r.type,
|
||||
}));
|
||||
|
||||
result.push({
|
||||
locationId: row.location_id || undefined,
|
||||
entity,
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// Returns a mapping from e.g. component:default/foo to the relations whose
|
||||
// source_full_name matches that.
|
||||
private async getRelationsPerFullName(
|
||||
tx: Knex.Transaction,
|
||||
sourceFullNames: string[],
|
||||
): Promise<Record<string, DbEntitiesRelationsRow[]>> {
|
||||
const batches = lodash.chunk(lodash.uniq(sourceFullNames), 500);
|
||||
|
||||
const relations = new Array<DbEntitiesRelationsRow>();
|
||||
for (const batch of batches) {
|
||||
relations.push(
|
||||
...(await tx<DbEntitiesRelationsRow>('entities_relations')
|
||||
.whereIn('source_full_name', batch)
|
||||
.orderBy(['type', 'target_full_name'])
|
||||
.select()),
|
||||
);
|
||||
}
|
||||
|
||||
return lodash.groupBy(
|
||||
deduplicateRelations(relations),
|
||||
r => r.source_full_name,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function parsePagination(input?: EntityPagination): {
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
} {
|
||||
if (!input) {
|
||||
return {};
|
||||
}
|
||||
|
||||
let { limit, offset } = input;
|
||||
|
||||
if (input.after !== undefined) {
|
||||
let cursor;
|
||||
try {
|
||||
const json = Buffer.from(input.after, 'base64').toString('utf8');
|
||||
cursor = JSON.parse(json);
|
||||
} catch {
|
||||
throw new InputError('Malformed after cursor, could not be parsed');
|
||||
}
|
||||
if (cursor.limit !== undefined) {
|
||||
if (!Number.isInteger(cursor.limit)) {
|
||||
throw new InputError('Malformed after cursor, limit was not an number');
|
||||
}
|
||||
limit = cursor.limit;
|
||||
}
|
||||
if (cursor.offset !== undefined) {
|
||||
if (!Number.isInteger(cursor.offset)) {
|
||||
throw new InputError('Malformed after cursor, offset was not a number');
|
||||
}
|
||||
offset = cursor.offset;
|
||||
}
|
||||
}
|
||||
|
||||
return { limit, offset };
|
||||
}
|
||||
|
||||
function stringifyPagination(input: { limit: number; offset: number }) {
|
||||
const json = JSON.stringify({ limit: input.limit, offset: input.offset });
|
||||
const base64 = Buffer.from(json, 'utf8').toString('base64');
|
||||
return base64;
|
||||
}
|
||||
|
||||
function deduplicateRelations(
|
||||
rows: DbEntitiesRelationsRow[],
|
||||
): DbEntitiesRelationsRow[] {
|
||||
return lodash.uniqBy(
|
||||
rows,
|
||||
r => `${r.source_full_name}:${r.target_full_name}:${r.type}`,
|
||||
);
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common';
|
||||
import knexFactory, { Knex } from 'knex';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { Logger } from 'winston';
|
||||
import { CommonDatabase } from './CommonDatabase';
|
||||
import { Database } from './types';
|
||||
|
||||
const migrationsDir = resolvePackagePath(
|
||||
'@backstage/plugin-catalog-backend',
|
||||
'migrations',
|
||||
);
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type CreateDatabaseOptions = {
|
||||
logger: Logger;
|
||||
};
|
||||
|
||||
const defaultOptions: CreateDatabaseOptions = {
|
||||
logger: getVoidLogger(),
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export class DatabaseManager {
|
||||
public static async createDatabase(
|
||||
knex: Knex,
|
||||
options: Partial<CreateDatabaseOptions> = {},
|
||||
): Promise<Database> {
|
||||
await knex.migrate.latest({
|
||||
directory: migrationsDir,
|
||||
});
|
||||
const { logger } = { ...defaultOptions, ...options };
|
||||
return new CommonDatabase(knex, logger);
|
||||
}
|
||||
|
||||
public static async createInMemoryDatabase(): Promise<Database> {
|
||||
const knex = await this.createInMemoryDatabaseConnection();
|
||||
return await this.createDatabase(knex);
|
||||
}
|
||||
|
||||
public static async createInMemoryDatabaseConnection(): Promise<Knex> {
|
||||
const knex = knexFactory({
|
||||
client: 'sqlite3',
|
||||
connection: ':memory:',
|
||||
useNullAsDefault: true,
|
||||
});
|
||||
|
||||
knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
|
||||
resource.run('PRAGMA foreign_keys = ON', () => {});
|
||||
});
|
||||
|
||||
return knex;
|
||||
}
|
||||
|
||||
public static async createTestDatabase(): Promise<Database> {
|
||||
const knex = await this.createTestDatabaseConnection();
|
||||
return await this.createDatabase(knex);
|
||||
}
|
||||
|
||||
public static async createTestDatabaseConnection(): Promise<Knex> {
|
||||
const config: Knex.Config<any> = {
|
||||
/*
|
||||
client: 'pg',
|
||||
connection: {
|
||||
host: 'localhost',
|
||||
user: 'postgres',
|
||||
password: 'postgres',
|
||||
},
|
||||
*/
|
||||
client: 'sqlite3',
|
||||
connection: ':memory:',
|
||||
useNullAsDefault: true,
|
||||
};
|
||||
|
||||
let knex = knexFactory(config);
|
||||
if (typeof config.connection !== 'string') {
|
||||
const tempDbName = `d${uuidv4().replace(/-/g, '')}`;
|
||||
await knex.raw(`CREATE DATABASE ${tempDbName};`);
|
||||
knex = knexFactory({
|
||||
...config,
|
||||
connection: {
|
||||
...config.connection,
|
||||
database: tempDbName,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
|
||||
resource.run('PRAGMA foreign_keys = ON', () => {});
|
||||
});
|
||||
|
||||
return knex;
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { CommonDatabase } from './CommonDatabase';
|
||||
export { DatabaseManager } from './DatabaseManager';
|
||||
export type { CreateDatabaseOptions } from './DatabaseManager';
|
||||
export type {
|
||||
Database,
|
||||
DbEntityRequest,
|
||||
DbEntityResponse,
|
||||
Transaction,
|
||||
DbEntitiesRequest,
|
||||
DbEntitiesResponse,
|
||||
DbLocationsRowWithStatus,
|
||||
DatabaseLocationUpdateLogEvent,
|
||||
DbLocationsRow,
|
||||
DatabaseLocationUpdateLogStatus,
|
||||
DbPageInfo,
|
||||
} from './types';
|
||||
@@ -1,138 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
|
||||
import { buildEntitySearch, mapToRows, traverse } from './search';
|
||||
|
||||
describe('search', () => {
|
||||
describe('traverse', () => {
|
||||
it('expands lists of strings to several rows', () => {
|
||||
const input = { a: ['b', 'c', 'd'] };
|
||||
const output = traverse(input);
|
||||
expect(output).toEqual([
|
||||
{ key: 'a', value: 'b' },
|
||||
{ key: 'a.b', value: true },
|
||||
{ key: 'a', value: 'c' },
|
||||
{ key: 'a.c', value: true },
|
||||
{ key: 'a', value: 'd' },
|
||||
{ key: 'a.d', value: true },
|
||||
]);
|
||||
});
|
||||
|
||||
it('expands objects', () => {
|
||||
const input = { a: { b: { c: 'd' }, e: 'f' } };
|
||||
const output = traverse(input);
|
||||
expect(output).toEqual([
|
||||
{ key: 'a.b.c', value: 'd' },
|
||||
{ key: 'a.e', value: 'f' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('expands list of objects', () => {
|
||||
const input = { root: { list: [{ a: 1 }, { a: 2 }] } };
|
||||
const output = traverse(input);
|
||||
expect(output).toEqual([
|
||||
{ key: 'root.list.a', value: 1 },
|
||||
{ key: 'root.list.a', value: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('skips over special keys', () => {
|
||||
const input = {
|
||||
a: 'a',
|
||||
metadata: {
|
||||
b: 'b',
|
||||
name: 'name',
|
||||
namespace: 'namespace',
|
||||
uid: 'uid',
|
||||
etag: 'etag',
|
||||
generation: 'generation',
|
||||
c: 'c',
|
||||
},
|
||||
d: 'd',
|
||||
};
|
||||
const output = traverse(input);
|
||||
expect(output).toEqual([
|
||||
{ key: 'a', value: 'a' },
|
||||
{ key: 'metadata.b', value: 'b' },
|
||||
{ key: 'metadata.c', value: 'c' },
|
||||
{ key: 'd', value: 'd' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('mapToRows', () => {
|
||||
it('converts base types to strings or null', () => {
|
||||
const input = [
|
||||
{ key: 'a', value: true },
|
||||
{ key: 'b', value: false },
|
||||
{ key: 'c', value: 7 },
|
||||
{ key: 'd', value: 'string' },
|
||||
{ key: 'e', value: null },
|
||||
{ key: 'f', value: undefined },
|
||||
];
|
||||
const output = mapToRows(input, 'eid');
|
||||
expect(output).toEqual([
|
||||
{ entity_id: 'eid', key: 'a', value: 'true' },
|
||||
{ entity_id: 'eid', key: 'b', value: 'false' },
|
||||
{ entity_id: 'eid', key: 'c', value: '7' },
|
||||
{ entity_id: 'eid', key: 'd', value: 'string' },
|
||||
{ entity_id: 'eid', key: 'e', value: null },
|
||||
{ entity_id: 'eid', key: 'f', value: null },
|
||||
]);
|
||||
});
|
||||
|
||||
it('emits lowercase version of keys and values', () => {
|
||||
const input = [{ key: 'fOo', value: 'BaR' }];
|
||||
const output = mapToRows(input, 'eid');
|
||||
expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: 'bar' }]);
|
||||
});
|
||||
|
||||
it('skips very large keys', () => {
|
||||
const input = [{ key: 'a'.repeat(10000), value: 'foo' }];
|
||||
const output = mapToRows(input, 'eid');
|
||||
expect(output).toEqual([]);
|
||||
});
|
||||
|
||||
it('skips very large values', () => {
|
||||
const input = [{ key: 'foo', value: 'a'.repeat(10000) }];
|
||||
const output = mapToRows(input, 'eid');
|
||||
expect(output).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildEntitySearch', () => {
|
||||
it('adds special keys even if missing', () => {
|
||||
const input: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: { name: 'n' },
|
||||
};
|
||||
expect(buildEntitySearch('eid', input)).toEqual([
|
||||
{ entity_id: 'eid', key: 'apiversion', value: 'a' },
|
||||
{ entity_id: 'eid', key: 'kind', value: 'b' },
|
||||
{ entity_id: 'eid', key: 'metadata.name', value: 'n' },
|
||||
{ entity_id: 'eid', key: 'metadata.namespace', value: null },
|
||||
{ entity_id: 'eid', key: 'metadata.uid', value: null },
|
||||
{
|
||||
entity_id: 'eid',
|
||||
key: 'metadata.namespace',
|
||||
value: ENTITY_DEFAULT_NAMESPACE,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,176 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
|
||||
import type { DbEntitiesSearchRow } from './types';
|
||||
|
||||
// These are excluded in the generic loop, either because they do not make sense
|
||||
// to index, or because they are special-case always inserted whether they are
|
||||
// null or not
|
||||
const SPECIAL_KEYS = [
|
||||
'metadata.name',
|
||||
'metadata.namespace',
|
||||
'metadata.uid',
|
||||
'metadata.etag',
|
||||
'metadata.generation',
|
||||
];
|
||||
|
||||
// The maximum length allowed for search values. These columns are indexed, and
|
||||
// database engines do not like to index on massive values. For example,
|
||||
// postgres will balk after 8191 byte line sizes.
|
||||
const MAX_KEY_LENGTH = 200;
|
||||
const MAX_VALUE_LENGTH = 200;
|
||||
|
||||
type Kv = {
|
||||
key: string;
|
||||
value: unknown;
|
||||
};
|
||||
|
||||
// Helper for traversing through a nested structure and outputting a list of
|
||||
// path->value entries of the leaves.
|
||||
//
|
||||
// For example, this yaml structure
|
||||
//
|
||||
// a: 1
|
||||
// b:
|
||||
// c: null
|
||||
// e: [f, g]
|
||||
// h:
|
||||
// - i: 1
|
||||
// j: k
|
||||
// - i: 2
|
||||
// j: l
|
||||
//
|
||||
// will result in
|
||||
//
|
||||
// "a", 1
|
||||
// "b.c", null
|
||||
// "b.e": "f"
|
||||
// "b.e.f": true
|
||||
// "b.e": "g"
|
||||
// "b.e.g": true
|
||||
// "h.i": 1
|
||||
// "h.j": "k"
|
||||
// "h.i": 2
|
||||
// "h.j": "l"
|
||||
export function traverse(root: unknown): Kv[] {
|
||||
const output: Kv[] = [];
|
||||
|
||||
function visit(path: string, current: unknown) {
|
||||
if (SPECIAL_KEYS.includes(path)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// empty or scalar
|
||||
if (
|
||||
current === undefined ||
|
||||
current === null ||
|
||||
['string', 'number', 'boolean'].includes(typeof current)
|
||||
) {
|
||||
output.push({ key: path, value: current });
|
||||
return;
|
||||
}
|
||||
|
||||
// unknown
|
||||
if (typeof current !== 'object') {
|
||||
return;
|
||||
}
|
||||
|
||||
// array
|
||||
if (Array.isArray(current)) {
|
||||
for (const item of current) {
|
||||
// NOTE(freben): The reason that these are output in two different ways,
|
||||
// is to support use cases where you want to express that MORE than one
|
||||
// tag is present in a list. Since the EntityFilters structure is a
|
||||
// record, you can't have several entries of the same key. Therefore
|
||||
// you will have to match on
|
||||
//
|
||||
// { "a.b": ["true"], "a.c": ["true"] }
|
||||
//
|
||||
// rather than
|
||||
//
|
||||
// { "a": ["b", "c"] }
|
||||
//
|
||||
// because the latter means EITHER b or c has to be present.
|
||||
visit(path, item);
|
||||
if (typeof item === 'string') {
|
||||
output.push({ key: `${path}.${item}`, value: true });
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// object
|
||||
for (const [key, value] of Object.entries(current!)) {
|
||||
visit(path ? `${path}.${key}` : key, value);
|
||||
}
|
||||
}
|
||||
|
||||
visit('', root);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
// Translates a number of raw data rows to search table rows
|
||||
export function mapToRows(
|
||||
input: Kv[],
|
||||
entityId: string,
|
||||
): DbEntitiesSearchRow[] {
|
||||
const result: DbEntitiesSearchRow[] = [];
|
||||
|
||||
for (const { key: rawKey, value: rawValue } of input) {
|
||||
const key = rawKey.toLowerCase();
|
||||
if (rawValue === undefined || rawValue === null) {
|
||||
result.push({ entity_id: entityId, key, value: null });
|
||||
} else {
|
||||
const value = String(rawValue).toLowerCase();
|
||||
if (key.length <= MAX_KEY_LENGTH && value.length <= MAX_VALUE_LENGTH) {
|
||||
result.push({ entity_id: entityId, key, value });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates all of the search rows that are relevant for this entity.
|
||||
*
|
||||
* @param entityId - The uid of the entity
|
||||
* @param entity - The entity
|
||||
* @returns A list of entity search rows
|
||||
*/
|
||||
export function buildEntitySearch(
|
||||
entityId: string,
|
||||
entity: Entity,
|
||||
): DbEntitiesSearchRow[] {
|
||||
// Visit the entire structure recursively
|
||||
const raw = traverse(entity);
|
||||
|
||||
// Start with some special keys that are always present because you want to
|
||||
// be able to easily search for null specifically
|
||||
raw.push({ key: 'metadata.name', value: entity.metadata.name });
|
||||
raw.push({ key: 'metadata.namespace', value: entity.metadata.namespace });
|
||||
raw.push({ key: 'metadata.uid', value: entity.metadata.uid });
|
||||
|
||||
// Namespace not specified has the default value "default", so we want to
|
||||
// match on that as well
|
||||
if (!entity.metadata.namespace) {
|
||||
raw.push({ key: 'metadata.namespace', value: ENTITY_DEFAULT_NAMESPACE });
|
||||
}
|
||||
|
||||
return mapToRows(raw, entityId);
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import type {
|
||||
Entity,
|
||||
EntityName,
|
||||
EntityRelationSpec,
|
||||
Location,
|
||||
} from '@backstage/catalog-model';
|
||||
import { EntityFilter, EntityPagination } from '../../catalog/types';
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type DbEntitiesRow = {
|
||||
id: string;
|
||||
location_id: string | null;
|
||||
etag: string;
|
||||
generation: number;
|
||||
full_name: string;
|
||||
data: string;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type DbEntityRequest = {
|
||||
locationId?: string;
|
||||
entity: Entity;
|
||||
relations: EntityRelationSpec[];
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type DbEntitiesRequest = {
|
||||
filter?: EntityFilter;
|
||||
pagination?: EntityPagination;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type DbEntitiesResponse = {
|
||||
entities: DbEntityResponse[];
|
||||
pageInfo: DbPageInfo;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type DbPageInfo =
|
||||
| {
|
||||
hasNextPage: false;
|
||||
}
|
||||
| {
|
||||
hasNextPage: true;
|
||||
endCursor: string;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type DbEntityResponse = {
|
||||
locationId?: string;
|
||||
entity: Entity;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type DbEntitiesRelationsRow = {
|
||||
originating_entity_id: string;
|
||||
source_full_name: string;
|
||||
type: string;
|
||||
target_full_name: string;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type DbEntitiesSearchRow = {
|
||||
entity_id: string;
|
||||
key: string;
|
||||
value: string | null;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type DbLocationsRow = {
|
||||
id: string;
|
||||
type: string;
|
||||
target: string;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type DbLocationsRowWithStatus = DbLocationsRow & {
|
||||
status: string | null;
|
||||
timestamp: string | null;
|
||||
message: string | null;
|
||||
};
|
||||
|
||||
export enum DatabaseLocationUpdateLogStatus {
|
||||
FAIL = 'fail',
|
||||
SUCCESS = 'success',
|
||||
}
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type DatabaseLocationUpdateLogEvent = {
|
||||
id: string;
|
||||
status: DatabaseLocationUpdateLogStatus;
|
||||
location_id: string;
|
||||
entity_name: string;
|
||||
created_at?: string;
|
||||
message?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* An abstraction for transactions of the underlying database technology.
|
||||
*
|
||||
* @deprecated This was part of the legacy catalog engine
|
||||
*/
|
||||
export type Transaction = {
|
||||
rollback(): Promise<unknown>;
|
||||
};
|
||||
|
||||
/**
|
||||
* An abstraction on top of the underlying database, wrapping the basic CRUD
|
||||
* needs.
|
||||
* @deprecated This was part of the legacy catalog engine
|
||||
*/
|
||||
export type Database = {
|
||||
/**
|
||||
* Runs a transaction.
|
||||
*
|
||||
* The callback is expected to make calls back into this class. When it
|
||||
* completes, the transaction is closed.
|
||||
*
|
||||
* @param fn - The callback that implements the transaction
|
||||
*/
|
||||
transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T>;
|
||||
|
||||
/**
|
||||
* Adds a set of new entities to the catalog.
|
||||
*
|
||||
* @param tx - An ongoing transaction
|
||||
* @param request - The entities being added
|
||||
*/
|
||||
addEntities(
|
||||
tx: Transaction,
|
||||
request: DbEntityRequest[],
|
||||
): Promise<DbEntityResponse[]>;
|
||||
|
||||
/**
|
||||
* Updates an existing entity in the catalog.
|
||||
*
|
||||
* The given entity must contain an uid to identify an already stored entity
|
||||
* in the catalog. If it is missing or if no matching entity is found, the
|
||||
* operation fails.
|
||||
*
|
||||
* If matchingEtag or matchingGeneration are given, they are taken into
|
||||
* account. Attempts to update a matching entity, but where the etag and/or
|
||||
* generation are not equal to the passed values, will fail.
|
||||
*
|
||||
* @param tx - An ongoing transaction
|
||||
* @param request - The entity being updated
|
||||
* @param matchingEtag - If specified, reject with ConflictError if not
|
||||
* matching the entry in the database
|
||||
* @param matchingGeneration - If specified, reject with ConflictError if not
|
||||
* matching the entry in the database
|
||||
* @returns The updated entity
|
||||
*/
|
||||
updateEntity(
|
||||
tx: Transaction,
|
||||
request: DbEntityRequest,
|
||||
matchingEtag?: string,
|
||||
matchingGeneration?: number,
|
||||
): Promise<DbEntityResponse>;
|
||||
|
||||
entities(
|
||||
tx: Transaction,
|
||||
request?: DbEntitiesRequest,
|
||||
): Promise<DbEntitiesResponse>;
|
||||
|
||||
entityByName(
|
||||
tx: Transaction,
|
||||
name: EntityName,
|
||||
): Promise<DbEntityResponse | undefined>;
|
||||
|
||||
entityByUid(
|
||||
tx: Transaction,
|
||||
uid: string,
|
||||
): Promise<DbEntityResponse | undefined>;
|
||||
|
||||
removeEntityByUid(tx: Transaction, uid: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Remove current relations for the entity and replace them with the new
|
||||
* relations array.
|
||||
*
|
||||
* @param tx - An ongoing transaction
|
||||
* @param entityUid - The entity uid
|
||||
* @param relations - The relationships to be set
|
||||
*/
|
||||
setRelations(
|
||||
tx: Transaction,
|
||||
entityUid: string,
|
||||
relations: EntityRelationSpec[],
|
||||
): Promise<void>;
|
||||
|
||||
addLocation(tx: Transaction, location: Location): Promise<DbLocationsRow>;
|
||||
|
||||
removeLocation(tx: Transaction, id: string): Promise<void>;
|
||||
|
||||
location(id: string): Promise<DbLocationsRowWithStatus>;
|
||||
|
||||
locations(): Promise<DbLocationsRowWithStatus[]>;
|
||||
|
||||
locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]>;
|
||||
|
||||
addLocationUpdateLogEvent(
|
||||
locationId: string,
|
||||
status: DatabaseLocationUpdateLogStatus,
|
||||
entityName?: string | string[],
|
||||
message?: string,
|
||||
): Promise<void>;
|
||||
};
|
||||
@@ -1,20 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export * from './catalog';
|
||||
export * from './ingestion';
|
||||
export * from './service';
|
||||
export * from './database';
|
||||
@@ -1,415 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { Entity, Location, LocationSpec } from '@backstage/catalog-model';
|
||||
import { EntitiesCatalog } from '../../catalog';
|
||||
import { LocationsCatalog } from '../catalog';
|
||||
import { LocationUpdateStatus } from '../catalog/types';
|
||||
import { DatabaseLocationUpdateLogStatus } from '../database/types';
|
||||
import { HigherOrderOperations } from './HigherOrderOperations';
|
||||
import { LocationReader } from './types';
|
||||
|
||||
describe('HigherOrderOperations', () => {
|
||||
let entitiesCatalog: jest.Mocked<Required<EntitiesCatalog>>;
|
||||
let locationsCatalog: jest.Mocked<LocationsCatalog>;
|
||||
let locationReader: jest.Mocked<LocationReader>;
|
||||
let higherOrderOperation: HigherOrderOperations;
|
||||
|
||||
beforeAll(() => {
|
||||
entitiesCatalog = {
|
||||
entities: jest.fn(),
|
||||
removeEntityByUid: jest.fn(),
|
||||
batchAddOrUpdateEntities: jest.fn(),
|
||||
entityAncestry: jest.fn(),
|
||||
};
|
||||
locationsCatalog = {
|
||||
addLocation: jest.fn(),
|
||||
removeLocation: jest.fn(),
|
||||
locations: jest.fn(),
|
||||
location: jest.fn(),
|
||||
locationHistory: jest.fn(),
|
||||
logUpdateSuccess: jest.fn(),
|
||||
logUpdateFailure: jest.fn(),
|
||||
};
|
||||
locationReader = {
|
||||
read: jest.fn(),
|
||||
};
|
||||
higherOrderOperation = new HigherOrderOperations(
|
||||
entitiesCatalog,
|
||||
locationsCatalog,
|
||||
locationReader,
|
||||
getVoidLogger(),
|
||||
);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
describe('addLocation', () => {
|
||||
it('just inserts the location when there are no entities to read', async () => {
|
||||
const spec = {
|
||||
type: 'a',
|
||||
target: 'b',
|
||||
};
|
||||
locationsCatalog.addLocation.mockImplementation(x => Promise.resolve(x));
|
||||
locationsCatalog.locations.mockResolvedValue([]);
|
||||
locationReader.read.mockResolvedValue({
|
||||
entities: [],
|
||||
errors: [],
|
||||
});
|
||||
|
||||
const result = await higherOrderOperation.addLocation(spec);
|
||||
|
||||
expect(result.location).toEqual(
|
||||
expect.objectContaining({
|
||||
id: expect.anything(),
|
||||
...spec,
|
||||
}),
|
||||
);
|
||||
expect(result.entities).toEqual([]);
|
||||
expect(locationsCatalog.locations).toBeCalledTimes(1);
|
||||
expect(locationReader.read).toBeCalledTimes(1);
|
||||
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled();
|
||||
expect(locationsCatalog.addLocation).toBeCalledTimes(1);
|
||||
expect(locationsCatalog.addLocation).toBeCalledWith(
|
||||
expect.objectContaining({
|
||||
id: expect.anything(),
|
||||
...spec,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('insert the location and its entities', async () => {
|
||||
const spec = {
|
||||
type: 'a',
|
||||
target: 'b',
|
||||
};
|
||||
const location: LocationSpec = { type: '', target: '' };
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: { name: 'n' },
|
||||
};
|
||||
locationsCatalog.addLocation.mockImplementation(x => Promise.resolve(x));
|
||||
locationsCatalog.locations.mockResolvedValue([]);
|
||||
locationsCatalog.locations.mockResolvedValue([]);
|
||||
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
|
||||
{
|
||||
entityId: 'id',
|
||||
entity,
|
||||
},
|
||||
]);
|
||||
|
||||
locationReader.read.mockResolvedValue({
|
||||
entities: [
|
||||
{
|
||||
location,
|
||||
entity,
|
||||
relations: [],
|
||||
},
|
||||
],
|
||||
errors: [],
|
||||
});
|
||||
|
||||
const result = await higherOrderOperation.addLocation(spec);
|
||||
|
||||
expect(result.location).toEqual(
|
||||
expect.objectContaining({
|
||||
id: expect.anything(),
|
||||
...spec,
|
||||
}),
|
||||
);
|
||||
expect(result.entities).toEqual([entity]);
|
||||
expect(locationsCatalog.locations).toBeCalledTimes(1);
|
||||
expect(locationsCatalog.addLocation).toBeCalledTimes(1);
|
||||
expect(locationsCatalog.addLocation).toBeCalledWith(
|
||||
expect.objectContaining({
|
||||
id: expect.anything(),
|
||||
...spec,
|
||||
}),
|
||||
);
|
||||
expect(locationReader.read).toBeCalledTimes(1);
|
||||
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledTimes(1);
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
locationId: expect.anything(),
|
||||
dryRun: false,
|
||||
outputEntities: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('reuses the location if a match already existed', async () => {
|
||||
const spec = {
|
||||
type: 'a',
|
||||
target: 'b',
|
||||
};
|
||||
const location = {
|
||||
id: 'dd12620d-0436-422f-93bd-929aa0788123',
|
||||
...spec,
|
||||
};
|
||||
|
||||
locationsCatalog.locations.mockResolvedValue([
|
||||
{
|
||||
currentStatus: { timestamp: '', status: '', message: '' },
|
||||
data: location,
|
||||
},
|
||||
]);
|
||||
locationReader.read.mockResolvedValue({
|
||||
entities: [],
|
||||
errors: [],
|
||||
});
|
||||
|
||||
const result = await higherOrderOperation.addLocation(spec);
|
||||
|
||||
expect(result.location).toEqual(location);
|
||||
expect(result.entities).toEqual([]);
|
||||
expect(locationsCatalog.locations).toBeCalledTimes(1);
|
||||
expect(locationReader.read).toBeCalledTimes(1);
|
||||
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled();
|
||||
expect(locationsCatalog.addLocation).not.toBeCalled();
|
||||
});
|
||||
|
||||
it('rejects the whole operation if any entity could not be read', async () => {
|
||||
const spec = {
|
||||
type: 'a',
|
||||
target: 'b',
|
||||
};
|
||||
const location: LocationSpec = { type: '', target: '' };
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: { name: 'n' },
|
||||
};
|
||||
|
||||
locationsCatalog.locations.mockResolvedValue([]);
|
||||
locationReader.read.mockResolvedValue({
|
||||
entities: [{ entity, location, relations: [] }],
|
||||
errors: [{ error: new Error('abcd'), location }],
|
||||
});
|
||||
|
||||
await expect(higherOrderOperation.addLocation(spec)).rejects.toThrow(
|
||||
/abcd/,
|
||||
);
|
||||
expect(locationsCatalog.locations).toBeCalledTimes(1);
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled();
|
||||
expect(locationsCatalog.addLocation).not.toBeCalled();
|
||||
});
|
||||
|
||||
it('rollback everything after a dry run', async () => {
|
||||
const spec = {
|
||||
type: 'a',
|
||||
target: 'b',
|
||||
};
|
||||
const location: LocationSpec = { type: '', target: '' };
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: { name: 'n' },
|
||||
};
|
||||
locationsCatalog.locations.mockResolvedValue([]);
|
||||
locationsCatalog.locations.mockResolvedValue([]);
|
||||
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
|
||||
{
|
||||
entityId: 'id',
|
||||
entity,
|
||||
},
|
||||
]);
|
||||
locationReader.read.mockResolvedValue({
|
||||
entities: [
|
||||
{
|
||||
location,
|
||||
entity,
|
||||
relations: [],
|
||||
},
|
||||
],
|
||||
errors: [],
|
||||
});
|
||||
|
||||
const result = await higherOrderOperation.addLocation(spec, {
|
||||
dryRun: true,
|
||||
});
|
||||
|
||||
expect(result.location).toEqual(
|
||||
expect.objectContaining({
|
||||
id: expect.anything(),
|
||||
...spec,
|
||||
}),
|
||||
);
|
||||
expect(result.entities).toEqual([entity]);
|
||||
expect(locationsCatalog.locations).toBeCalledTimes(1);
|
||||
expect(locationReader.read).toBeCalledTimes(1);
|
||||
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledTimes(1);
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
dryRun: true,
|
||||
outputEntities: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('refreshLocations', () => {
|
||||
it('works with no locations added', async () => {
|
||||
locationsCatalog.locations.mockResolvedValue([]);
|
||||
|
||||
await expect(
|
||||
higherOrderOperation.refreshAllLocations(),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(locationsCatalog.locations).toHaveBeenCalledTimes(1);
|
||||
expect(locationReader.read).not.toHaveBeenCalled();
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('can update a single location where a matching entity did not exist', async () => {
|
||||
const locationStatus: LocationUpdateStatus = {
|
||||
message: '',
|
||||
status: DatabaseLocationUpdateLogStatus.SUCCESS,
|
||||
timestamp: new Date(314159265).toISOString(),
|
||||
};
|
||||
const location: Location = {
|
||||
id: '123',
|
||||
type: 'some',
|
||||
target: 'thing',
|
||||
};
|
||||
const desc: Entity = {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
metadata: { name: 'c1' },
|
||||
spec: { type: 'service' },
|
||||
};
|
||||
const entityId = 'xyz123';
|
||||
|
||||
locationsCatalog.locations.mockResolvedValue([
|
||||
{ currentStatus: locationStatus, data: location },
|
||||
]);
|
||||
locationReader.read.mockResolvedValue({
|
||||
entities: [{ entity: desc, location, relations: [] }],
|
||||
errors: [],
|
||||
});
|
||||
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
|
||||
{ entityId },
|
||||
]);
|
||||
|
||||
await expect(
|
||||
higherOrderOperation.refreshAllLocations(),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(locationsCatalog.locations).toHaveBeenCalledTimes(1);
|
||||
expect(locationReader.read).toHaveBeenCalledTimes(1);
|
||||
expect(locationReader.read).toHaveBeenNthCalledWith(1, {
|
||||
type: 'some',
|
||||
target: 'thing',
|
||||
});
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledWith(
|
||||
[
|
||||
expect.objectContaining({
|
||||
entity: expect.objectContaining({ metadata: { name: 'c1' } }),
|
||||
relations: [],
|
||||
}),
|
||||
],
|
||||
{
|
||||
locationId: '123',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('logs successful updates', async () => {
|
||||
const locationStatus: LocationUpdateStatus = {
|
||||
message: '',
|
||||
status: DatabaseLocationUpdateLogStatus.SUCCESS,
|
||||
timestamp: new Date(314159265).toISOString(),
|
||||
};
|
||||
const location: Location = {
|
||||
id: '123',
|
||||
type: 'some',
|
||||
target: 'thing',
|
||||
};
|
||||
const desc: Entity = {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
metadata: { name: 'c1' },
|
||||
spec: { type: 'service' },
|
||||
};
|
||||
|
||||
locationsCatalog.locations.mockResolvedValue([
|
||||
{ currentStatus: locationStatus, data: location },
|
||||
]);
|
||||
locationReader.read.mockResolvedValue({
|
||||
entities: [{ entity: desc, location, relations: [] }],
|
||||
errors: [],
|
||||
});
|
||||
entitiesCatalog.entities.mockResolvedValue({
|
||||
entities: [],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([]);
|
||||
|
||||
await expect(
|
||||
higherOrderOperation.refreshAllLocations(),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledTimes(2);
|
||||
expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith(
|
||||
'123',
|
||||
undefined,
|
||||
);
|
||||
expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith('123', [
|
||||
'c1',
|
||||
]);
|
||||
});
|
||||
|
||||
it('logs unsuccessful updates when reader fails', async () => {
|
||||
const locationStatus: LocationUpdateStatus = {
|
||||
message: '',
|
||||
status: DatabaseLocationUpdateLogStatus.SUCCESS,
|
||||
timestamp: new Date(314159265).toISOString(),
|
||||
};
|
||||
const location: Location = {
|
||||
id: '123',
|
||||
type: 'some',
|
||||
target: 'thing',
|
||||
};
|
||||
|
||||
locationsCatalog.locations.mockResolvedValue([
|
||||
{ currentStatus: locationStatus, data: location },
|
||||
]);
|
||||
locationReader.read.mockRejectedValue(new Error('reader error message'));
|
||||
|
||||
await expect(
|
||||
higherOrderOperation.refreshAllLocations(),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(locationReader.read).toHaveBeenCalledTimes(1);
|
||||
expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledTimes(1);
|
||||
expect(locationsCatalog.logUpdateSuccess).not.toHaveBeenCalled();
|
||||
expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledWith(
|
||||
'123',
|
||||
expect.objectContaining({ message: 'reader error message' }),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,218 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
Location,
|
||||
LocationSpec,
|
||||
stringifyLocationReference,
|
||||
} from '@backstage/catalog-model';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { Logger } from 'winston';
|
||||
import { EntitiesCatalog } from '../../catalog';
|
||||
import { LocationsCatalog } from '../catalog';
|
||||
import { durationText } from '../../util';
|
||||
import {
|
||||
AddLocationResult,
|
||||
HigherOrderOperation,
|
||||
LocationReader,
|
||||
} from './types';
|
||||
|
||||
/**
|
||||
* Placeholder for operations that span several catalogs and/or stretches out
|
||||
* in time.
|
||||
*
|
||||
* @deprecated This was part of the legacy catalog engine
|
||||
*/
|
||||
export class HigherOrderOperations implements HigherOrderOperation {
|
||||
constructor(
|
||||
private readonly entitiesCatalog: EntitiesCatalog,
|
||||
private readonly locationsCatalog: LocationsCatalog,
|
||||
private readonly locationReader: LocationReader,
|
||||
private readonly logger: Logger,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Adds a single location to the catalog.
|
||||
*
|
||||
* The location is inspected and fetched, and all of the resulting data is
|
||||
* validated. If everything goes well, the location and entities are stored
|
||||
* in the catalog.
|
||||
*
|
||||
* If the location already existed, the old location is returned instead and
|
||||
* the catalog is left unchanged.
|
||||
*
|
||||
* @param spec - The location to add
|
||||
*/
|
||||
async addLocation(
|
||||
spec: LocationSpec,
|
||||
options?: { dryRun?: boolean },
|
||||
): Promise<AddLocationResult> {
|
||||
const dryRun = options?.dryRun || false;
|
||||
|
||||
// Attempt to find a previous location matching the spec
|
||||
const previousLocations = await this.locationsCatalog.locations();
|
||||
const previousLocation = previousLocations.find(
|
||||
l => spec.type === l.data.type && spec.target === l.data.target,
|
||||
);
|
||||
const location: Location = previousLocation
|
||||
? previousLocation.data
|
||||
: {
|
||||
id: uuidv4(),
|
||||
type: spec.type,
|
||||
target: spec.target,
|
||||
};
|
||||
|
||||
// Read the location fully, bailing on any errors
|
||||
const readerOutput = await this.locationReader.read(spec);
|
||||
if (!(spec.presence === 'optional') && readerOutput.errors.length) {
|
||||
const item = readerOutput.errors[0];
|
||||
throw item.error;
|
||||
}
|
||||
|
||||
// TODO(freben): At this point, we could detect orphaned entities, by way
|
||||
// of having a location annotation pointing to the location but not being
|
||||
// in the entities list. But we aren't sure what to do about those yet.
|
||||
|
||||
// Write
|
||||
if (!previousLocation && !dryRun) {
|
||||
// TODO: We do not include location operations in the dryRun. We might perform
|
||||
// this operation as a separate dry run.
|
||||
await this.locationsCatalog.addLocation(location);
|
||||
}
|
||||
if (readerOutput.entities.length === 0) {
|
||||
return { location, entities: [] };
|
||||
}
|
||||
|
||||
const writtenEntities = await this.entitiesCatalog
|
||||
.batchAddOrUpdateEntities!(readerOutput.entities, {
|
||||
locationId: dryRun ? undefined : location.id,
|
||||
dryRun,
|
||||
outputEntities: true,
|
||||
});
|
||||
|
||||
const entities = writtenEntities.map(e => e.entity!);
|
||||
|
||||
return { location, entities };
|
||||
}
|
||||
|
||||
/**
|
||||
* Goes through all registered locations, and performs a refresh of each one.
|
||||
*
|
||||
* Entities are read from their respective sources, are parsed and validated
|
||||
* according to the entity policy, and get inserted or updated in the catalog.
|
||||
* Entities that have disappeared from their location are left orphaned,
|
||||
* without changes.
|
||||
*/
|
||||
async refreshAllLocations(): Promise<void> {
|
||||
const startTimestamp = process.hrtime();
|
||||
const logger = this.logger.child({
|
||||
component: 'catalog-all-locations-refresh',
|
||||
});
|
||||
|
||||
logger.info('Locations Refresh: Beginning locations refresh');
|
||||
|
||||
const locations = await this.locationsCatalog.locations();
|
||||
logger.info(`Locations Refresh: Visiting ${locations.length} locations`);
|
||||
|
||||
for (const { data: location } of locations) {
|
||||
logger.info(
|
||||
`Locations Refresh: Refreshing location ${stringifyLocationReference(
|
||||
location,
|
||||
)}`,
|
||||
);
|
||||
try {
|
||||
await this.refreshSingleLocation(location, logger);
|
||||
await this.locationsCatalog.logUpdateSuccess(location.id, undefined);
|
||||
} catch (e) {
|
||||
logger.warn(
|
||||
`Locations Refresh: Failed to refresh location ${stringifyLocationReference(
|
||||
location,
|
||||
)}, ${e.stack}`,
|
||||
);
|
||||
await this.locationsCatalog.logUpdateFailure(location.id, e);
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Locations Refresh: Completed locations refresh in ${durationText(
|
||||
startTimestamp,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Performs a full refresh of a single location
|
||||
private async refreshSingleLocation(
|
||||
location: Location,
|
||||
optionalLogger?: Logger,
|
||||
) {
|
||||
let startTimestamp = process.hrtime();
|
||||
const logger = optionalLogger || this.logger;
|
||||
|
||||
const readerOutput = await this.locationReader.read({
|
||||
type: location.type,
|
||||
target: location.target,
|
||||
});
|
||||
|
||||
for (const item of readerOutput.errors) {
|
||||
logger.warn(
|
||||
`Failed item in location ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}, ${item.error.stack}`,
|
||||
);
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Read ${
|
||||
readerOutput.entities.length
|
||||
} entities from location ${stringifyLocationReference(
|
||||
location,
|
||||
)} in ${durationText(startTimestamp)}`,
|
||||
);
|
||||
|
||||
startTimestamp = process.hrtime();
|
||||
|
||||
try {
|
||||
await this.entitiesCatalog.batchAddOrUpdateEntities!(
|
||||
readerOutput.entities,
|
||||
{ locationId: location.id },
|
||||
);
|
||||
} catch (e) {
|
||||
for (const entity of readerOutput.entities) {
|
||||
await this.locationsCatalog.logUpdateFailure(
|
||||
location.id,
|
||||
e,
|
||||
entity.entity.metadata.name,
|
||||
);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
logger.debug(`Posting update success markers`);
|
||||
|
||||
await this.locationsCatalog.logUpdateSuccess(
|
||||
location.id,
|
||||
readerOutput.entities.map(e => e.entity.metadata.name),
|
||||
);
|
||||
|
||||
logger.info(
|
||||
`Wrote ${
|
||||
readerOutput.entities.length
|
||||
} entities from location ${stringifyLocationReference(
|
||||
location,
|
||||
)} in ${durationText(startTimestamp)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,353 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { NotAllowedError } from '@backstage/errors';
|
||||
import { UrlReader } from '@backstage/backend-common';
|
||||
import {
|
||||
Entity,
|
||||
EntityPolicy,
|
||||
EntityRelationSpec,
|
||||
ENTITY_DEFAULT_NAMESPACE,
|
||||
LocationSpec,
|
||||
stringifyLocationReference,
|
||||
} from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import { Logger } from 'winston';
|
||||
import { CatalogRulesEnforcer } from '../../ingestion/CatalogRules';
|
||||
import * as result from '../../ingestion/processors/results';
|
||||
import {
|
||||
CatalogProcessor,
|
||||
CatalogProcessorEmit,
|
||||
CatalogProcessorEntityResult,
|
||||
CatalogProcessorErrorResult,
|
||||
CatalogProcessorLocationResult,
|
||||
CatalogProcessorParser,
|
||||
CatalogProcessorResult,
|
||||
} from '../../ingestion/processors/types';
|
||||
import { LocationReader, ReadLocationResult } from './types';
|
||||
|
||||
// The max amount of nesting depth of generated work items
|
||||
const MAX_DEPTH = 10;
|
||||
|
||||
type Options = {
|
||||
reader: UrlReader;
|
||||
parser: CatalogProcessorParser;
|
||||
logger: Logger;
|
||||
config: Config;
|
||||
processors: CatalogProcessor[];
|
||||
rulesEnforcer: CatalogRulesEnforcer;
|
||||
policy: EntityPolicy;
|
||||
};
|
||||
|
||||
const noopCache = {
|
||||
async get() {
|
||||
return undefined;
|
||||
},
|
||||
async set() {},
|
||||
};
|
||||
|
||||
/**
|
||||
* Implements the reading of a location through a series of processor tasks.
|
||||
*
|
||||
* @deprecated This was part of the legacy catalog engine
|
||||
*/
|
||||
export class LocationReaders implements LocationReader {
|
||||
private readonly options: Options;
|
||||
|
||||
constructor(options: Options) {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
async read(location: LocationSpec): Promise<ReadLocationResult> {
|
||||
const { rulesEnforcer, logger } = this.options;
|
||||
|
||||
const output: ReadLocationResult = {
|
||||
entities: [],
|
||||
errors: [],
|
||||
};
|
||||
let items: CatalogProcessorResult[] = [result.location(location, false)];
|
||||
|
||||
for (let depth = 0; depth < MAX_DEPTH; ++depth) {
|
||||
const newItems: CatalogProcessorResult[] = [];
|
||||
const emit: CatalogProcessorEmit = i => newItems.push(i);
|
||||
|
||||
for (const item of items) {
|
||||
if (item.type === 'location') {
|
||||
await this.handleLocation(item, emit);
|
||||
} else if (item.type === 'entity') {
|
||||
if (rulesEnforcer.isAllowed(item.entity, item.location)) {
|
||||
const relations = Array<EntityRelationSpec>();
|
||||
|
||||
const entity = await this.handleEntity(
|
||||
item,
|
||||
emitResult => {
|
||||
if (emitResult.type === 'relation') {
|
||||
relations.push(emitResult.relation);
|
||||
return;
|
||||
}
|
||||
emit(emitResult);
|
||||
},
|
||||
location,
|
||||
);
|
||||
|
||||
if (entity) {
|
||||
output.entities.push({
|
||||
entity,
|
||||
location: item.location,
|
||||
relations,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
output.errors.push({
|
||||
location: item.location,
|
||||
error: new NotAllowedError(
|
||||
`Entity of kind ${
|
||||
item.entity.kind
|
||||
} is not allowed from location ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}`,
|
||||
),
|
||||
});
|
||||
}
|
||||
} else if (item.type === 'error') {
|
||||
await this.handleError(item, emit);
|
||||
output.errors.push({
|
||||
location: item.location,
|
||||
error: item.error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (newItems.length === 0) {
|
||||
return output;
|
||||
}
|
||||
|
||||
items = newItems;
|
||||
}
|
||||
|
||||
const message = `Max recursion depth ${MAX_DEPTH} reached for location ${location.type} ${location.target}`;
|
||||
logger.warn(message);
|
||||
output.errors.push({ location, error: new Error(message) });
|
||||
return output;
|
||||
}
|
||||
|
||||
private async handleLocation(
|
||||
item: CatalogProcessorLocationResult,
|
||||
emit: CatalogProcessorEmit,
|
||||
) {
|
||||
const { processors, logger } = this.options;
|
||||
|
||||
const validatedEmit: CatalogProcessorEmit = emitResult => {
|
||||
if (emitResult.type === 'relation') {
|
||||
throw new Error('readLocation may not emit entity relations');
|
||||
}
|
||||
if (
|
||||
emitResult.type === 'location' &&
|
||||
emitResult.location.type === item.location.type &&
|
||||
emitResult.location.target === item.location.target
|
||||
) {
|
||||
// Ignore self-referential locations silently (this can happen for
|
||||
// example if you use a glob target like "**/*.yaml" in a Location
|
||||
// entity)
|
||||
return;
|
||||
}
|
||||
emit(emitResult);
|
||||
};
|
||||
|
||||
for (const processor of processors) {
|
||||
if (processor.readLocation) {
|
||||
try {
|
||||
if (
|
||||
await processor.readLocation(
|
||||
item.location,
|
||||
item.optional,
|
||||
validatedEmit,
|
||||
this.options.parser,
|
||||
noopCache,
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
} catch (e) {
|
||||
const message = `Processor ${
|
||||
processor.constructor.name
|
||||
} threw an error while reading location ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}, ${e}`;
|
||||
emit(result.generalError(item.location, message));
|
||||
logger.warn(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const message = `No processor was able to read location ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}`;
|
||||
emit(result.inputError(item.location, message));
|
||||
logger.warn(message);
|
||||
}
|
||||
|
||||
private async handleEntity(
|
||||
item: CatalogProcessorEntityResult,
|
||||
emit: CatalogProcessorEmit,
|
||||
originLocation: LocationSpec,
|
||||
): Promise<Entity | undefined> {
|
||||
const { processors, logger } = this.options;
|
||||
|
||||
let current = item.entity;
|
||||
|
||||
// Construct the name carefully, this happens before validation below
|
||||
// so we do not want to crash here due to missing metadata or so
|
||||
const kind = current.kind || '';
|
||||
const namespace = !current.metadata
|
||||
? ''
|
||||
: current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE;
|
||||
const name = !current.metadata ? '' : current.metadata.name;
|
||||
|
||||
for (const processor of processors) {
|
||||
if (processor.preProcessEntity) {
|
||||
try {
|
||||
current = await processor.preProcessEntity(
|
||||
current,
|
||||
item.location,
|
||||
emit,
|
||||
originLocation,
|
||||
noopCache,
|
||||
);
|
||||
} catch (e) {
|
||||
const message = `Processor ${
|
||||
processor.constructor.name
|
||||
} threw an error while preprocessing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}, ${e}`;
|
||||
emit(result.generalError(item.location, e.message));
|
||||
logger.warn(message);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const next = await this.options.policy.enforce(current);
|
||||
if (!next) {
|
||||
const message = `Policy unexpectedly returned no data while analyzing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}`;
|
||||
emit(result.generalError(item.location, message));
|
||||
logger.warn(message);
|
||||
return undefined;
|
||||
}
|
||||
current = next;
|
||||
} catch (e) {
|
||||
const message = `Policy check failed while analyzing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}, ${e}`;
|
||||
emit(result.inputError(item.location, message));
|
||||
logger.warn(message);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
let handled = false;
|
||||
for (const processor of processors) {
|
||||
if (processor.validateEntityKind) {
|
||||
try {
|
||||
handled = await processor.validateEntityKind(current);
|
||||
if (handled) {
|
||||
break;
|
||||
}
|
||||
} catch (e) {
|
||||
const message = `Processor ${
|
||||
processor.constructor.name
|
||||
} threw an error while validating the entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}, ${e}`;
|
||||
emit(result.inputError(item.location, message));
|
||||
logger.warn(message);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!handled) {
|
||||
const message = `No processor recognized the entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}`;
|
||||
emit(result.inputError(item.location, message));
|
||||
logger.warn(message);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
for (const processor of processors) {
|
||||
if (processor.postProcessEntity) {
|
||||
try {
|
||||
current = await processor.postProcessEntity(
|
||||
current,
|
||||
item.location,
|
||||
emit,
|
||||
noopCache,
|
||||
);
|
||||
} catch (e) {
|
||||
const message = `Processor ${
|
||||
processor.constructor.name
|
||||
} threw an error while postprocessing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}, ${e}`;
|
||||
emit(result.generalError(item.location, message));
|
||||
logger.warn(message);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
private async handleError(
|
||||
item: CatalogProcessorErrorResult,
|
||||
emit: CatalogProcessorEmit,
|
||||
) {
|
||||
const { processors, logger } = this.options;
|
||||
|
||||
logger.debug(
|
||||
`Encountered error at location ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}, ${item.error}`,
|
||||
);
|
||||
|
||||
const validatedEmit: CatalogProcessorEmit = emitResult => {
|
||||
if (emitResult.type === 'relation') {
|
||||
throw new Error('handleError may not emit entity relations');
|
||||
}
|
||||
|
||||
emit(emitResult);
|
||||
};
|
||||
|
||||
for (const processor of processors) {
|
||||
if (processor.handleError) {
|
||||
try {
|
||||
await processor.handleError(item.error, item.location, validatedEmit);
|
||||
} catch (e) {
|
||||
const message = `Processor ${
|
||||
processor.constructor.name
|
||||
} threw an error while handling another error at ${stringifyLocationReference(
|
||||
item.location,
|
||||
)}, ${e}`;
|
||||
emit(result.generalError(item.location, message));
|
||||
logger.warn(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { HigherOrderOperations } from './HigherOrderOperations';
|
||||
export { LocationReaders } from './LocationReaders';
|
||||
export type {
|
||||
AddLocationResult,
|
||||
HigherOrderOperation,
|
||||
LocationReader,
|
||||
ReadLocationEntity,
|
||||
ReadLocationError,
|
||||
ReadLocationResult,
|
||||
} from './types';
|
||||
@@ -1,76 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
Entity,
|
||||
EntityRelationSpec,
|
||||
Location,
|
||||
LocationSpec,
|
||||
} from '@backstage/catalog-model';
|
||||
|
||||
//
|
||||
// LocationReader
|
||||
//
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type HigherOrderOperation = {
|
||||
addLocation(
|
||||
spec: LocationSpec,
|
||||
options?: { dryRun?: boolean },
|
||||
): Promise<AddLocationResult>;
|
||||
refreshAllLocations(): Promise<void>;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type AddLocationResult = {
|
||||
location: Location;
|
||||
entities: Entity[];
|
||||
};
|
||||
|
||||
//
|
||||
// LocationReader
|
||||
//
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type LocationReader = {
|
||||
/**
|
||||
* Reads the contents of a location.
|
||||
*
|
||||
* @param location - The location to read
|
||||
* @throws An error if the location was handled by this reader, but could not
|
||||
* be read
|
||||
*/
|
||||
read(location: LocationSpec): Promise<ReadLocationResult>;
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type ReadLocationResult = {
|
||||
entities: ReadLocationEntity[];
|
||||
errors: ReadLocationError[];
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type ReadLocationEntity = {
|
||||
location: LocationSpec;
|
||||
entity: Entity;
|
||||
relations: EntityRelationSpec[];
|
||||
};
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export type ReadLocationError = {
|
||||
location: LocationSpec;
|
||||
error: Error;
|
||||
};
|
||||
@@ -1,260 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
getVoidLogger,
|
||||
PluginEndpointDiscovery,
|
||||
ServerTokenManager,
|
||||
UrlReader,
|
||||
} from '@backstage/backend-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { Knex } from 'knex';
|
||||
import yaml from 'yaml';
|
||||
import { DatabaseManager } from '../database';
|
||||
import { CatalogProcessorParser } from '../../ingestion';
|
||||
import * as result from '../../ingestion/processors/results';
|
||||
import { CatalogBuilder } from './CatalogBuilder';
|
||||
import { CatalogEnvironment } from '../../service';
|
||||
import { ServerPermissionClient } from '@backstage/plugin-permission-node';
|
||||
|
||||
const dummyEntity = {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
metadata: {
|
||||
name: 'n',
|
||||
},
|
||||
spec: {
|
||||
type: 't',
|
||||
owner: 'o',
|
||||
lifecycle: 'l',
|
||||
},
|
||||
};
|
||||
|
||||
const dummyEntityYaml = yaml.stringify(dummyEntity);
|
||||
|
||||
describe('CatalogBuilder', () => {
|
||||
let db: Knex<any, unknown[]>;
|
||||
const reader: jest.Mocked<UrlReader> = {
|
||||
read: jest.fn(),
|
||||
readTree: jest.fn(),
|
||||
search: jest.fn(),
|
||||
};
|
||||
const config = new ConfigReader({});
|
||||
const mockBaseUrl = 'http://backstage:9191/i-am-a-mock-base';
|
||||
const discovery: PluginEndpointDiscovery = {
|
||||
async getBaseUrl() {
|
||||
return mockBaseUrl;
|
||||
},
|
||||
async getExternalBaseUrl() {
|
||||
return mockBaseUrl;
|
||||
},
|
||||
};
|
||||
const env: CatalogEnvironment = {
|
||||
logger: getVoidLogger(),
|
||||
database: { getClient: async () => db },
|
||||
config,
|
||||
reader,
|
||||
permissions: ServerPermissionClient.fromConfig(config, {
|
||||
discovery,
|
||||
tokenManager: ServerTokenManager.noop(),
|
||||
}),
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
db = await DatabaseManager.createTestDatabaseConnection();
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
it('works with no changes', async () => {
|
||||
const builder = new CatalogBuilder(env);
|
||||
const built = await builder.build();
|
||||
await expect(built.entitiesCatalog.entities()).resolves.toEqual({
|
||||
entities: [],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
await expect(built.locationsCatalog.locations()).resolves.toEqual([
|
||||
expect.objectContaining({
|
||||
data: expect.objectContaining({ type: 'bootstrap' }),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('works with everything replaced', async () => {
|
||||
reader.read.mockResolvedValueOnce(Buffer.from('junk'));
|
||||
|
||||
const builder = new CatalogBuilder(env)
|
||||
.replaceEntityPolicies([
|
||||
{
|
||||
async enforce(entity: Entity) {
|
||||
expect(entity.metadata.namespace).toBe('ns');
|
||||
return entity;
|
||||
},
|
||||
},
|
||||
])
|
||||
.setPlaceholderResolver('t', async ({ value }) => {
|
||||
expect(value).toBe('tt');
|
||||
return 'tt2';
|
||||
})
|
||||
.setFieldFormatValidators({
|
||||
isValidEntityName: n => {
|
||||
expect(n).toBe('n');
|
||||
return true;
|
||||
},
|
||||
})
|
||||
.replaceProcessors([
|
||||
{
|
||||
async readLocation(location, _optional, emit) {
|
||||
expect(location.type).toBe('test');
|
||||
emit(
|
||||
result.entity(location, {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
metadata: { name: 'n', replaced: { $t: 'tt' } },
|
||||
spec: { type: 't', owner: 'o', lifecycle: 'l' },
|
||||
}),
|
||||
);
|
||||
return true;
|
||||
},
|
||||
async preProcessEntity(entity) {
|
||||
expect(entity.apiVersion).toBe('backstage.io/v1alpha1');
|
||||
return {
|
||||
...entity,
|
||||
metadata: { ...entity.metadata, namespace: 'ns' },
|
||||
};
|
||||
},
|
||||
async postProcessEntity(entity) {
|
||||
expect(entity.metadata.namespace).toBe('ns');
|
||||
return {
|
||||
...entity,
|
||||
metadata: { ...entity.metadata, post: 'p' },
|
||||
};
|
||||
},
|
||||
},
|
||||
]);
|
||||
const out = await builder.build();
|
||||
|
||||
const added = await out.higherOrderOperation.addLocation({
|
||||
type: 'test',
|
||||
target: '',
|
||||
});
|
||||
expect.assertions(6);
|
||||
expect(added.entities).toEqual([
|
||||
{
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
metadata: {
|
||||
name: 'n',
|
||||
namespace: 'ns',
|
||||
post: 'p',
|
||||
replaced: 'tt2',
|
||||
uid: expect.any(String),
|
||||
etag: expect.any(String),
|
||||
generation: expect.any(Number),
|
||||
},
|
||||
spec: {
|
||||
type: 't',
|
||||
owner: 'o',
|
||||
lifecycle: 'l',
|
||||
},
|
||||
relations: expect.anything(),
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('addProcessor works', async () => {
|
||||
reader.read.mockResolvedValueOnce(Buffer.from(dummyEntityYaml));
|
||||
|
||||
const builder = new CatalogBuilder(env);
|
||||
builder.addProcessor({
|
||||
async preProcessEntity(e) {
|
||||
return { ...e, metadata: { ...e.metadata, foo: 7 } };
|
||||
},
|
||||
});
|
||||
|
||||
const { entitiesCatalog, higherOrderOperation } = await builder.build();
|
||||
await higherOrderOperation.addLocation({
|
||||
type: 'url',
|
||||
target: 'https://github.com/a/b/x.yaml',
|
||||
});
|
||||
const { entities } = await entitiesCatalog.entities();
|
||||
|
||||
expect(entities).toEqual([
|
||||
expect.objectContaining({
|
||||
metadata: expect.objectContaining({
|
||||
foo: 7,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('replaceProcessors works', async () => {
|
||||
reader.read.mockResolvedValueOnce(Buffer.from(dummyEntityYaml));
|
||||
|
||||
const builder = new CatalogBuilder(env);
|
||||
builder.replaceProcessors([
|
||||
{
|
||||
async readLocation(location, _optional, emit) {
|
||||
expect(location.type).toBe('x');
|
||||
emit(result.entity(location, dummyEntity));
|
||||
return true;
|
||||
},
|
||||
async preProcessEntity(e) {
|
||||
expect(e.metadata.name).toBe('n');
|
||||
return { ...e, metadata: { ...e.metadata, foo: 7 } };
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const { entitiesCatalog, higherOrderOperation } = await builder.build();
|
||||
await higherOrderOperation.addLocation({
|
||||
type: 'x',
|
||||
target: 'y',
|
||||
});
|
||||
const { entities } = await entitiesCatalog.entities();
|
||||
|
||||
expect.assertions(3);
|
||||
expect(entities).toEqual([
|
||||
expect.objectContaining({
|
||||
metadata: expect.objectContaining({
|
||||
foo: 7,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('setEntityDataParser works', async () => {
|
||||
const mockParser: CatalogProcessorParser = jest
|
||||
.fn()
|
||||
.mockImplementation(() => {});
|
||||
|
||||
const builder = new CatalogBuilder(env)
|
||||
.setEntityDataParser(mockParser)
|
||||
.replaceProcessors([
|
||||
{
|
||||
async readLocation(_location, _optional, _emit, parser) {
|
||||
expect(parser).toBe(mockParser);
|
||||
return true;
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const { higherOrderOperation } = await builder.build();
|
||||
await higherOrderOperation.addLocation({ type: 'x', target: 'y' });
|
||||
|
||||
expect.assertions(1);
|
||||
});
|
||||
});
|
||||
@@ -1,373 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
DefaultNamespaceEntityPolicy,
|
||||
EntityPolicies,
|
||||
EntityPolicy,
|
||||
FieldFormatEntityPolicy,
|
||||
makeValidator,
|
||||
NoForeignRootFieldsEntityPolicy,
|
||||
SchemaValidEntityPolicy,
|
||||
Validators,
|
||||
} from '@backstage/catalog-model';
|
||||
import {
|
||||
ScmIntegrations,
|
||||
DefaultGithubCredentialsProvider,
|
||||
} from '@backstage/integration';
|
||||
import lodash from 'lodash';
|
||||
import { EntitiesCatalog } from '../../catalog';
|
||||
import {
|
||||
DatabaseEntitiesCatalog,
|
||||
DatabaseLocationsCatalog,
|
||||
LocationsCatalog,
|
||||
} from '../catalog';
|
||||
import { DatabaseManager } from '../database';
|
||||
import {
|
||||
AnnotateLocationEntityProcessor,
|
||||
BitbucketDiscoveryProcessor,
|
||||
BuiltinKindsEntityProcessor,
|
||||
CatalogProcessor,
|
||||
CatalogProcessorParser,
|
||||
CodeOwnersProcessor,
|
||||
FileReaderProcessor,
|
||||
GithubDiscoveryProcessor,
|
||||
AzureDevOpsDiscoveryProcessor,
|
||||
GithubOrgReaderProcessor,
|
||||
GitLabDiscoveryProcessor,
|
||||
LocationEntityProcessor,
|
||||
PlaceholderProcessor,
|
||||
PlaceholderResolver,
|
||||
StaticLocationProcessor,
|
||||
UrlReaderProcessor,
|
||||
} from '../../ingestion';
|
||||
import {
|
||||
HigherOrderOperation,
|
||||
HigherOrderOperations,
|
||||
LocationReaders,
|
||||
} from '../ingestion';
|
||||
import { DefaultCatalogRulesEnforcer } from '../../ingestion/CatalogRules';
|
||||
import { RepoLocationAnalyzer } from '../../ingestion/LocationAnalyzer';
|
||||
import {
|
||||
jsonPlaceholderResolver,
|
||||
textPlaceholderResolver,
|
||||
yamlPlaceholderResolver,
|
||||
} from '../../ingestion/processors/PlaceholderProcessor';
|
||||
import { defaultEntityDataParser } from '../../ingestion/processors/util/parse';
|
||||
import { LocationAnalyzer } from '../../ingestion/types';
|
||||
import { CatalogEnvironment } from '../../service';
|
||||
|
||||
/**
|
||||
* A builder that helps wire up all of the component parts of the catalog.
|
||||
*
|
||||
* The touch points where you can replace or extend behavior are as follows:
|
||||
*
|
||||
* - Entity policies can be added or replaced. These are automatically run
|
||||
* after the processors' pre-processing steps. All policies are given the
|
||||
* chance to inspect the entity, and all of them have to pass in order for
|
||||
* the entity to be considered valid from an overall point of view.
|
||||
* - Placeholder resolvers can be replaced or added. These run on the raw
|
||||
* structured data between the parsing and pre-processing steps, to replace
|
||||
* dollar-prefixed entries with their actual values (like $file).
|
||||
* - Field format validators can be replaced. These check the format of
|
||||
* individual core fields such as metadata.name, to ensure that they adhere
|
||||
* to certain rules.
|
||||
* - Processors can be added or replaced. These implement the functionality of
|
||||
* reading, parsing, validating, and processing the entity data before it is
|
||||
* persisted in the catalog.
|
||||
*
|
||||
* NOTE(freben): Not actually marking the class as deprecated formally, since
|
||||
* it would appear to end users that even using `create` is deprecated. We will
|
||||
* instead hot-swap the entire exported class when we are ready.
|
||||
*/
|
||||
export class CatalogBuilder {
|
||||
private readonly env: CatalogEnvironment;
|
||||
private entityPolicies: EntityPolicy[];
|
||||
private entityPoliciesReplace: boolean;
|
||||
private placeholderResolvers: Record<string, PlaceholderResolver>;
|
||||
private fieldFormatValidators: Partial<Validators>;
|
||||
private processors: CatalogProcessor[];
|
||||
private processorsReplace: boolean;
|
||||
private parser: CatalogProcessorParser | undefined;
|
||||
|
||||
static async create(env: CatalogEnvironment): Promise<CatalogBuilder> {
|
||||
return new CatalogBuilder(env);
|
||||
}
|
||||
|
||||
/** @deprecated Please use CatalogBuilder.create() instead */
|
||||
constructor(env: CatalogEnvironment) {
|
||||
this.env = env;
|
||||
this.entityPolicies = [];
|
||||
this.entityPoliciesReplace = false;
|
||||
this.placeholderResolvers = {};
|
||||
this.fieldFormatValidators = {};
|
||||
this.processors = [];
|
||||
this.processorsReplace = false;
|
||||
this.parser = undefined;
|
||||
|
||||
env.logger.warn(
|
||||
"Creating the catalog with 'new CatalogBuilder(env)' is deprecated! Use CatalogBuilder.create(env) instead",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds policies that are used to validate entities between the pre-
|
||||
* processing and post-processing stages. All such policies must pass for the
|
||||
* entity to be considered valid.
|
||||
*
|
||||
* If what you want to do is to replace the rules for what format is allowed
|
||||
* in various core entity fields (such as metadata.name), you may want to use
|
||||
* {@link CatalogBuilder#setFieldFormatValidators} instead.
|
||||
*
|
||||
* @param policies - One or more policies
|
||||
*/
|
||||
addEntityPolicy(...policies: EntityPolicy[]): CatalogBuilder {
|
||||
this.entityPolicies.push(...policies);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets what policies to use for validation of entities between the pre-
|
||||
* processing and post-processing stages. All such policies must pass for the
|
||||
* entity to be considered valid.
|
||||
*
|
||||
* If what you want to do is to replace the rules for what format is allowed
|
||||
* in various core entity fields (such as metadata.name), you may want to use
|
||||
* {@link CatalogBuilder#setFieldFormatValidators} instead.
|
||||
*
|
||||
* This function replaces the default set of policies; use with care.
|
||||
*
|
||||
* @param policies - One or more policies
|
||||
*/
|
||||
replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder {
|
||||
this.entityPolicies = [...policies];
|
||||
this.entityPoliciesReplace = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds, or overwrites, a handler for placeholders (e.g. $file) in entity
|
||||
* definition files.
|
||||
*
|
||||
* @param key - The key that identifies the placeholder, e.g. "file"
|
||||
* @param resolver - The resolver that gets values for this placeholder
|
||||
*/
|
||||
setPlaceholderResolver(
|
||||
key: string,
|
||||
resolver: PlaceholderResolver,
|
||||
): CatalogBuilder {
|
||||
this.placeholderResolvers[key] = resolver;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the validator function to use for one or more special fields of an
|
||||
* entity. This is useful if the default rules for formatting of fields are
|
||||
* not sufficient.
|
||||
*
|
||||
* This function has no effect if used together with
|
||||
* {@link CatalogBuilder#replaceEntityPolicies}.
|
||||
*
|
||||
* @param validators - The (subset of) validators to set
|
||||
*/
|
||||
setFieldFormatValidators(validators: Partial<Validators>): CatalogBuilder {
|
||||
lodash.merge(this.fieldFormatValidators, validators);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds entity processors. These are responsible for reading, parsing, and
|
||||
* processing entities before they are persisted in the catalog.
|
||||
*
|
||||
* @param processors - One or more processors
|
||||
*/
|
||||
addProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
|
||||
this.processors.push(...processors);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets what entity processors to use. These are responsible for reading,
|
||||
* parsing, and processing entities before they are persisted in the catalog.
|
||||
*
|
||||
* This function replaces the default set of processors; use with care.
|
||||
*
|
||||
* @param processors - One or more processors
|
||||
*/
|
||||
replaceProcessors(processors: CatalogProcessor[]): CatalogBuilder {
|
||||
this.processors = [...processors];
|
||||
this.processorsReplace = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets up the catalog to use a custom parser for entity data.
|
||||
*
|
||||
* This is the function that gets called immediately after some raw entity
|
||||
* specification data has been read from a remote source, and needs to be
|
||||
* parsed and emitted as structured data.
|
||||
*
|
||||
* @param parser - The custom parser
|
||||
*/
|
||||
setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder {
|
||||
this.parser = parser;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wires up and returns all of the component parts of the catalog
|
||||
*/
|
||||
async build(): Promise<{
|
||||
entitiesCatalog: EntitiesCatalog;
|
||||
locationsCatalog: LocationsCatalog;
|
||||
higherOrderOperation: HigherOrderOperation;
|
||||
locationAnalyzer: LocationAnalyzer;
|
||||
}> {
|
||||
const { config, database, logger } = this.env;
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
|
||||
const policy = this.buildEntityPolicy();
|
||||
const processors = this.buildProcessors();
|
||||
const rulesEnforcer = DefaultCatalogRulesEnforcer.fromConfig(config);
|
||||
const parser = this.parser || defaultEntityDataParser;
|
||||
|
||||
const locationReader = new LocationReaders({
|
||||
...this.env,
|
||||
parser,
|
||||
processors,
|
||||
rulesEnforcer,
|
||||
policy,
|
||||
});
|
||||
|
||||
const db = await DatabaseManager.createDatabase(
|
||||
await database.getClient(),
|
||||
{ logger },
|
||||
);
|
||||
|
||||
const entitiesCatalog = new DatabaseEntitiesCatalog(db, this.env.logger);
|
||||
const locationsCatalog = new DatabaseLocationsCatalog(db);
|
||||
const higherOrderOperation = new HigherOrderOperations(
|
||||
entitiesCatalog,
|
||||
locationsCatalog,
|
||||
locationReader,
|
||||
logger,
|
||||
);
|
||||
const locationAnalyzer = new RepoLocationAnalyzer(logger, integrations);
|
||||
|
||||
return {
|
||||
entitiesCatalog,
|
||||
locationsCatalog,
|
||||
higherOrderOperation,
|
||||
locationAnalyzer,
|
||||
};
|
||||
}
|
||||
|
||||
private buildEntityPolicy(): EntityPolicy {
|
||||
const entityPolicies: EntityPolicy[] = this.entityPoliciesReplace
|
||||
? [new SchemaValidEntityPolicy(), ...this.entityPolicies]
|
||||
: [
|
||||
new SchemaValidEntityPolicy(),
|
||||
new DefaultNamespaceEntityPolicy(),
|
||||
new NoForeignRootFieldsEntityPolicy(),
|
||||
new FieldFormatEntityPolicy(
|
||||
makeValidator(this.fieldFormatValidators),
|
||||
),
|
||||
...this.entityPolicies,
|
||||
];
|
||||
|
||||
return EntityPolicies.allOf(entityPolicies);
|
||||
}
|
||||
|
||||
private buildProcessors(): CatalogProcessor[] {
|
||||
const { config, logger, reader } = this.env;
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
const githubCredentialsProvider =
|
||||
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
|
||||
|
||||
this.checkDeprecatedReaderProcessors();
|
||||
|
||||
const placeholderResolvers: Record<string, PlaceholderResolver> = {
|
||||
json: jsonPlaceholderResolver,
|
||||
yaml: yamlPlaceholderResolver,
|
||||
text: textPlaceholderResolver,
|
||||
...this.placeholderResolvers,
|
||||
};
|
||||
|
||||
// These are always there no matter what
|
||||
const processors: CatalogProcessor[] = [
|
||||
StaticLocationProcessor.fromConfig(config),
|
||||
new PlaceholderProcessor({
|
||||
resolvers: placeholderResolvers,
|
||||
reader,
|
||||
integrations,
|
||||
}),
|
||||
new BuiltinKindsEntityProcessor(),
|
||||
];
|
||||
|
||||
// These are only added unless the user replaced them all
|
||||
if (!this.processorsReplace) {
|
||||
processors.push(
|
||||
new FileReaderProcessor(),
|
||||
BitbucketDiscoveryProcessor.fromConfig(config, { logger }),
|
||||
GithubDiscoveryProcessor.fromConfig(config, {
|
||||
logger,
|
||||
githubCredentialsProvider,
|
||||
}),
|
||||
AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }),
|
||||
GithubOrgReaderProcessor.fromConfig(config, {
|
||||
logger,
|
||||
githubCredentialsProvider,
|
||||
}),
|
||||
GitLabDiscoveryProcessor.fromConfig(config, { logger }),
|
||||
new UrlReaderProcessor({ reader, logger }),
|
||||
CodeOwnersProcessor.fromConfig(config, { logger, reader }),
|
||||
new LocationEntityProcessor({ integrations }),
|
||||
new AnnotateLocationEntityProcessor({ integrations }),
|
||||
);
|
||||
}
|
||||
|
||||
// Add the ones (if any) that the user added
|
||||
processors.push(...this.processors);
|
||||
|
||||
return processors;
|
||||
}
|
||||
|
||||
// TODO(Rugvip): These old processors are removed, for a while we'll be throwing
|
||||
// errors here to make sure people know where to move the config
|
||||
private checkDeprecatedReaderProcessors() {
|
||||
const pc = this.env.config.getOptionalConfig('catalog.processors');
|
||||
if (pc?.has('github')) {
|
||||
throw new Error(
|
||||
`Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`,
|
||||
);
|
||||
}
|
||||
if (pc?.has('gitlabApi')) {
|
||||
throw new Error(
|
||||
`Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`,
|
||||
);
|
||||
}
|
||||
if (pc?.has('bitbucketApi')) {
|
||||
throw new Error(
|
||||
`Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`,
|
||||
);
|
||||
}
|
||||
if (pc?.has('azureApi')) {
|
||||
throw new Error(
|
||||
`Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { CatalogBuilder } from './CatalogBuilder';
|
||||
export { createRouter } from './router';
|
||||
export type { RouterOptions } from './router';
|
||||
@@ -1,529 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { NotFoundError } from '@backstage/errors';
|
||||
import type { Entity, LocationSpec } from '@backstage/catalog-model';
|
||||
import express from 'express';
|
||||
import request from 'supertest';
|
||||
import { EntitiesCatalog } from '../../catalog';
|
||||
import { LocationResponse, LocationsCatalog } from '../catalog/types';
|
||||
import { HigherOrderOperation } from '../ingestion/types';
|
||||
import { createRouter } from './router';
|
||||
import { basicEntityFilter } from '../../service/request';
|
||||
import { RefreshService } from '../../service';
|
||||
|
||||
describe('createRouter readonly disabled', () => {
|
||||
let entitiesCatalog: jest.Mocked<Required<EntitiesCatalog>>;
|
||||
let locationsCatalog: jest.Mocked<LocationsCatalog>;
|
||||
let higherOrderOperation: jest.Mocked<HigherOrderOperation>;
|
||||
let app: express.Express;
|
||||
let refreshService: RefreshService;
|
||||
|
||||
beforeAll(async () => {
|
||||
entitiesCatalog = {
|
||||
entities: jest.fn(),
|
||||
removeEntityByUid: jest.fn(),
|
||||
batchAddOrUpdateEntities: jest.fn(),
|
||||
entityAncestry: jest.fn(),
|
||||
};
|
||||
locationsCatalog = {
|
||||
addLocation: jest.fn(),
|
||||
removeLocation: jest.fn(),
|
||||
locations: jest.fn(),
|
||||
location: jest.fn(),
|
||||
locationHistory: jest.fn(),
|
||||
logUpdateSuccess: jest.fn(),
|
||||
logUpdateFailure: jest.fn(),
|
||||
};
|
||||
higherOrderOperation = {
|
||||
addLocation: jest.fn(),
|
||||
refreshAllLocations: jest.fn(),
|
||||
};
|
||||
refreshService = { refresh: jest.fn() };
|
||||
const router = await createRouter({
|
||||
entitiesCatalog,
|
||||
locationsCatalog,
|
||||
higherOrderOperation,
|
||||
logger: getVoidLogger(),
|
||||
refreshService,
|
||||
config: new ConfigReader(undefined),
|
||||
});
|
||||
app = express().use(router);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
describe('POST /refresh', () => {
|
||||
it('refreshes an entity using the refresh service', async () => {
|
||||
const response = await request(app)
|
||||
.post('/refresh')
|
||||
.set('Content-Type', 'application/json')
|
||||
.send({ entityRef: 'Component/default:foo' });
|
||||
expect(response.status).toBe(200);
|
||||
expect(refreshService.refresh).toHaveBeenCalledWith({
|
||||
entityRef: 'Component/default:foo',
|
||||
});
|
||||
});
|
||||
});
|
||||
describe('GET /entities', () => {
|
||||
it('happy path: lists entities', async () => {
|
||||
const entities: Entity[] = [
|
||||
{ apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
|
||||
];
|
||||
|
||||
entitiesCatalog.entities.mockResolvedValueOnce({
|
||||
entities: [entities[0]],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
|
||||
const response = await request(app).get('/entities');
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual(entities);
|
||||
});
|
||||
|
||||
it('parses single and multiple request parameters and passes them down', async () => {
|
||||
entitiesCatalog.entities.mockResolvedValueOnce({
|
||||
entities: [],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
const response = await request(app).get(
|
||||
'/entities?filter=a=1,a=2,b=3&filter=c=4',
|
||||
);
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filter: {
|
||||
anyOf: [
|
||||
{
|
||||
allOf: [
|
||||
{ key: 'a', values: ['1', '2'] },
|
||||
{ key: 'b', values: ['3'] },
|
||||
],
|
||||
},
|
||||
{ allOf: [{ key: 'c', values: ['4'] }] },
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /entities/by-uid/:uid', () => {
|
||||
it('can fetch entity by uid', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
},
|
||||
};
|
||||
entitiesCatalog.entities.mockResolvedValue({
|
||||
entities: [entity],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
|
||||
const response = await request(app).get('/entities/by-uid/zzz');
|
||||
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filter: basicEntityFilter({ 'metadata.uid': 'zzz' }),
|
||||
});
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual(expect.objectContaining(entity));
|
||||
});
|
||||
|
||||
it('responds with a 404 for missing entities', async () => {
|
||||
entitiesCatalog.entities.mockResolvedValue({
|
||||
entities: [],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
|
||||
const response = await request(app).get('/entities/by-uid/zzz');
|
||||
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filter: basicEntityFilter({ 'metadata.uid': 'zzz' }),
|
||||
});
|
||||
expect(response.status).toEqual(404);
|
||||
expect(response.text).toMatch(/uid/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /entities/by-name/:kind/:namespace/:name', () => {
|
||||
it('can fetch entity by name', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'k',
|
||||
metadata: {
|
||||
name: 'n',
|
||||
namespace: 'ns',
|
||||
},
|
||||
};
|
||||
entitiesCatalog.entities.mockResolvedValue({
|
||||
entities: [entity],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
|
||||
const response = await request(app).get('/entities/by-name/k/ns/n');
|
||||
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filter: basicEntityFilter({
|
||||
kind: 'k',
|
||||
'metadata.namespace': 'ns',
|
||||
'metadata.name': 'n',
|
||||
}),
|
||||
});
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual(expect.objectContaining(entity));
|
||||
});
|
||||
|
||||
it('responds with a 404 for missing entities', async () => {
|
||||
entitiesCatalog.entities.mockResolvedValue({
|
||||
entities: [],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
|
||||
const response = await request(app).get('/entities/by-name/b/d/c');
|
||||
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filter: basicEntityFilter({
|
||||
kind: 'b',
|
||||
'metadata.namespace': 'd',
|
||||
'metadata.name': 'c',
|
||||
}),
|
||||
});
|
||||
expect(response.status).toEqual(404);
|
||||
expect(response.text).toMatch(/name/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /entities', () => {
|
||||
it('requires a body', async () => {
|
||||
const response = await request(app)
|
||||
.post('/entities')
|
||||
.set('Content-Type', 'application/json')
|
||||
.send();
|
||||
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toHaveBeenCalled();
|
||||
expect(response.status).toEqual(400);
|
||||
expect(response.text).toMatch(/body/);
|
||||
});
|
||||
|
||||
it('passes the body down', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
};
|
||||
|
||||
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
|
||||
{ entityId: 'u' },
|
||||
]);
|
||||
entitiesCatalog.entities.mockResolvedValue({
|
||||
entities: [entity],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.post('/entities')
|
||||
.send(entity)
|
||||
.set('Content-Type', 'application/json');
|
||||
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledWith([
|
||||
{ entity, relations: [] },
|
||||
]);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filter: basicEntityFilter({ 'metadata.uid': 'u' }),
|
||||
});
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual(entity);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DELETE /entities/by-uid/:uid', () => {
|
||||
it('can remove', async () => {
|
||||
entitiesCatalog.removeEntityByUid.mockResolvedValue(undefined);
|
||||
|
||||
const response = await request(app).delete('/entities/by-uid/apa');
|
||||
|
||||
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledWith('apa');
|
||||
expect(response.status).toEqual(204);
|
||||
});
|
||||
|
||||
it('responds with a 404 for missing entities', async () => {
|
||||
entitiesCatalog.removeEntityByUid.mockRejectedValue(
|
||||
new NotFoundError('nope'),
|
||||
);
|
||||
|
||||
const response = await request(app).delete('/entities/by-uid/apa');
|
||||
|
||||
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledWith('apa');
|
||||
expect(response.status).toEqual(404);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /locations', () => {
|
||||
it('happy path: lists locations', async () => {
|
||||
const locations: LocationResponse[] = [
|
||||
{
|
||||
currentStatus: { timestamp: '', status: '', message: '' },
|
||||
data: { id: 'a', type: 'b', target: 'c' },
|
||||
},
|
||||
];
|
||||
locationsCatalog.locations.mockResolvedValueOnce(locations);
|
||||
|
||||
const response = await request(app).get('/locations');
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual(locations);
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /locations', () => {
|
||||
it('rejects malformed locations', async () => {
|
||||
const spec = {
|
||||
typez: 'b',
|
||||
target: 'c',
|
||||
} as unknown as LocationSpec;
|
||||
|
||||
const response = await request(app).post('/locations').send(spec);
|
||||
|
||||
expect(higherOrderOperation.addLocation).not.toHaveBeenCalled();
|
||||
expect(response.status).toEqual(400);
|
||||
});
|
||||
|
||||
it('passes the body down', async () => {
|
||||
const spec: LocationSpec = {
|
||||
type: 'b',
|
||||
target: 'c',
|
||||
};
|
||||
|
||||
higherOrderOperation.addLocation.mockResolvedValue({
|
||||
location: { id: 'a', ...spec },
|
||||
entities: [],
|
||||
});
|
||||
|
||||
const response = await request(app).post('/locations').send(spec);
|
||||
|
||||
expect(higherOrderOperation.addLocation).toHaveBeenCalledTimes(1);
|
||||
expect(higherOrderOperation.addLocation).toHaveBeenCalledWith(spec, {
|
||||
dryRun: false,
|
||||
});
|
||||
expect(response.status).toEqual(201);
|
||||
expect(response.body).toEqual(
|
||||
expect.objectContaining({
|
||||
location: { id: 'a', ...spec },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('supports dry run', async () => {
|
||||
const spec: LocationSpec = {
|
||||
type: 'b',
|
||||
target: 'c',
|
||||
};
|
||||
|
||||
higherOrderOperation.addLocation.mockResolvedValue({
|
||||
location: { id: 'a', ...spec },
|
||||
entities: [],
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.post('/locations?dryRun=true')
|
||||
.send(spec);
|
||||
|
||||
expect(higherOrderOperation.addLocation).toHaveBeenCalledTimes(1);
|
||||
expect(higherOrderOperation.addLocation).toHaveBeenCalledWith(spec, {
|
||||
dryRun: true,
|
||||
});
|
||||
expect(response.status).toEqual(201);
|
||||
expect(response.body).toEqual(
|
||||
expect.objectContaining({
|
||||
location: { id: 'a', ...spec },
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('createRouter readonly enabled', () => {
|
||||
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
|
||||
let locationsCatalog: jest.Mocked<LocationsCatalog>;
|
||||
let higherOrderOperation: jest.Mocked<HigherOrderOperation>;
|
||||
let app: express.Express;
|
||||
|
||||
beforeAll(async () => {
|
||||
entitiesCatalog = {
|
||||
entities: jest.fn(),
|
||||
removeEntityByUid: jest.fn(),
|
||||
batchAddOrUpdateEntities: jest.fn(),
|
||||
entityAncestry: jest.fn(),
|
||||
};
|
||||
locationsCatalog = {
|
||||
addLocation: jest.fn(),
|
||||
removeLocation: jest.fn(),
|
||||
locations: jest.fn(),
|
||||
location: jest.fn(),
|
||||
locationHistory: jest.fn(),
|
||||
logUpdateSuccess: jest.fn(),
|
||||
logUpdateFailure: jest.fn(),
|
||||
};
|
||||
higherOrderOperation = {
|
||||
addLocation: jest.fn(),
|
||||
refreshAllLocations: jest.fn(),
|
||||
};
|
||||
const router = await createRouter({
|
||||
entitiesCatalog,
|
||||
locationsCatalog,
|
||||
higherOrderOperation,
|
||||
logger: getVoidLogger(),
|
||||
config: new ConfigReader({
|
||||
catalog: {
|
||||
readonly: true,
|
||||
},
|
||||
}),
|
||||
});
|
||||
app = express().use(router);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
describe('GET /entities', () => {
|
||||
it('happy path: lists entities', async () => {
|
||||
const entities: Entity[] = [
|
||||
{ apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
|
||||
];
|
||||
|
||||
entitiesCatalog.entities.mockResolvedValueOnce({
|
||||
entities: [entities[0]],
|
||||
pageInfo: { hasNextPage: false },
|
||||
});
|
||||
|
||||
const response = await request(app).get('/entities');
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual(entities);
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /entities', () => {
|
||||
it('is not allowed', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
};
|
||||
|
||||
const response = await request(app)
|
||||
.post('/entities')
|
||||
.set('Content-Type', 'application/json')
|
||||
.send(entity);
|
||||
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toHaveBeenCalled();
|
||||
expect(response.status).toEqual(403);
|
||||
expect(response.text).toMatch(/not allowed in readonly/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DELETE /entities/by-uid/:uid', () => {
|
||||
// this delete is allowed as there is no other way to remove entities
|
||||
it('is allowed', async () => {
|
||||
const response = await request(app).delete('/entities/by-uid/apa');
|
||||
|
||||
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledWith('apa');
|
||||
expect(response.status).toEqual(204);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /locations', () => {
|
||||
it('happy path: lists locations', async () => {
|
||||
const locations: LocationResponse[] = [
|
||||
{
|
||||
currentStatus: { timestamp: '', status: '', message: '' },
|
||||
data: { id: 'a', type: 'b', target: 'c' },
|
||||
},
|
||||
];
|
||||
locationsCatalog.locations.mockResolvedValueOnce(locations);
|
||||
|
||||
const response = await request(app).get('/locations');
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual(locations);
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /locations', () => {
|
||||
it('is not allowed', async () => {
|
||||
const spec: LocationSpec = {
|
||||
type: 'b',
|
||||
target: 'c',
|
||||
};
|
||||
|
||||
const response = await request(app).post('/locations').send(spec);
|
||||
|
||||
expect(higherOrderOperation.addLocation).not.toHaveBeenCalled();
|
||||
expect(response.status).toEqual(403);
|
||||
expect(response.text).toMatch(/not allowed in readonly/);
|
||||
});
|
||||
|
||||
it('supports dry run', async () => {
|
||||
const spec: LocationSpec = {
|
||||
type: 'b',
|
||||
target: 'c',
|
||||
};
|
||||
|
||||
higherOrderOperation.addLocation.mockResolvedValue({
|
||||
location: { id: 'a', ...spec },
|
||||
entities: [],
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.post('/locations?dryRun=true')
|
||||
.send(spec);
|
||||
|
||||
expect(higherOrderOperation.addLocation).toHaveBeenCalledTimes(1);
|
||||
expect(higherOrderOperation.addLocation).toHaveBeenCalledWith(spec, {
|
||||
dryRun: true,
|
||||
});
|
||||
expect(response.status).toEqual(201);
|
||||
expect(response.body).toEqual(
|
||||
expect.objectContaining({
|
||||
location: { id: 'a', ...spec },
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,254 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { errorHandler } from '@backstage/backend-common';
|
||||
import type { Entity } from '@backstage/catalog-model';
|
||||
import {
|
||||
analyzeLocationSchema,
|
||||
locationSpecSchema,
|
||||
} from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import { NotFoundError } from '@backstage/errors';
|
||||
import express from 'express';
|
||||
import Router from 'express-promise-router';
|
||||
import { Logger } from 'winston';
|
||||
import yn from 'yn';
|
||||
import { EntitiesCatalog } from '../../catalog';
|
||||
import { LocationsCatalog } from '../catalog';
|
||||
import { LocationAnalyzer } from '../../ingestion/types';
|
||||
import { HigherOrderOperation } from '../ingestion/types';
|
||||
import {
|
||||
RefreshService,
|
||||
LocationService,
|
||||
RefreshOptions,
|
||||
} from '../../service/types';
|
||||
import {
|
||||
basicEntityFilter,
|
||||
parseEntityFilterParams,
|
||||
parseEntityPaginationParams,
|
||||
parseEntityTransformParams,
|
||||
} from '../../service/request';
|
||||
import {
|
||||
disallowReadonlyMode,
|
||||
requireRequestBody,
|
||||
validateRequestBody,
|
||||
} from '../../service/util';
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export interface RouterOptions {
|
||||
entitiesCatalog?: EntitiesCatalog;
|
||||
locationsCatalog?: LocationsCatalog;
|
||||
higherOrderOperation?: HigherOrderOperation;
|
||||
locationAnalyzer?: LocationAnalyzer;
|
||||
locationService?: LocationService;
|
||||
refreshService?: RefreshService;
|
||||
logger: Logger;
|
||||
config: Config;
|
||||
}
|
||||
|
||||
/** @deprecated This was part of the legacy catalog engine */
|
||||
export async function createRouter(
|
||||
options: RouterOptions,
|
||||
): Promise<express.Router> {
|
||||
const {
|
||||
entitiesCatalog,
|
||||
locationsCatalog,
|
||||
higherOrderOperation,
|
||||
locationAnalyzer,
|
||||
locationService,
|
||||
refreshService,
|
||||
config,
|
||||
logger,
|
||||
} = options;
|
||||
|
||||
const router = Router();
|
||||
router.use(express.json());
|
||||
|
||||
const readonlyEnabled =
|
||||
config.getOptionalBoolean('catalog.readonly') || false;
|
||||
if (readonlyEnabled) {
|
||||
logger.info('Catalog is running in readonly mode');
|
||||
}
|
||||
|
||||
if (refreshService) {
|
||||
router.post('/refresh', async (req, res) => {
|
||||
const refreshOptions: RefreshOptions = req.body;
|
||||
await refreshService.refresh(refreshOptions);
|
||||
res.status(200).send();
|
||||
});
|
||||
}
|
||||
|
||||
if (entitiesCatalog) {
|
||||
router
|
||||
.get('/entities', async (req, res) => {
|
||||
const { entities, pageInfo } = await entitiesCatalog.entities({
|
||||
filter: parseEntityFilterParams(req.query),
|
||||
fields: parseEntityTransformParams(req.query),
|
||||
pagination: parseEntityPaginationParams(req.query),
|
||||
});
|
||||
|
||||
// Add a Link header to the next page
|
||||
if (pageInfo.hasNextPage) {
|
||||
const url = new URL(`http://ignored${req.url}`);
|
||||
url.searchParams.delete('offset');
|
||||
url.searchParams.set('after', pageInfo.endCursor);
|
||||
res.setHeader('link', `<${url.pathname}${url.search}>; rel="next"`);
|
||||
}
|
||||
|
||||
// TODO(freben): encode the pageInfo in the response
|
||||
res.json(entities);
|
||||
})
|
||||
.post('/entities', async (req, res) => {
|
||||
/*
|
||||
* NOTE: THIS METHOD IS DEPRECATED AND NOT RECOMMENDED TO USE
|
||||
*
|
||||
* Posting entities to this method has unclear semantics and will not
|
||||
* properly subject them to limitations, processing, or resolution of
|
||||
* relations.
|
||||
*
|
||||
* It stays around in the service for the time being, but may be
|
||||
* removed or change semantics at any time without prior notice.
|
||||
*/
|
||||
disallowReadonlyMode(readonlyEnabled);
|
||||
|
||||
const body = await requireRequestBody(req);
|
||||
const [result] = await entitiesCatalog.batchAddOrUpdateEntities!([
|
||||
{ entity: body as Entity, relations: [] },
|
||||
]);
|
||||
const response = await entitiesCatalog.entities({
|
||||
filter: basicEntityFilter({ 'metadata.uid': result.entityId }),
|
||||
});
|
||||
res.status(200).json(response.entities[0]);
|
||||
})
|
||||
.get('/entities/by-uid/:uid', async (req, res) => {
|
||||
const { uid } = req.params;
|
||||
const { entities } = await entitiesCatalog.entities({
|
||||
filter: basicEntityFilter({ 'metadata.uid': uid }),
|
||||
});
|
||||
if (!entities.length) {
|
||||
throw new NotFoundError(`No entity with uid ${uid}`);
|
||||
}
|
||||
res.status(200).json(entities[0]);
|
||||
})
|
||||
.delete('/entities/by-uid/:uid', async (req, res) => {
|
||||
const { uid } = req.params;
|
||||
await entitiesCatalog.removeEntityByUid(uid);
|
||||
res.status(204).end();
|
||||
})
|
||||
.get('/entities/by-name/:kind/:namespace/:name', async (req, res) => {
|
||||
const { kind, namespace, name } = req.params;
|
||||
const { entities } = await entitiesCatalog.entities({
|
||||
filter: basicEntityFilter({
|
||||
kind: kind,
|
||||
'metadata.namespace': namespace,
|
||||
'metadata.name': name,
|
||||
}),
|
||||
});
|
||||
if (!entities.length) {
|
||||
throw new NotFoundError(
|
||||
`No entity named '${name}' found, with kind '${kind}' in namespace '${namespace}'`,
|
||||
);
|
||||
}
|
||||
res.status(200).json(entities[0]);
|
||||
});
|
||||
}
|
||||
|
||||
if (locationService) {
|
||||
router
|
||||
.post('/locations', async (req, res) => {
|
||||
const input = await validateRequestBody(req, locationSpecSchema);
|
||||
const dryRun = yn(req.query.dryRun, { default: false });
|
||||
|
||||
// when in dryRun addLocation is effectively a read operation so we don't
|
||||
// need to disallow readonly
|
||||
if (!dryRun) {
|
||||
disallowReadonlyMode(readonlyEnabled);
|
||||
}
|
||||
|
||||
const output = await locationService.createLocation(input, dryRun);
|
||||
res.status(201).json(output);
|
||||
})
|
||||
.get('/locations', async (_req, res) => {
|
||||
const locations = await locationService.listLocations();
|
||||
res.status(200).json(locations.map(l => ({ data: l })));
|
||||
})
|
||||
|
||||
.get('/locations/:id', async (req, res) => {
|
||||
const { id } = req.params;
|
||||
const output = await locationService.getLocation(id);
|
||||
res.status(200).json(output);
|
||||
})
|
||||
.delete('/locations/:id', async (req, res) => {
|
||||
disallowReadonlyMode(readonlyEnabled);
|
||||
|
||||
const { id } = req.params;
|
||||
await locationService.deleteLocation(id);
|
||||
res.status(204).end();
|
||||
});
|
||||
}
|
||||
|
||||
if (higherOrderOperation) {
|
||||
router.post('/locations', async (req, res) => {
|
||||
const input = await validateRequestBody(req, locationSpecSchema);
|
||||
const dryRun = yn(req.query.dryRun, { default: false });
|
||||
|
||||
// when in dryRun addLocation is effectively a read operation so we don't
|
||||
// need to disallow readonly
|
||||
if (!dryRun) {
|
||||
disallowReadonlyMode(readonlyEnabled);
|
||||
}
|
||||
|
||||
const output = await higherOrderOperation.addLocation(input, { dryRun });
|
||||
res.status(201).json(output);
|
||||
});
|
||||
}
|
||||
|
||||
if (locationsCatalog) {
|
||||
router
|
||||
.get('/locations', async (_req, res) => {
|
||||
const output = await locationsCatalog.locations();
|
||||
res.status(200).json(output);
|
||||
})
|
||||
.get('/locations/:id/history', async (req, res) => {
|
||||
const { id } = req.params;
|
||||
const output = await locationsCatalog.locationHistory(id);
|
||||
res.status(200).json(output);
|
||||
})
|
||||
.get('/locations/:id', async (req, res) => {
|
||||
const { id } = req.params;
|
||||
const output = await locationsCatalog.location(id);
|
||||
res.status(200).json(output);
|
||||
})
|
||||
.delete('/locations/:id', async (req, res) => {
|
||||
disallowReadonlyMode(readonlyEnabled);
|
||||
|
||||
const { id } = req.params;
|
||||
await locationsCatalog.removeLocation(id);
|
||||
res.status(204).end();
|
||||
});
|
||||
}
|
||||
|
||||
if (locationAnalyzer) {
|
||||
router.post('/analyze-location', async (req, res) => {
|
||||
const input = await validateRequestBody(req, analyzeLocationSchema);
|
||||
const output = await locationAnalyzer.analyzeLocation(input);
|
||||
res.status(200).json(output);
|
||||
});
|
||||
}
|
||||
|
||||
router.use(errorHandler());
|
||||
return router;
|
||||
}
|
||||
Reference in New Issue
Block a user