refactor: don't dryRun addLocation
This allows to keep the HigherOderOperations free from db Operations
This commit is contained in:
@@ -86,7 +86,7 @@ describe('DatabaseEntitiesCatalog', () => {
|
||||
expect(result).toEqual([{ entityId: 'u' }]);
|
||||
});
|
||||
|
||||
it('adds using existing transaction', async () => {
|
||||
it('dry run of add operation', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
@@ -95,10 +95,6 @@ describe('DatabaseEntitiesCatalog', () => {
|
||||
namespace: 'd',
|
||||
},
|
||||
};
|
||||
const existingTransaction: jest.Mocked<Transaction> = {
|
||||
rollback: jest.fn(),
|
||||
};
|
||||
|
||||
db.entities.mockResolvedValue([]);
|
||||
db.addEntities.mockResolvedValue([
|
||||
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
|
||||
@@ -107,7 +103,7 @@ describe('DatabaseEntitiesCatalog', () => {
|
||||
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
|
||||
const result = await catalog.batchAddOrUpdateEntities(
|
||||
[{ entity, relations: [] }],
|
||||
{ tx: existingTransaction },
|
||||
{ dryRun: true },
|
||||
);
|
||||
|
||||
expect(db.entities).toHaveBeenCalledTimes(1);
|
||||
@@ -121,9 +117,53 @@ describe('DatabaseEntitiesCatalog', () => {
|
||||
expect(db.setRelations).toHaveBeenCalledTimes(1);
|
||||
expect(db.setRelations).toHaveBeenCalledWith(expect.anything(), 'u', []);
|
||||
expect(db.addEntities).toHaveBeenCalledTimes(1);
|
||||
expect(transaction.rollback).toBeCalledTimes(1);
|
||||
expect(result).toEqual([{ entityId: 'u' }]);
|
||||
});
|
||||
|
||||
it('output modified entities', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
},
|
||||
};
|
||||
const dbEntity: Entity = {
|
||||
apiVersion: 'a',
|
||||
kind: 'b',
|
||||
metadata: {
|
||||
name: 'c',
|
||||
namespace: 'd',
|
||||
uid: 'u',
|
||||
},
|
||||
};
|
||||
db.entities.mockResolvedValue([
|
||||
{
|
||||
entity: dbEntity,
|
||||
},
|
||||
]);
|
||||
db.addEntities.mockResolvedValue([
|
||||
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
|
||||
]);
|
||||
|
||||
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
|
||||
const result = await catalog.batchAddOrUpdateEntities(
|
||||
[{ entity, relations: [] }],
|
||||
{ outputEntities: true },
|
||||
);
|
||||
|
||||
expect(db.entities).toHaveBeenCalledTimes(2);
|
||||
expect(db.setRelations).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual([
|
||||
{
|
||||
entityId: 'u',
|
||||
entity: dbEntity,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('updates when given uid', async () => {
|
||||
const entity: Entity = {
|
||||
apiVersion: 'a',
|
||||
|
||||
@@ -65,16 +65,10 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
|
||||
private readonly logger: Logger,
|
||||
) {}
|
||||
|
||||
async entities(options?: {
|
||||
filters?: EntityFilters[];
|
||||
tx?: Transaction;
|
||||
}): Promise<Entity[]> {
|
||||
const filters = options?.filters;
|
||||
const items = options?.tx
|
||||
? await this.database.entities(options?.tx, filters)
|
||||
: await this.database.transaction(tx =>
|
||||
this.database.entities(tx, filters),
|
||||
);
|
||||
async entities(filters?: EntityFilters[]): Promise<Entity[]> {
|
||||
const items = await this.database.transaction(tx =>
|
||||
this.database.entities(tx, filters),
|
||||
);
|
||||
return items.map(i => i.entity);
|
||||
}
|
||||
|
||||
@@ -148,109 +142,110 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
|
||||
requests: EntityUpsertRequest[],
|
||||
options?: {
|
||||
locationId?: string;
|
||||
tx?: Transaction;
|
||||
dryRun?: boolean;
|
||||
outputEntities?: boolean;
|
||||
},
|
||||
): Promise<EntityUpsertResponse[]> {
|
||||
const locationId = options?.locationId;
|
||||
|
||||
if (options?.tx) {
|
||||
return this.batchAddOrUpdateEntitiesInTransaction(
|
||||
requests,
|
||||
options.tx,
|
||||
locationId,
|
||||
);
|
||||
}
|
||||
return await this.database.transaction(async tx => {
|
||||
// Group the entities by unique kind+namespace combinations
|
||||
const entitiesByKindAndNamespace = groupBy(requests, ({ entity }) => {
|
||||
const name = getEntityName(entity);
|
||||
return `${name.kind}:${name.namespace}`.toLowerCase();
|
||||
});
|
||||
|
||||
return await this.database.transaction(
|
||||
async tx =>
|
||||
await this.batchAddOrUpdateEntitiesInTransaction(
|
||||
requests,
|
||||
tx,
|
||||
locationId,
|
||||
),
|
||||
);
|
||||
}
|
||||
const limiter = limiterFactory(BATCH_CONCURRENCY);
|
||||
const tasks: Promise<EntityUpsertResponse[]>[] = [];
|
||||
|
||||
private async batchAddOrUpdateEntitiesInTransaction(
|
||||
requests: EntityUpsertRequest[],
|
||||
tx: Transaction,
|
||||
locationId?: string,
|
||||
): Promise<EntityUpsertResponse[]> {
|
||||
// Group the entities by unique kind+namespace combinations
|
||||
const entitiesByKindAndNamespace = groupBy(requests, ({ entity }) => {
|
||||
const name = getEntityName(entity);
|
||||
return `${name.kind}:${name.namespace}`.toLowerCase();
|
||||
});
|
||||
for (const groupRequests of Object.values(entitiesByKindAndNamespace)) {
|
||||
const { kind, namespace } = getEntityName(groupRequests[0].entity);
|
||||
|
||||
const limiter = limiterFactory(BATCH_CONCURRENCY);
|
||||
const tasks: Promise<EntityUpsertResponse[]>[] = [];
|
||||
// Go through the new entities in reasonable chunk sizes (sometimes,
|
||||
// sources produce tens of thousands of entities, and those are too large
|
||||
// batch sizes to reasonably send to the database)
|
||||
for (const batch of chunk(groupRequests, BATCH_SIZE)) {
|
||||
tasks.push(
|
||||
limiter(async () => {
|
||||
const first = serializeEntityRef(batch[0].entity);
|
||||
const last = serializeEntityRef(batch[batch.length - 1].entity);
|
||||
let modifiedEntityIds: EntityUpsertResponse[] = [];
|
||||
|
||||
for (const groupRequests of Object.values(entitiesByKindAndNamespace)) {
|
||||
const { kind, namespace } = getEntityName(groupRequests[0].entity);
|
||||
this.logger.debug(
|
||||
`Considering batch ${first}-${last} (${batch.length} entries)`,
|
||||
);
|
||||
|
||||
// Go through the new entities in reasonable chunk sizes (sometimes,
|
||||
// sources produce tens of thousands of entities, and those are too large
|
||||
// batch sizes to reasonably send to the database)
|
||||
for (const batch of chunk(groupRequests, BATCH_SIZE)) {
|
||||
tasks.push(
|
||||
limiter(async () => {
|
||||
const first = serializeEntityRef(batch[0].entity);
|
||||
const last = serializeEntityRef(batch[batch.length - 1].entity);
|
||||
const modifiedEntityIds: EntityUpsertResponse[] = [];
|
||||
this.logger.debug(
|
||||
`Considering batch ${first}-${last} (${batch.length} entries)`,
|
||||
);
|
||||
|
||||
// Retry the batch write a few times to deal with contention
|
||||
const context = {
|
||||
kind,
|
||||
namespace,
|
||||
locationId,
|
||||
};
|
||||
for (let attempt = 1; attempt <= BATCH_ATTEMPTS; ++attempt) {
|
||||
try {
|
||||
const { toAdd, toUpdate, toIgnore } = await this.analyzeBatch(
|
||||
batch,
|
||||
context,
|
||||
tx,
|
||||
);
|
||||
if (toAdd.length) {
|
||||
modifiedEntityIds.push(
|
||||
...(await this.batchAdd(toAdd, context, tx)),
|
||||
// Retry the batch write a few times to deal with contention
|
||||
const context = {
|
||||
kind,
|
||||
namespace,
|
||||
locationId,
|
||||
};
|
||||
for (let attempt = 1; attempt <= BATCH_ATTEMPTS; ++attempt) {
|
||||
try {
|
||||
const { toAdd, toUpdate, toIgnore } = await this.analyzeBatch(
|
||||
batch,
|
||||
context,
|
||||
tx,
|
||||
);
|
||||
}
|
||||
if (toUpdate.length) {
|
||||
modifiedEntityIds.push(
|
||||
...(await this.batchUpdate(toUpdate, context, tx)),
|
||||
);
|
||||
}
|
||||
// TODO(Rugvip): We currently always update relations, but we
|
||||
// likely want to figure out a way to avoid that
|
||||
for (const { entity, relations } of toIgnore) {
|
||||
const entityId = entity.metadata.uid!;
|
||||
await this.setRelations(entityId, relations, tx);
|
||||
modifiedEntityIds.push({ entityId });
|
||||
}
|
||||
if (toAdd.length) {
|
||||
modifiedEntityIds.push(
|
||||
...(await this.batchAdd(toAdd, context, tx)),
|
||||
);
|
||||
}
|
||||
if (toUpdate.length) {
|
||||
modifiedEntityIds.push(
|
||||
...(await this.batchUpdate(toUpdate, context, tx)),
|
||||
);
|
||||
}
|
||||
// TODO(Rugvip): We currently always update relations, but we
|
||||
// likely want to figure out a way to avoid that
|
||||
for (const { entity, relations } of toIgnore) {
|
||||
const entityId = entity.metadata.uid!;
|
||||
await this.setRelations(entityId, relations, tx);
|
||||
modifiedEntityIds.push({ entityId });
|
||||
}
|
||||
|
||||
break;
|
||||
} catch (e) {
|
||||
if (e instanceof ConflictError && attempt < BATCH_ATTEMPTS) {
|
||||
this.logger.warn(
|
||||
`Failed to write batch at attempt ${attempt}/${BATCH_ATTEMPTS}, ${e}`,
|
||||
);
|
||||
} else {
|
||||
throw e;
|
||||
break;
|
||||
} catch (e) {
|
||||
if (e instanceof ConflictError && attempt < BATCH_ATTEMPTS) {
|
||||
this.logger.warn(
|
||||
`Failed to write batch at attempt ${attempt}/${BATCH_ATTEMPTS}, ${e}`,
|
||||
);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return modifiedEntityIds;
|
||||
}),
|
||||
);
|
||||
if (options?.outputEntities) {
|
||||
const writtenEntities = await this.database.entities(tx, [
|
||||
{ 'metadata.uid': modifiedEntityIds.map(e => e.entityId) },
|
||||
]);
|
||||
|
||||
modifiedEntityIds = writtenEntities.map(e => ({
|
||||
entityId: e.entity.metadata.uid!,
|
||||
entity: e.entity,
|
||||
}));
|
||||
}
|
||||
|
||||
return modifiedEntityIds;
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return (await Promise.all(tasks)).flat();
|
||||
const entityUpserts = (await Promise.all(tasks)).flat();
|
||||
|
||||
if (options?.dryRun) {
|
||||
// If this is only a dry run, cancel the database transaction even if it was successful.
|
||||
await tx.rollback();
|
||||
|
||||
this.logger.debug(`Perfomed successful dry run of adding entities`);
|
||||
}
|
||||
|
||||
return entityUpserts;
|
||||
});
|
||||
}
|
||||
|
||||
// Set the relations originating from an entity using the DB layer
|
||||
@@ -278,19 +273,16 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
|
||||
const markTimestamp = process.hrtime();
|
||||
|
||||
const names = requests.map(({ entity }) => entity.metadata.name);
|
||||
const oldEntities = await this.entities({
|
||||
filters: [
|
||||
{
|
||||
kind: kind,
|
||||
'metadata.namespace': namespace,
|
||||
'metadata.name': names,
|
||||
},
|
||||
],
|
||||
tx,
|
||||
});
|
||||
const oldEntities = await this.database.entities(tx, [
|
||||
{
|
||||
kind: kind,
|
||||
'metadata.namespace': namespace,
|
||||
'metadata.name': names,
|
||||
},
|
||||
]);
|
||||
|
||||
const oldEntitiesByName = new Map(
|
||||
oldEntities.map(e => [e.metadata.name, e]),
|
||||
oldEntities.map(e => [e.entity.metadata.name, e.entity]),
|
||||
);
|
||||
|
||||
const toAdd: EntityUpsertRequest[] = [];
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
*/
|
||||
|
||||
import { Location } from '@backstage/catalog-model';
|
||||
import type { Database, Transaction } from '../database';
|
||||
import type { Database } from '../database';
|
||||
import {
|
||||
DatabaseLocationUpdateLogEvent,
|
||||
DatabaseLocationUpdateLogStatus,
|
||||
@@ -25,16 +25,7 @@ import { LocationResponse, LocationsCatalog } from './types';
|
||||
export class DatabaseLocationsCatalog implements LocationsCatalog {
|
||||
constructor(private readonly database: Database) {}
|
||||
|
||||
async addLocation(
|
||||
location: Location,
|
||||
options?: { tx?: Transaction },
|
||||
): Promise<Location> {
|
||||
const transaction = options?.tx;
|
||||
|
||||
if (transaction) {
|
||||
return await this.database.addLocation(transaction, location);
|
||||
}
|
||||
|
||||
async addLocation(location: Location): Promise<Location> {
|
||||
return await this.database.transaction(
|
||||
async tx => await this.database.addLocation(tx, location),
|
||||
);
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
*/
|
||||
|
||||
import { Entity, EntityRelationSpec, Location } from '@backstage/catalog-model';
|
||||
import type { EntityFilters, Transaction } from '../database';
|
||||
import type { EntityFilters } from '../database';
|
||||
|
||||
//
|
||||
// Entities
|
||||
@@ -28,13 +28,11 @@ export type EntityUpsertRequest = {
|
||||
|
||||
export type EntityUpsertResponse = {
|
||||
entityId: string;
|
||||
entity?: Entity;
|
||||
};
|
||||
|
||||
export type EntitiesCatalog = {
|
||||
entities(options?: {
|
||||
filters?: EntityFilters[];
|
||||
tx?: Transaction;
|
||||
}): Promise<Entity[]>;
|
||||
entities(filters?: EntityFilters[]): Promise<Entity[]>;
|
||||
removeEntityByUid(uid: string): Promise<void>;
|
||||
|
||||
/**
|
||||
@@ -46,8 +44,9 @@ export type EntitiesCatalog = {
|
||||
batchAddOrUpdateEntities(
|
||||
entities: EntityUpsertRequest[],
|
||||
options?: {
|
||||
tx?: Transaction;
|
||||
locationId?: string;
|
||||
dryRun?: boolean;
|
||||
outputEntities?: boolean;
|
||||
},
|
||||
): Promise<EntityUpsertResponse[]>;
|
||||
};
|
||||
@@ -76,10 +75,7 @@ export type LocationResponse = {
|
||||
};
|
||||
|
||||
export type LocationsCatalog = {
|
||||
addLocation(
|
||||
location: Location,
|
||||
options?: { tx?: Transaction },
|
||||
): Promise<Location>;
|
||||
addLocation(location: Location): Promise<Location>;
|
||||
removeLocation(id: string): Promise<void>;
|
||||
locations(): Promise<LocationResponse[]>;
|
||||
location(id: string): Promise<LocationResponse>;
|
||||
|
||||
@@ -18,11 +18,7 @@ import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { Entity, Location, LocationSpec } from '@backstage/catalog-model';
|
||||
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
|
||||
import { LocationUpdateStatus } from '../catalog/types';
|
||||
import {
|
||||
Database,
|
||||
DatabaseLocationUpdateLogStatus,
|
||||
Transaction,
|
||||
} from '../database/types';
|
||||
import { DatabaseLocationUpdateLogStatus } from '../database/types';
|
||||
import { HigherOrderOperations } from './HigherOrderOperations';
|
||||
import { LocationReader } from './types';
|
||||
|
||||
@@ -30,8 +26,6 @@ describe('HigherOrderOperations', () => {
|
||||
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
|
||||
let locationsCatalog: jest.Mocked<LocationsCatalog>;
|
||||
let locationReader: jest.Mocked<LocationReader>;
|
||||
let transaction: jest.Mocked<Transaction>;
|
||||
let database: jest.Mocked<Database>;
|
||||
let higherOrderOperation: HigherOrderOperations;
|
||||
|
||||
beforeAll(() => {
|
||||
@@ -52,37 +46,16 @@ describe('HigherOrderOperations', () => {
|
||||
locationReader = {
|
||||
read: jest.fn(),
|
||||
};
|
||||
transaction = {
|
||||
rollback: jest.fn(),
|
||||
};
|
||||
database = {
|
||||
transaction: jest.fn(),
|
||||
addEntities: jest.fn(),
|
||||
updateEntity: jest.fn(),
|
||||
entities: jest.fn(),
|
||||
entityByName: jest.fn(),
|
||||
entityByUid: jest.fn(),
|
||||
removeEntityByUid: jest.fn(),
|
||||
setRelations: jest.fn(),
|
||||
addLocation: jest.fn(),
|
||||
removeLocation: jest.fn(),
|
||||
location: jest.fn(),
|
||||
locations: jest.fn(),
|
||||
locationHistory: jest.fn(),
|
||||
addLocationUpdateLogEvent: jest.fn(),
|
||||
};
|
||||
higherOrderOperation = new HigherOrderOperations(
|
||||
entitiesCatalog,
|
||||
locationsCatalog,
|
||||
locationReader,
|
||||
database,
|
||||
getVoidLogger(),
|
||||
);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
database.transaction.mockImplementation(async f => f(transaction));
|
||||
});
|
||||
|
||||
describe('addLocation', () => {
|
||||
@@ -117,9 +90,7 @@ describe('HigherOrderOperations', () => {
|
||||
id: expect.anything(),
|
||||
...spec,
|
||||
}),
|
||||
{ tx: transaction },
|
||||
);
|
||||
expect(transaction.rollback).not.toBeCalled();
|
||||
});
|
||||
|
||||
it('insert the location and its entities', async () => {
|
||||
@@ -139,9 +110,10 @@ describe('HigherOrderOperations', () => {
|
||||
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
|
||||
{
|
||||
entityId: 'id',
|
||||
entity,
|
||||
},
|
||||
]);
|
||||
entitiesCatalog.entities.mockResolvedValue([entity]);
|
||||
|
||||
locationReader.read.mockResolvedValue({
|
||||
entities: [
|
||||
{
|
||||
@@ -169,13 +141,18 @@ describe('HigherOrderOperations', () => {
|
||||
id: expect.anything(),
|
||||
...spec,
|
||||
}),
|
||||
{ tx: transaction },
|
||||
);
|
||||
expect(locationReader.read).toBeCalledTimes(1);
|
||||
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
|
||||
expect(entitiesCatalog.entities).toBeCalledTimes(1);
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledTimes(1);
|
||||
expect(transaction.rollback).not.toBeCalled();
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
locationId: expect.anything(),
|
||||
dryRun: false,
|
||||
outputEntities: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('reuses the location if a match already existed', async () => {
|
||||
@@ -208,7 +185,6 @@ describe('HigherOrderOperations', () => {
|
||||
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled();
|
||||
expect(locationsCatalog.addLocation).not.toBeCalled();
|
||||
expect(transaction.rollback).not.toBeCalled();
|
||||
});
|
||||
|
||||
it('rejects the whole operation if any entity could not be read', async () => {
|
||||
@@ -235,7 +211,6 @@ describe('HigherOrderOperations', () => {
|
||||
expect(locationsCatalog.locations).toBeCalledTimes(1);
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled();
|
||||
expect(locationsCatalog.addLocation).not.toBeCalled();
|
||||
expect(transaction.rollback).not.toBeCalled();
|
||||
});
|
||||
|
||||
it('rollback everything after a dry run', async () => {
|
||||
@@ -249,15 +224,14 @@ describe('HigherOrderOperations', () => {
|
||||
kind: 'b',
|
||||
metadata: { name: 'n' },
|
||||
};
|
||||
locationsCatalog.addLocation.mockImplementation(x => Promise.resolve(x));
|
||||
locationsCatalog.locations.mockResolvedValue([]);
|
||||
locationsCatalog.locations.mockResolvedValue([]);
|
||||
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
|
||||
{
|
||||
entityId: 'id',
|
||||
entity,
|
||||
},
|
||||
]);
|
||||
entitiesCatalog.entities.mockResolvedValue([entity]);
|
||||
locationReader.read.mockResolvedValue({
|
||||
entities: [
|
||||
{
|
||||
@@ -281,19 +255,16 @@ describe('HigherOrderOperations', () => {
|
||||
);
|
||||
expect(result.entities).toEqual([entity]);
|
||||
expect(locationsCatalog.locations).toBeCalledTimes(1);
|
||||
expect(locationsCatalog.addLocation).toBeCalledTimes(1);
|
||||
expect(locationsCatalog.addLocation).toBeCalledWith(
|
||||
expect.objectContaining({
|
||||
id: expect.anything(),
|
||||
...spec,
|
||||
}),
|
||||
{ tx: transaction },
|
||||
);
|
||||
expect(locationReader.read).toBeCalledTimes(1);
|
||||
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
|
||||
expect(entitiesCatalog.entities).toBeCalledTimes(1);
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledTimes(1);
|
||||
expect(transaction.rollback).toBeCalled();
|
||||
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
dryRun: true,
|
||||
outputEntities: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
import { Location, LocationSpec } from '@backstage/catalog-model';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { Logger } from 'winston';
|
||||
import { Database } from '../database';
|
||||
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
|
||||
import { durationText } from '../util/timing';
|
||||
import {
|
||||
@@ -38,7 +37,6 @@ export class HigherOrderOperations implements HigherOrderOperation {
|
||||
private readonly entitiesCatalog: EntitiesCatalog,
|
||||
private readonly locationsCatalog: LocationsCatalog,
|
||||
private readonly locationReader: LocationReader,
|
||||
private readonly database: Database,
|
||||
private readonly logger: Logger,
|
||||
) {}
|
||||
|
||||
@@ -85,33 +83,25 @@ export class HigherOrderOperations implements HigherOrderOperation {
|
||||
// in the entities list. But we aren't sure what to do about those yet.
|
||||
|
||||
// Write
|
||||
const entities = await this.database.transaction(async tx => {
|
||||
if (!previousLocation) {
|
||||
await this.locationsCatalog.addLocation(location, { tx });
|
||||
}
|
||||
if (readerOutput.entities.length === 0) {
|
||||
return [];
|
||||
}
|
||||
if (!previousLocation && !dryRun) {
|
||||
// TODO: We do not include location operations in the dryRun. We might perform
|
||||
// this operation as a seperate dry run.
|
||||
await this.locationsCatalog.addLocation(location);
|
||||
}
|
||||
if (readerOutput.entities.length === 0) {
|
||||
return { location, entities: [] };
|
||||
}
|
||||
|
||||
const writtenEntities = await this.entitiesCatalog.batchAddOrUpdateEntities(
|
||||
readerOutput.entities,
|
||||
{ locationId: location.id, tx },
|
||||
);
|
||||
const writtenEntities = await this.entitiesCatalog.batchAddOrUpdateEntities(
|
||||
readerOutput.entities,
|
||||
{
|
||||
locationId: dryRun ? undefined : location.id,
|
||||
dryRun,
|
||||
outputEntities: true,
|
||||
},
|
||||
);
|
||||
|
||||
const outputEntities = await this.entitiesCatalog.entities({
|
||||
filters: [{ 'metadata.uid': writtenEntities.map(e => e.entityId) }],
|
||||
tx,
|
||||
});
|
||||
|
||||
if (dryRun) {
|
||||
// If this is only a dry run, cancel the database transaction even if it was successful.
|
||||
await tx.rollback();
|
||||
|
||||
this.logger.debug(`Perfomed successful dry run of adding a location`);
|
||||
}
|
||||
|
||||
return outputEntities;
|
||||
});
|
||||
const entities = writtenEntities.map(e => e.entity!);
|
||||
|
||||
return { location, entities };
|
||||
}
|
||||
|
||||
@@ -227,7 +227,6 @@ export class CatalogBuilder {
|
||||
entitiesCatalog,
|
||||
locationsCatalog,
|
||||
locationReader,
|
||||
db,
|
||||
logger,
|
||||
);
|
||||
|
||||
|
||||
@@ -83,15 +83,13 @@ describe('createRouter', () => {
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filters: [
|
||||
{
|
||||
a: ['1', null, '3'],
|
||||
b: ['4'],
|
||||
},
|
||||
{ c: [null] },
|
||||
],
|
||||
});
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith([
|
||||
{
|
||||
a: ['1', null, '3'],
|
||||
b: ['4'],
|
||||
},
|
||||
{ c: [null] },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -109,9 +107,9 @@ describe('createRouter', () => {
|
||||
const response = await request(app).get('/entities/by-uid/zzz');
|
||||
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filters: [{ 'metadata.uid': 'zzz' }],
|
||||
});
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith([
|
||||
{ 'metadata.uid': 'zzz' },
|
||||
]);
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual(expect.objectContaining(entity));
|
||||
});
|
||||
@@ -122,9 +120,9 @@ describe('createRouter', () => {
|
||||
const response = await request(app).get('/entities/by-uid/zzz');
|
||||
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filters: [{ 'metadata.uid': 'zzz' }],
|
||||
});
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith([
|
||||
{ 'metadata.uid': 'zzz' },
|
||||
]);
|
||||
expect(response.status).toEqual(404);
|
||||
expect(response.text).toMatch(/uid/);
|
||||
});
|
||||
@@ -145,15 +143,13 @@ describe('createRouter', () => {
|
||||
const response = await request(app).get('/entities/by-name/k/ns/n');
|
||||
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filters: [
|
||||
{
|
||||
kind: 'k',
|
||||
'metadata.namespace': 'ns',
|
||||
'metadata.name': 'n',
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith([
|
||||
{
|
||||
kind: 'k',
|
||||
'metadata.namespace': 'ns',
|
||||
'metadata.name': 'n',
|
||||
},
|
||||
]);
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual(expect.objectContaining(entity));
|
||||
});
|
||||
@@ -164,15 +160,13 @@ describe('createRouter', () => {
|
||||
const response = await request(app).get('/entities/by-name/b/d/c');
|
||||
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filters: [
|
||||
{
|
||||
kind: 'b',
|
||||
'metadata.namespace': 'd',
|
||||
'metadata.name': 'c',
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith([
|
||||
{
|
||||
kind: 'b',
|
||||
'metadata.namespace': 'd',
|
||||
'metadata.name': 'c',
|
||||
},
|
||||
]);
|
||||
expect(response.status).toEqual(404);
|
||||
expect(response.text).toMatch(/name/);
|
||||
});
|
||||
@@ -215,9 +209,9 @@ describe('createRouter', () => {
|
||||
{ entity, relations: [] },
|
||||
]);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
|
||||
filters: [{ 'metadata.uid': 'u' }],
|
||||
});
|
||||
expect(entitiesCatalog.entities).toHaveBeenCalledWith([
|
||||
{ 'metadata.uid': 'u' },
|
||||
]);
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual(entity);
|
||||
});
|
||||
|
||||
@@ -49,7 +49,7 @@ export async function createRouter(
|
||||
.get('/entities', async (req, res) => {
|
||||
const filters = translateQueryToEntityFilters(req.query);
|
||||
const fieldMapper = translateQueryToFieldMapper(req.query);
|
||||
const entities = await entitiesCatalog.entities({ filters });
|
||||
const entities = await entitiesCatalog.entities(filters);
|
||||
res.status(200).send(entities.map(fieldMapper));
|
||||
})
|
||||
.post('/entities', async (req, res) => {
|
||||
@@ -57,20 +57,18 @@ export async function createRouter(
|
||||
const [result] = await entitiesCatalog.batchAddOrUpdateEntities([
|
||||
{ entity: body as Entity, relations: [] },
|
||||
]);
|
||||
const [entity] = await entitiesCatalog.entities({
|
||||
filters: [{ 'metadata.uid': result.entityId }],
|
||||
});
|
||||
const [entity] = await entitiesCatalog.entities([
|
||||
{ 'metadata.uid': result.entityId },
|
||||
]);
|
||||
res.status(200).send(entity);
|
||||
})
|
||||
.get('/entities/by-uid/:uid', async (req, res) => {
|
||||
const { uid } = req.params;
|
||||
const entities = await entitiesCatalog.entities({
|
||||
filters: [
|
||||
{
|
||||
'metadata.uid': uid,
|
||||
},
|
||||
],
|
||||
});
|
||||
const entities = await entitiesCatalog.entities([
|
||||
{
|
||||
'metadata.uid': uid,
|
||||
},
|
||||
]);
|
||||
if (!entities.length) {
|
||||
res.status(404).send(`No entity with uid ${uid}`);
|
||||
}
|
||||
@@ -83,15 +81,13 @@ export async function createRouter(
|
||||
})
|
||||
.get('/entities/by-name/:kind/:namespace/:name', async (req, res) => {
|
||||
const { kind, namespace, name } = req.params;
|
||||
const entities = await entitiesCatalog.entities({
|
||||
filters: [
|
||||
{
|
||||
kind: kind,
|
||||
'metadata.namespace': namespace,
|
||||
'metadata.name': name,
|
||||
},
|
||||
],
|
||||
});
|
||||
const entities = await entitiesCatalog.entities([
|
||||
{
|
||||
kind: kind,
|
||||
'metadata.namespace': namespace,
|
||||
'metadata.name': name,
|
||||
},
|
||||
]);
|
||||
if (!entities.length) {
|
||||
res
|
||||
.status(404)
|
||||
|
||||
Reference in New Issue
Block a user