paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 43 Packages Projects Releases Wiki Activity

Merge pull request #8952 from backstage/freben/not-next

Browse Source 
Deprecate the legacy catalog engine
This commit is contained in:
Fredrik Adelöw
2022-01-17 14:26:40 +01:00
committed by GitHub
parent cc13505856 ca26a420c8
commit 2ab5c268b8
36 changed files with 196 additions and 6320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/khaki-adults-own.md
+73
View File
@@ -0,0 +1,73 @@
---
'@backstage/plugin-catalog-backend': minor
---
**BREAKING**: Removed all remnants of the old catalog engine implementation.
The old implementation has been deprecated for over half a year. To ensure that
you are not using the old implementation, check that your
`packages/backend/src/plugins/catalog.ts` creates the catalog builder using
`CatalogBuilder.create`. If you instead call `new CatalogBuilder`, you are on
the old implementation and will experience breakage if you upgrade to this
version. If you are still on the old version, see [the relevant change log
entry](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/CHANGELOG.md#patch-changes-27)
for migration instructions.
The minimal `packages/backend/src/plugins/catalog.ts` file is now:
```ts
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
builder.addProcessor(new ScaffolderEntitiesProcessor());
const { processingEngine, router } = await builder.build();
await processingEngine.start();
return router;
}
```
The following classes and interfaces have been removed:
- The `CatalogBuilder` constructor (see above; use `CatalogBuilder.create`
instead)
- `AddLocationResult`
- `CommonDatabase`
- `CreateDatabaseOptions`
- `createNextRouter` (use `createRouter` instead - or preferably, use the
`router` field returned for you by `catalogBuilder.build()`)
- `Database`
- `DatabaseEntitiesCatalog` (use `EntitiesCatalog` instead)
- `DatabaseLocationsCatalog` (use `LocationService` instead)
- `DatabaseLocationUpdateLogEvent`
- `DatabaseLocationUpdateLogStatus`
- `DatabaseManager`
- `DbEntitiesRequest`
- `DbEntitiesResponse`
- `DbEntityRequest`
- `DbEntityResponse`
- `DbLocationsRow`
- `DbLocationsRowWithStatus`
- `DbPageInfo`
- `EntitiesCatalog.batchAddOrUpdateEntities` (was only used by the legacy
engine)
- `EntityUpsertRequest`
- `EntityUpsertResponse`
- `HigherOrderOperation`
- `HigherOrderOperations`
- `LocationReader`
- `LocationReaders`
- `LocationResponse`
- `LocationsCatalog`
- `LocationUpdateLogEvent`
- `LocationUpdateStatus`
- `NextCatalogBuilder` (use `CatalogBuilder.create` instead)
- `NextRouterOptions` (use `RouterOptions` instead)
- `ReadLocationEntity`
- `ReadLocationError`
- `ReadLocationResult`
- `Transaction`
The `RouterOptions` interface has been un-deprecated, and has instead found use
for passing into `createRouter`. Its shape has been significantly changed to
accommodate the new router.
plugins/catalog-backend/api-report.md
+21 -520
View File
@@ -12,7 +12,6 @@ import { CatalogEntitiesRequest } from '@backstage/catalog-client';
import { Config } from '@backstage/config';
import { DocumentCollator } from '@backstage/search-common';
import { Entity } from '@backstage/catalog-model';
import { EntityName } from '@backstage/catalog-model';
import { EntityPolicy } from '@backstage/catalog-model';
import { EntityRelationSpec } from '@backstage/catalog-model';
import express from 'express';
@@ -21,7 +20,6 @@ import { GitHubIntegrationConfig } from '@backstage/integration';
import { IndexableDocument } from '@backstage/search-common';
import { JsonObject } from '@backstage/types';
import { JsonValue } from '@backstage/types';
import { Knex } from 'knex';
import { Location as Location_2 } from '@backstage/catalog-model';
import { LocationSpec } from '@backstage/catalog-model';
import { Logger as Logger_2 } from 'winston';
@@ -38,14 +36,6 @@ import { TokenManager } from '@backstage/backend-common';
import { UrlReader } from '@backstage/backend-common';
import { Validators } from '@backstage/catalog-model';
// Warning: (ae-missing-release-tag) "AddLocationResult" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type AddLocationResult = {
location: Location_2;
entities: Entity[];
};
// Warning: (ae-missing-release-tag) "AnalyzeLocationEntityField" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
@@ -246,30 +236,38 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor {
validateEntityKind(entity: Entity): Promise<boolean>;
}
// Warning: (ae-missing-release-tag) "CatalogBuilder" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
export class CatalogBuilder {
// @deprecated
constructor(env: CatalogEnvironment);
addEntityPolicy(...policies: EntityPolicy[]): CatalogBuilder;
addEntityProvider(...providers: EntityProvider[]): CatalogBuilder;
addPermissionRules(
...permissionRules: PermissionRule<
Entity,
EntitiesSearchFilter,
unknown[]
>[]
): void;
addProcessor(...processors: CatalogProcessor[]): CatalogBuilder;
build(): Promise<{
entitiesCatalog: EntitiesCatalog;
locationsCatalog: LocationsCatalog;
higherOrderOperation: HigherOrderOperation;
locationAnalyzer: LocationAnalyzer;
processingEngine: CatalogProcessingEngine;
locationService: LocationService;
router: Router;
}>;
// (undocumented)
static create(env: CatalogEnvironment): Promise<NextCatalogBuilder>;
static create(env: CatalogEnvironment): CatalogBuilder;
getDefaultProcessors(): CatalogProcessor[];
replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder;
replaceProcessors(processors: CatalogProcessor[]): CatalogBuilder;
setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder;
setFieldFormatValidators(validators: Partial<Validators>): CatalogBuilder;
setLocationAnalyzer(locationAnalyzer: LocationAnalyzer): CatalogBuilder;
setPlaceholderResolver(
key: string,
resolver: PlaceholderResolver,
): CatalogBuilder;
setRefreshInterval(refreshInterval: RefreshIntervalFunction): CatalogBuilder;
setRefreshIntervalSeconds(seconds: number): CatalogBuilder;
}
// Warning: (ae-missing-release-tag) "CatalogEntityDocument" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -451,89 +449,11 @@ export class CodeOwnersProcessor implements CatalogProcessor {
preProcessEntity(entity: Entity, location: LocationSpec): Promise<Entity>;
}
// Warning: (ae-missing-release-tag) "CommonDatabase" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated
export class CommonDatabase implements Database {
constructor(database: Knex, logger: Logger_2);
// (undocumented)
addEntities(
txOpaque: Transaction,
request: DbEntityRequest[],
): Promise<DbEntityResponse[]>;
// (undocumented)
addLocation(
txOpaque: Transaction,
location: Location_2,
): Promise<DbLocationsRow>;
// (undocumented)
addLocationUpdateLogEvent(
locationId: string,
status: DatabaseLocationUpdateLogStatus,
entityName?: string | string[],
message?: string,
): Promise<void>;
// (undocumented)
entities(
txOpaque: Transaction,
request?: DbEntitiesRequest,
): Promise<DbEntitiesResponse>;
// (undocumented)
entityByName(
txOpaque: Transaction,
name: EntityName,
): Promise<DbEntityResponse | undefined>;
// (undocumented)
entityByUid(
txOpaque: Transaction,
uid: string,
): Promise<DbEntityResponse | undefined>;
// (undocumented)
location(id: string): Promise<DbLocationsRowWithStatus>;
// (undocumented)
locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]>;
// (undocumented)
locations(): Promise<DbLocationsRowWithStatus[]>;
// (undocumented)
removeEntityByUid(txOpaque: Transaction, uid: string): Promise<void>;
// (undocumented)
removeLocation(txOpaque: Transaction, id: string): Promise<void>;
// (undocumented)
setRelations(
txOpaque: Transaction,
originatingEntityId: string,
relations: EntityRelationSpec[],
): Promise<void>;
// (undocumented)
transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T>;
// (undocumented)
updateEntity(
txOpaque: Transaction,
request: DbEntityRequest,
matchingEtag?: string,
matchingGeneration?: number,
): Promise<DbEntityResponse>;
}
// @public
export const createCatalogPermissionRule: <TParams extends unknown[]>(
rule: PermissionRule<Entity, EntitiesSearchFilter, TParams>,
) => PermissionRule<Entity, EntitiesSearchFilter, TParams>;
// Warning: (ae-missing-release-tag) "CreateDatabaseOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type CreateDatabaseOptions = {
logger: Logger_2;
};
// Warning: (ae-missing-release-tag) "createNextRouter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export function createNextRouter(
options: NextRouterOptions,
): Promise<express.Router>;
// Warning: (ae-missing-release-tag) "createRandomRefreshInterval" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
@@ -542,211 +462,9 @@ export function createRandomRefreshInterval(options: {
maxSeconds: number;
}): RefreshIntervalFunction;
// Warning: (ae-missing-release-tag) "createRouter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
// @public
export function createRouter(options: RouterOptions): Promise<express.Router>;
// Warning: (ae-missing-release-tag) "Database" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated
export type Database = {
transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T>;
addEntities(
tx: Transaction,
request: DbEntityRequest[],
): Promise<DbEntityResponse[]>;
updateEntity(
tx: Transaction,
request: DbEntityRequest,
matchingEtag?: string,
matchingGeneration?: number,
): Promise<DbEntityResponse>;
entities(
tx: Transaction,
request?: DbEntitiesRequest,
): Promise<DbEntitiesResponse>;
entityByName(
tx: Transaction,
name: EntityName,
): Promise<DbEntityResponse | undefined>;
entityByUid(
tx: Transaction,
uid: string,
): Promise<DbEntityResponse | undefined>;
removeEntityByUid(tx: Transaction, uid: string): Promise<void>;
setRelations(
tx: Transaction,
entityUid: string,
relations: EntityRelationSpec[],
): Promise<void>;
addLocation(tx: Transaction, location: Location_2): Promise<DbLocationsRow>;
removeLocation(tx: Transaction, id: string): Promise<void>;
location(id: string): Promise<DbLocationsRowWithStatus>;
locations(): Promise<DbLocationsRowWithStatus[]>;
locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]>;
addLocationUpdateLogEvent(
locationId: string,
status: DatabaseLocationUpdateLogStatus,
entityName?: string | string[],
message?: string,
): Promise<void>;
};
// Warning: (ae-missing-release-tag) "DatabaseEntitiesCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export class DatabaseEntitiesCatalog implements EntitiesCatalog {
constructor(database: Database, logger: Logger_2);
// (undocumented)
batchAddOrUpdateEntities(
requests: EntityUpsertRequest[],
options?: {
locationId?: string;
dryRun?: boolean;
outputEntities?: boolean;
},
): Promise<EntityUpsertResponse[]>;
// (undocumented)
entities(request?: EntitiesRequest): Promise<EntitiesResponse>;
// (undocumented)
entityAncestry(): Promise<never>;
// (undocumented)
removeEntityByUid(uid: string): Promise<void>;
}
// Warning: (ae-missing-release-tag) "DatabaseLocationsCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export class DatabaseLocationsCatalog implements LocationsCatalog {
constructor(database: Database);
// (undocumented)
addLocation(location: Location_2): Promise<Location_2>;
// (undocumented)
location(id: string): Promise<LocationResponse>;
// (undocumented)
locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]>;
// (undocumented)
locations(): Promise<LocationResponse[]>;
// (undocumented)
logUpdateFailure(
locationId: string,
error?: Error,
entityName?: string,
): Promise<void>;
// (undocumented)
logUpdateSuccess(
locationId: string,
entityName?: string | string[],
): Promise<void>;
// (undocumented)
removeLocation(id: string): Promise<void>;
}
// Warning: (ae-missing-release-tag) "DatabaseLocationUpdateLogEvent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type DatabaseLocationUpdateLogEvent = {
id: string;
status: DatabaseLocationUpdateLogStatus;
location_id: string;
entity_name: string;
created_at?: string;
message?: string;
};
// Warning: (ae-missing-release-tag) "DatabaseLocationUpdateLogStatus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export enum DatabaseLocationUpdateLogStatus {
// (undocumented)
FAIL = 'fail',
// (undocumented)
SUCCESS = 'success',
}
// Warning: (ae-missing-release-tag) "DatabaseManager" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export class DatabaseManager {
// (undocumented)
static createDatabase(
knex: Knex,
options?: Partial<CreateDatabaseOptions>,
): Promise<Database>;
// (undocumented)
static createInMemoryDatabase(): Promise<Database>;
// (undocumented)
static createInMemoryDatabaseConnection(): Promise<Knex>;
// (undocumented)
static createTestDatabase(): Promise<Database>;
// (undocumented)
static createTestDatabaseConnection(): Promise<Knex>;
}
// Warning: (ae-missing-release-tag) "DbEntitiesRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type DbEntitiesRequest = {
filter?: EntityFilter;
pagination?: EntityPagination;
};
// Warning: (ae-missing-release-tag) "DbEntitiesResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type DbEntitiesResponse = {
entities: DbEntityResponse[];
pageInfo: DbPageInfo;
};
// Warning: (ae-missing-release-tag) "DbEntityRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type DbEntityRequest = {
locationId?: string;
entity: Entity;
relations: EntityRelationSpec[];
};
// Warning: (ae-missing-release-tag) "DbEntityResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type DbEntityResponse = {
locationId?: string;
entity: Entity;
};
// Warning: (ae-missing-release-tag) "DbLocationsRow" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type DbLocationsRow = {
id: string;
type: string;
target: string;
};
// Warning: (ae-missing-release-tag) "DbLocationsRowWithStatus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type DbLocationsRowWithStatus = DbLocationsRow & {
status: string | null;
timestamp: string | null;
message: string | null;
};
// Warning: (ae-missing-release-tag) "DbPageInfo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type DbPageInfo =
| {
hasNextPage: false;
}
| {
hasNextPage: true;
endCursor: string;
};
// Warning: (ae-missing-release-tag) "DefaultCatalogCollator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
@@ -836,14 +554,6 @@ export type EntitiesCatalog = {
authorizationToken?: string;
},
): Promise<void>;
batchAddOrUpdateEntities?(
requests: EntityUpsertRequest[],
options?: {
locationId?: string;
dryRun?: boolean;
outputEntities?: boolean;
},
): Promise<EntityUpsertResponse[]>;
entityAncestry(entityRef: string): Promise<EntityAncestryResponse>;
};
@@ -971,22 +681,6 @@ export type EntityProviderMutation =
removed: DeferredEntity[];
};
// Warning: (ae-missing-release-tag) "EntityUpsertRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type EntityUpsertRequest = {
entity: Entity;
relations: EntityRelationSpec[];
};
// Warning: (ae-missing-release-tag) "EntityUpsertResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type EntityUpsertResponse = {
entityId: string;
entity?: Entity;
};
// Warning: (ae-missing-release-tag) "FileReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
@@ -1132,38 +826,6 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor {
updateLastActivity(): Promise<string | undefined>;
}
// Warning: (ae-missing-release-tag) "HigherOrderOperation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type HigherOrderOperation = {
addLocation(
spec: LocationSpec,
options?: {
dryRun?: boolean;
},
): Promise<AddLocationResult>;
refreshAllLocations(): Promise<void>;
};
// Warning: (ae-missing-release-tag) "HigherOrderOperations" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated
export class HigherOrderOperations implements HigherOrderOperation {
constructor(
entitiesCatalog: EntitiesCatalog,
locationsCatalog: LocationsCatalog,
locationReader: LocationReader,
logger: Logger_2,
);
addLocation(
spec: LocationSpec,
options?: {
dryRun?: boolean;
},
): Promise<AddLocationResult>;
refreshAllLocations(): Promise<void>;
}
// Warning: (ae-missing-release-tag) "inputError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
@@ -1209,51 +871,6 @@ export type LocationEntityProcessorOptions = {
integrations: ScmIntegrationRegistry;
};
// Warning: (ae-missing-release-tag) "LocationReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type LocationReader = {
read(location: LocationSpec): Promise<ReadLocationResult>;
};
// Warning: (ae-missing-release-tag) "LocationReaders" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated
export class LocationReaders implements LocationReader {
// Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts
constructor(options: Options_3);
// (undocumented)
read(location: LocationSpec): Promise<ReadLocationResult>;
}
// Warning: (ae-missing-release-tag) "LocationResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type LocationResponse = {
data: Location_2;
currentStatus: LocationUpdateStatus;
};
// Warning: (ae-missing-release-tag) "LocationsCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type LocationsCatalog = {
addLocation(location: Location_2): Promise<Location_2>;
removeLocation(id: string): Promise<void>;
locations(): Promise<LocationResponse[]>;
location(id: string): Promise<LocationResponse>;
locationHistory(id: string): Promise<LocationUpdateLogEvent[]>;
logUpdateSuccess(
locationId: string,
entityName?: string | string[],
): Promise<void>;
logUpdateFailure(
locationId: string,
error?: Error,
entityName?: string,
): Promise<void>;
};
// Warning: (ae-missing-release-tag) "LocationService" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
@@ -1289,86 +906,6 @@ export interface LocationStore {
listLocations(): Promise<Location_2[]>;
}
// Warning: (ae-missing-release-tag) "LocationUpdateLogEvent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type LocationUpdateLogEvent = {
id: string;
status: 'fail' | 'success';
location_id: string;
entity_name: string;
created_at?: string;
message?: string;
};
// Warning: (ae-missing-release-tag) "LocationUpdateStatus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type LocationUpdateStatus = {
timestamp: string | null;
status: string | null;
message: string | null;
};
// Warning: (ae-missing-release-tag) "NextCatalogBuilder" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
export class NextCatalogBuilder {
constructor(env: CatalogEnvironment);
addEntityPolicy(...policies: EntityPolicy[]): NextCatalogBuilder;
addEntityProvider(...providers: EntityProvider[]): NextCatalogBuilder;
addPermissionRules(
...permissionRules: PermissionRule<
Entity,
EntitiesSearchFilter,
unknown[]
>[]
): void;
addProcessor(...processors: CatalogProcessor[]): NextCatalogBuilder;
build(): Promise<{
entitiesCatalog: EntitiesCatalog;
locationsCatalog: LocationsCatalog;
locationAnalyzer: LocationAnalyzer;
processingEngine: CatalogProcessingEngine;
locationService: LocationService;
router: Router;
}>;
getDefaultProcessors(): CatalogProcessor[];
replaceEntityPolicies(policies: EntityPolicy[]): NextCatalogBuilder;
replaceProcessors(processors: CatalogProcessor[]): NextCatalogBuilder;
setEntityDataParser(parser: CatalogProcessorParser): NextCatalogBuilder;
setFieldFormatValidators(validators: Partial<Validators>): NextCatalogBuilder;
setLocationAnalyzer(locationAnalyzer: LocationAnalyzer): NextCatalogBuilder;
setPlaceholderResolver(
key: string,
resolver: PlaceholderResolver,
): NextCatalogBuilder;
setRefreshInterval(
refreshInterval: RefreshIntervalFunction,
): NextCatalogBuilder;
setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder;
}
// Warning: (ae-missing-release-tag) "NextRouterOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export interface NextRouterOptions {
// (undocumented)
config: Config;
// (undocumented)
entitiesCatalog?: EntitiesCatalog;
// (undocumented)
locationAnalyzer?: LocationAnalyzer;
// (undocumented)
locationService: LocationService;
// (undocumented)
logger: Logger_2;
// (undocumented)
permissionIntegrationRouter?: express.Router;
// (undocumented)
refreshService?: RefreshService;
}
// Warning: (ae-missing-release-tag) "notFoundError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
@@ -1472,31 +1009,6 @@ export type PlaceholderResolverResolveUrl = (
base: string,
) => string;
// Warning: (ae-missing-release-tag) "ReadLocationEntity" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type ReadLocationEntity = {
location: LocationSpec;
entity: Entity;
relations: EntityRelationSpec[];
};
// Warning: (ae-missing-release-tag) "ReadLocationError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type ReadLocationError = {
location: LocationSpec;
error: Error;
};
// Warning: (ae-missing-release-tag) "ReadLocationResult" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
export type ReadLocationResult = {
entities: ReadLocationEntity[];
errors: ReadLocationError[];
};
// Warning: (ae-missing-release-tag) "RecursivePartial" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
@@ -1541,25 +1053,21 @@ declare namespace results {
}
export { results };
// Warning: (ae-missing-release-tag) "RouterOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated (undocumented)
// @public
export interface RouterOptions {
// (undocumented)
config: Config;
// (undocumented)
entitiesCatalog?: EntitiesCatalog;
// (undocumented)
higherOrderOperation?: HigherOrderOperation;
// (undocumented)
locationAnalyzer?: LocationAnalyzer;
// (undocumented)
locationsCatalog?: LocationsCatalog;
// (undocumented)
locationService?: LocationService;
locationService: LocationService;
// (undocumented)
logger: Logger_2;
// (undocumented)
permissionIntegrationRouter?: express.Router;
// (undocumented)
refreshService?: RefreshService;
}
@@ -1583,13 +1091,6 @@ export class StaticLocationProcessor implements StaticLocationProcessor {
): Promise<boolean>;
}
// Warning: (ae-missing-release-tag) "Transaction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public @deprecated
export type Transaction = {
rollback(): Promise<unknown>;
};
// Warning: (ae-missing-release-tag) "UrlReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
plugins/catalog-backend/src/legacy/service/index.ts → plugins/catalog-backend/migrations/20220116144621_remove_legacy.js
+12 -3
View File
@@ -14,6 +14,15 @@
* limitations under the License.
*/
export { CatalogBuilder } from './CatalogBuilder';
export { createRouter } from './router';
export type { RouterOptions } from './router';
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex.schema.dropTable('entities_relations');
await knex.schema.dropTable('entities_search');
await knex.schema.dropTable('entities');
};
exports.down = async function down() {};
plugins/catalog-backend/src/catalog/index.ts
-2
View File
@@ -19,8 +19,6 @@ export type {
EntitiesRequest,
EntitiesResponse,
EntityAncestryResponse,
EntityUpsertRequest,
EntityUpsertResponse,
PageInfo,
EntitiesSearchFilter,
EntityFilter,
plugins/catalog-backend/src/catalog/types.ts
+1 -27
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { Entity, EntityRelationSpec } from '@backstage/catalog-model';
import { Entity } from '@backstage/catalog-model';
/**
* A filter expression for entities.
@@ -78,18 +78,6 @@ export type EntitiesResponse = {
pageInfo: PageInfo;
};
/** @deprecated This was part of the legacy catalog engine */
export type EntityUpsertRequest = {
entity: Entity;
relations: EntityRelationSpec[];
};
/** @deprecated This was part of the legacy catalog engine */
export type EntityUpsertResponse = {
entityId: string;
entity?: Entity;
};
/** @public */
export type EntityAncestryResponse = {
rootEntityRef: string;
@@ -118,20 +106,6 @@ export type EntitiesCatalog = {
options?: { authorizationToken?: string },
): Promise<void>;
/**
* Writes a number of entities efficiently to storage.
*
* @deprecated This method was part of the legacy catalog engine and will be removed.
*/
batchAddOrUpdateEntities?(
requests: EntityUpsertRequest[],
options?: {
locationId?: string;
dryRun?: boolean;
outputEntities?: boolean;
},
): Promise<EntityUpsertResponse[]>;
/**
* Returns the full ancestry tree upward along reference edges.
*
plugins/catalog-backend/src/index.ts
-1
View File
@@ -22,7 +22,6 @@
export * from './catalog';
export * from './ingestion';
export * from './legacy';
export * from './search';
export * from './util';
export * from './processing';
plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.test.ts
-426
View File
@@ -1,426 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, LOCATION_ANNOTATION } from '@backstage/catalog-model';
import { Database, DatabaseManager, Transaction } from '../database';
import { basicEntityFilter } from '../../service/request';
import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
import { EntityUpsertRequest } from '../../catalog/types';
describe('DatabaseEntitiesCatalog', () => {
let db: jest.Mocked<Database>;
let transaction: jest.Mocked<Transaction>;
beforeAll(() => {
db = {
transaction: jest.fn(),
addEntities: jest.fn(),
updateEntity: jest.fn(),
entities: jest.fn(),
entityByName: jest.fn(),
entityByUid: jest.fn(),
removeEntityByUid: jest.fn(),
setRelations: jest.fn(),
addLocation: jest.fn(),
removeLocation: jest.fn(),
location: jest.fn(),
locations: jest.fn(),
locationHistory: jest.fn(),
addLocationUpdateLogEvent: jest.fn(),
};
transaction = {
rollback: jest.fn(),
};
});
beforeEach(() => {
jest.resetAllMocks();
db.transaction.mockImplementation(async f => f(transaction));
});
describe('batchAddOrUpdateEntities', () => {
it('adds when no given uid and no matching by name', async () => {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
name: 'c',
namespace: 'd',
},
};
db.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNextPage: false },
});
db.addEntities.mockResolvedValue([
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
]);
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
const result = await catalog.batchAddOrUpdateEntities([
{ entity, relations: [] },
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.addEntities).toHaveBeenCalledTimes(1);
expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [
{ entity: expect.anything(), relations: [] },
]);
expect(result).toEqual([{ entityId: 'u' }]);
});
it('dry run of add operation', async () => {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
name: 'c',
namespace: 'd',
},
};
db.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNextPage: false },
});
db.addEntities.mockResolvedValue([
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
]);
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
const result = await catalog.batchAddOrUpdateEntities(
[{ entity, relations: [] }],
{ dryRun: true },
);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.addEntities).toHaveBeenCalledTimes(1);
expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [
{ entity: expect.anything(), relations: [] },
]);
expect(transaction.rollback).toBeCalledTimes(1);
expect(result).toEqual([{ entityId: 'u' }]);
});
it('output modified entities', async () => {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
name: 'c',
namespace: 'd',
annotations: {
[LOCATION_ANNOTATION]: 'mock',
},
},
};
const dbEntity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
name: 'c',
namespace: 'd',
description: 'changes',
uid: 'u',
annotations: {
[LOCATION_ANNOTATION]: 'mock',
},
},
};
db.entities.mockResolvedValue({
entities: [{ entity: dbEntity }],
pageInfo: { hasNextPage: false },
});
db.addEntities.mockResolvedValue([
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
]);
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
const result = await catalog.batchAddOrUpdateEntities(
[{ entity, relations: [] }],
{ outputEntities: true },
);
expect(db.entities).toHaveBeenCalledTimes(2);
expect(db.addEntities).toHaveBeenCalledTimes(1);
expect(result).toEqual([
{
entityId: 'u',
entity: dbEntity,
},
]);
});
it('updates when given uid', async () => {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
uid: 'u',
name: 'c',
namespace: 'd',
},
spec: {
x: 'b',
},
};
const existing = {
entity: {
apiVersion: 'a',
kind: 'b',
metadata: {
uid: 'u',
etag: 'e',
generation: 1,
name: 'c',
namespace: 'd',
},
spec: {
x: 'a',
},
},
};
db.entities.mockResolvedValue({
entities: [existing],
pageInfo: { hasNextPage: false },
});
db.entityByUid.mockResolvedValue(existing);
db.updateEntity.mockResolvedValue({ entity });
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
const result = await catalog.batchAddOrUpdateEntities([
{ entity, relations: [] },
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.entityByName).not.toHaveBeenCalled();
expect(db.entityByUid).toHaveBeenCalledTimes(1);
expect(db.entityByUid).toHaveBeenCalledWith(transaction, 'u');
expect(db.updateEntity).toHaveBeenCalledTimes(1);
expect(db.updateEntity).toHaveBeenCalledWith(
transaction,
{
entity: {
apiVersion: 'a',
kind: 'b',
metadata: {
uid: 'u',
etag: expect.any(String),
generation: 2,
name: 'c',
namespace: 'd',
},
spec: {
x: 'b',
},
},
relations: [],
},
'e',
1,
);
expect(result).toEqual([{ entityId: 'u' }]);
});
it('update when no given uid and matching by name', async () => {
const added: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
name: 'c',
namespace: 'd',
},
spec: {
x: 'b',
},
};
const existing = {
entity: {
apiVersion: 'a',
kind: 'b',
metadata: {
uid: 'u',
etag: 'e',
generation: 1,
name: 'c',
namespace: 'd',
},
spec: {
x: 'a',
},
},
};
db.entities.mockResolvedValue({
entities: [existing],
pageInfo: { hasNextPage: false },
});
db.entityByName.mockResolvedValue(existing);
db.updateEntity.mockResolvedValue(existing);
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
const result = await catalog.batchAddOrUpdateEntities([
{ entity: added, relations: [] },
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.entityByName).toHaveBeenCalledTimes(1);
expect(db.entityByName).toHaveBeenCalledWith(transaction, {
kind: 'b',
namespace: 'd',
name: 'c',
});
expect(db.updateEntity).toHaveBeenCalledTimes(1);
expect(db.updateEntity).toHaveBeenCalledWith(
transaction,
{
entity: {
apiVersion: 'a',
kind: 'b',
metadata: {
uid: 'u',
etag: expect.any(String),
generation: 2,
name: 'c',
namespace: 'd',
},
spec: {
x: 'b',
},
},
relations: [],
},
'e',
1,
);
expect(result).toEqual([{ entityId: 'u' }]);
});
it('should not update if entity is unchanged', async () => {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
uid: 'u',
name: 'c',
namespace: 'd',
},
spec: {
x: 'a',
},
};
db.entities.mockResolvedValue({
entities: [{ entity }],
pageInfo: { hasNextPage: false },
});
db.entityByUid.mockResolvedValue({ entity });
db.updateEntity.mockResolvedValue({ entity });
const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
const result = await catalog.batchAddOrUpdateEntities([
{ entity, relations: [] },
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.entityByName).not.toHaveBeenCalled();
expect(db.entityByUid).not.toHaveBeenCalled();
expect(db.updateEntity).not.toHaveBeenCalled();
expect(db.setRelations).toHaveBeenCalledTimes(1);
expect(db.setRelations).toHaveBeenCalledWith(expect.anything(), 'u', []);
expect(result).toEqual([{ entityId: 'u' }]);
});
it('both adds and updates', async () => {
const catalog = new DatabaseEntitiesCatalog(
await DatabaseManager.createTestDatabase(),
getVoidLogger(),
);
const entities: EntityUpsertRequest[] = [];
for (let i = 0; i < 300; ++i) {
entities.push({
entity: {
apiVersion: 'a',
kind: 'k',
metadata: { name: `n${i}` },
},
relations: [],
});
}
await catalog.batchAddOrUpdateEntities(entities);
const afterFirst = await catalog.entities();
expect(afterFirst.entities.length).toBe(300);
entities[40].entity.metadata.op = 'changed';
entities.push({
entity: {
apiVersion: 'a',
kind: 'k',
metadata: { name: `n300`, op: 'added' },
},
relations: [],
});
await catalog.batchAddOrUpdateEntities(entities);
const afterSecond = await catalog.entities();
expect(afterSecond.entities.length).toBe(301);
expect(
afterSecond.entities.find(e => e.metadata.op === 'changed'),
).toBeDefined();
expect(
afterSecond.entities.find(e => e.metadata.op === 'added'),
).toBeDefined();
}, 10000);
});
});
plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.ts
-377
View File
@@ -1,377 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Entity,
entityHasChanges,
generateUpdatedEntity,
getEntityName,
LOCATION_ANNOTATION,
serializeEntityRef,
} from '@backstage/catalog-model';
import { ConflictError } from '@backstage/errors';
import { chunk, groupBy } from 'lodash';
import limiterFactory from 'p-limit';
import { Logger } from 'winston';
import type { Database, DbEntityResponse, Transaction } from '../database';
import { DbEntitiesRequest } from '../database/types';
import { basicEntityFilter } from '../../service/request';
import { durationText } from '../../util/timing';
import type {
EntitiesCatalog,
EntitiesRequest,
EntitiesResponse,
EntityUpsertRequest,
EntityUpsertResponse,
} from '../../catalog/types';
type BatchContext = {
kind: string;
namespace: string;
locationId?: string;
};
// Some locations return tens or hundreds of thousands of entities. To make
// those payloads more manageable, we break work apart in batches of this
// many entities and write them to storage per batch.
const BATCH_SIZE = 100;
// When writing large batches, there's an increasing chance of contention in
// the form of conflicts where we compete with other writes. Each batch gets
// this many attempts at being written before giving up.
const BATCH_ATTEMPTS = 3;
// The number of batches that may be ongoing at the same time.
const BATCH_CONCURRENCY = 3;
/** @deprecated This was part of the legacy catalog engine */
export class DatabaseEntitiesCatalog implements EntitiesCatalog {
constructor(
private readonly database: Database,
private readonly logger: Logger,
) {}
async entities(request?: EntitiesRequest): Promise<EntitiesResponse> {
const dbRequest: DbEntitiesRequest = {
filter: request?.filter,
pagination: request?.pagination,
};
const dbResponse = await this.database.transaction(tx =>
this.database.entities(tx, dbRequest),
);
const entities = dbResponse.entities.map(e =>
request?.fields ? request.fields(e.entity) : e.entity,
);
return {
entities,
pageInfo: dbResponse.pageInfo,
};
}
async removeEntityByUid(uid: string): Promise<void> {
await this.database.transaction(async tx => {
await this.database.removeEntityByUid(tx, uid);
});
}
async batchAddOrUpdateEntities(
requests: EntityUpsertRequest[],
options?: {
locationId?: string;
dryRun?: boolean;
outputEntities?: boolean;
},
): Promise<EntityUpsertResponse[]> {
// Group the requests by unique kind+namespace combinations. The reason for
// this is that the change detection and merging logic requires finding
// pre-existing versions of the entities in the database. Those queries are
// easier and faster to make if every batch revolves around a single kind-
// namespace pair.
const requestsByKindAndNamespace = groupBy(requests, ({ entity }) => {
const name = getEntityName(entity);
return `${name.kind}:${name.namespace}`.toLowerCase();
});
// Go through the requests in reasonable batch sizes. Sometimes, sources
// produce tens of thousands of entities, and those are too large batch
// sizes to reasonably send to the database.
const batches = Object.values(requestsByKindAndNamespace)
.map(request => chunk(request, BATCH_SIZE))
.flat();
// Bound the number of concurrent batches. We want a bit of concurrency for
// performance reasons, but not so much that we starve the connection pool
// or start thrashing.
const limiter = limiterFactory(BATCH_CONCURRENCY);
const tasks = batches.map(batch =>
limiter(async () => {
// Retry the batch write a few times to deal with contention
for (let attempt = 1; ; ++attempt) {
try {
return this.batchAddOrUpdateEntitiesSingleBatch(batch, options);
} catch (e) {
if (e instanceof ConflictError && attempt < BATCH_ATTEMPTS) {
this.logger.warn(
`Failed to write batch at attempt ${attempt}/${BATCH_ATTEMPTS}, ${e}`,
);
} else {
throw e;
}
}
}
}),
);
const responses = await Promise.all(tasks);
return responses.flat();
}
// Defines the actual logic of running a single batch. All of these share a
// common kind and namespace.
private async batchAddOrUpdateEntitiesSingleBatch(
batch: EntityUpsertRequest[],
options?: {
locationId?: string;
dryRun?: boolean;
outputEntities?: boolean;
},
) {
const { kind, namespace } = getEntityName(batch[0].entity);
const context = {
kind,
namespace,
locationId: options?.locationId,
};
this.logger.debug(
`Considering batch ${serializeEntityRef(
batch[0].entity,
)}-${serializeEntityRef(batch[batch.length - 1].entity)} (${
batch.length
} entries)`,
);
return this.database.transaction(async tx => {
const { toAdd, toUpdate, toIgnore } = await this.analyzeBatch(
batch,
context,
tx,
);
let responses = new Array<EntityUpsertResponse>();
if (toAdd.length) {
const items = await this.batchAdd(toAdd, context, tx);
responses.push(...items);
}
if (toUpdate.length) {
const items = await this.batchUpdate(toUpdate, context, tx);
responses.push(...items);
}
for (const { entity, relations } of toIgnore) {
// TODO(Rugvip): We currently always update relations, but we
// likely want to figure out a way to avoid that
const entityId = entity.metadata.uid;
if (entityId) {
await this.database.setRelations(tx, entityId, relations);
responses.push({ entityId });
}
}
if (options?.outputEntities && responses.length > 0) {
const writtenEntities = await this.database.entities(tx, {
filter: basicEntityFilter({
'metadata.uid': responses.map(e => e.entityId),
}),
});
responses = writtenEntities.entities.map(e => ({
entityId: e.entity.metadata.uid!,
entity: e.entity,
}));
}
// If this is only a dry run, cancel the database transaction even if it
// was successful.
if (options?.dryRun) {
await tx.rollback();
this.logger.debug(`Performed successful dry run of adding entities`);
}
return responses;
});
}
// Given a batch of entities that were just read from a location, take them
// into consideration by comparing against the existing catalog entities and
// produce the list of entities to be added, and the list of entities to be
// updated
private async analyzeBatch(
requests: EntityUpsertRequest[],
{ kind, namespace }: BatchContext,
tx: Transaction,
): Promise<{
toAdd: EntityUpsertRequest[];
toUpdate: EntityUpsertRequest[];
toIgnore: EntityUpsertRequest[];
}> {
const markTimestamp = process.hrtime();
// Here we make use of the fact that all of the entities share kind and
// namespace within a batch
const names = requests.map(({ entity }) => entity.metadata.name);
const oldEntitiesResponse = await this.database.entities(tx, {
filter: basicEntityFilter({
kind: kind,
'metadata.namespace': namespace,
'metadata.name': names,
}),
});
const oldEntitiesByName = new Map(
oldEntitiesResponse.entities.map(e => [e.entity.metadata.name, e.entity]),
);
const toAdd: EntityUpsertRequest[] = [];
const toUpdate: EntityUpsertRequest[] = [];
const toIgnore: EntityUpsertRequest[] = [];
for (const request of requests) {
const newEntity = request.entity;
const oldEntity = oldEntitiesByName.get(newEntity.metadata.name);
const newLocation = newEntity.metadata.annotations?.[LOCATION_ANNOTATION];
const oldLocation =
oldEntity?.metadata.annotations?.[LOCATION_ANNOTATION];
if (!oldEntity) {
toAdd.push(request);
} else if (oldLocation !== newLocation) {
this.logger.warn(
`Rejecting write of entity ${serializeEntityRef(
newEntity,
)} from ${newLocation} because entity existed from ${oldLocation}`,
);
toIgnore.push(request);
} else if (entityHasChanges(oldEntity, newEntity)) {
// TODO(freben): This currently uses addOrUpdateEntity under the hood,
// but should probably calculate the end result entity right here
// instead and call a dedicated batch update database method
toUpdate.push(request);
} else {
// Use the existing entity to ensure that we're able to read it back by uid if needed
toIgnore.push({ ...request, entity: oldEntity });
}
}
this.logger.debug(
`Found ${toAdd.length} entities to add, ${
toUpdate.length
} entities to update in ${durationText(markTimestamp)}`,
);
return { toAdd, toUpdate, toIgnore };
}
// Efficiently adds the given entities to storage, under the assumption that
// they do not conflict with any existing entities
private async batchAdd(
requests: EntityUpsertRequest[],
{ locationId }: BatchContext,
tx: Transaction,
): Promise<EntityUpsertResponse[]> {
const markTimestamp = process.hrtime();
const res = await this.database.addEntities(
tx,
requests.map(({ entity, relations }) => ({
locationId,
entity,
relations,
})),
);
const responses = res.map(({ entity }) => ({
entityId: entity.metadata.uid!,
}));
this.logger.debug(
`Added ${requests.length} entities in ${durationText(markTimestamp)}`,
);
return responses;
}
// Efficiently updates the given entities into storage, under the assumption
// that there already exist entities with the same names
private async batchUpdate(
requests: EntityUpsertRequest[],
{ locationId }: BatchContext,
tx: Transaction,
): Promise<EntityUpsertResponse[]> {
const markTimestamp = process.hrtime();
const responses: EntityUpsertResponse[] = [];
// TODO(freben): Still not batched
for (const request of requests) {
const res = await this.addOrUpdateEntity(tx, request, locationId);
const entityId = res.metadata.uid!;
responses.push({ entityId });
}
this.logger.debug(
`Updated ${requests.length} entities in ${durationText(markTimestamp)}`,
);
return responses;
}
// TODO(freben): Incorporate this into batchUpdate which is the only caller
private async addOrUpdateEntity(
tx: Transaction,
{ entity, relations }: EntityUpsertRequest,
locationId?: string,
): Promise<Entity> {
// Find a matching (by uid, or by compound name, depending on the given
// entity) existing entity, to know whether to update or add
const existing = entity.metadata.uid
? await this.database.entityByUid(tx, entity.metadata.uid)
: await this.database.entityByName(tx, getEntityName(entity));
// If it's an update, run the algorithm for annotation merging, updating
// etag/generation, etc.
let response: DbEntityResponse;
if (existing) {
const updated = generateUpdatedEntity(existing.entity, entity);
response = await this.database.updateEntity(
tx,
{ locationId, entity: updated, relations },
existing.entity.metadata.etag,
existing.entity.metadata.generation,
);
} else {
const added = await this.database.addEntities(tx, [
{ locationId, entity, relations },
]);
response = added[0];
}
return response.entity;
}
async entityAncestry(): Promise<never> {
throw new Error('Not implemented');
}
}
plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts
-81
View File
@@ -1,81 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DatabaseManager } from '../database';
import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
const bootstrapLocation = {
id: expect.any(String),
type: 'bootstrap',
target: 'bootstrap',
};
describe('DatabaseLocationsCatalog', () => {
let catalog: DatabaseLocationsCatalog;
beforeEach(async () => {
const db = await DatabaseManager.createTestDatabase();
catalog = new DatabaseLocationsCatalog(db);
});
it('can add a location', async () => {
const location = {
id: 'dd12620d-0436-422f-93bd-929aa0788123',
type: 'valid_type',
target: 'valid_target',
};
await expect(catalog.addLocation(location)).resolves.toEqual(location);
await expect(
catalog.location('dd12620d-0436-422f-93bd-929aa0788123'),
).resolves.toEqual(expect.objectContaining({ data: location }));
await expect(catalog.locations()).resolves.toEqual(
expect.arrayContaining([
expect.objectContaining({ data: location }),
expect.objectContaining({ data: bootstrapLocation }),
]),
);
});
it('does not return duplicates of rows because of logs', async () => {
const location1 = {
id: 'dd12620d-0436-422f-93bd-929aa0788123',
type: 'valid_type',
target: 'valid_target1',
};
const location2 = {
id: '1a89c479-1a33-4f27-8927-6090ba488c42',
type: 'valid_type',
target: 'valid_target2',
};
await expect(catalog.addLocation(location1)).resolves.toEqual(location1);
await expect(catalog.addLocation(location2)).resolves.toEqual(location2);
await expect(
catalog.logUpdateSuccess(location1.id),
).resolves.toBeUndefined();
await expect(
catalog.logUpdateSuccess(location1.id),
).resolves.toBeUndefined();
const locations = await catalog.locations();
expect(locations.length).toBe(3);
expect(locations).toEqual(
expect.arrayContaining([
expect.objectContaining({ data: location1 }),
expect.objectContaining({ data: location2 }),
expect.objectContaining({ data: bootstrapLocation }),
]),
);
});
});
plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts
-91
View File
@@ -1,91 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Location } from '@backstage/catalog-model';
import type { Database } from '../database';
import {
DatabaseLocationUpdateLogEvent,
DatabaseLocationUpdateLogStatus,
} from '../database/types';
import { LocationResponse, LocationsCatalog } from './types';
/** @deprecated This was part of the legacy catalog engine */
export class DatabaseLocationsCatalog implements LocationsCatalog {
constructor(private readonly database: Database) {}
async addLocation(location: Location): Promise<Location> {
return await this.database.transaction(
async tx => await this.database.addLocation(tx, location),
);
}
async removeLocation(id: string): Promise<void> {
await this.database.transaction(tx => this.database.removeLocation(tx, id));
}
async locations(): Promise<LocationResponse[]> {
const items = await this.database.locations();
return items.map(({ message, status, timestamp, ...data }) => ({
currentStatus: {
message,
status,
timestamp,
},
data,
}));
}
async locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]> {
return this.database.locationHistory(id);
}
async location(id: string): Promise<LocationResponse> {
const { message, status, timestamp, ...data } =
await this.database.location(id);
return {
currentStatus: {
message,
status,
timestamp,
},
data,
};
}
async logUpdateSuccess(
locationId: string,
entityName?: string | string[],
): Promise<void> {
await this.database.addLocationUpdateLogEvent(
locationId,
DatabaseLocationUpdateLogStatus.SUCCESS,
entityName,
);
}
async logUpdateFailure(
locationId: string,
error?: Error,
entityName?: string,
): Promise<void> {
await this.database.addLocationUpdateLogEvent(
locationId,
DatabaseLocationUpdateLogStatus.FAIL,
entityName,
error?.message,
);
}
}
plugins/catalog-backend/src/legacy/catalog/index.ts
-24
View File
@@ -1,24 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
export type {
LocationResponse,
LocationsCatalog,
LocationUpdateLogEvent,
LocationUpdateStatus,
} from './types';
plugins/catalog-backend/src/legacy/catalog/types.ts
-62
View File
@@ -1,62 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Location } from '@backstage/catalog-model';
//
// Locations
//
/** @deprecated This was part of the legacy catalog engine */
export type LocationUpdateStatus = {
timestamp: string | null;
status: string | null;
message: string | null;
};
/** @deprecated This was part of the legacy catalog engine */
export type LocationUpdateLogEvent = {
id: string;
status: 'fail' | 'success';
location_id: string;
entity_name: string;
created_at?: string;
message?: string;
};
/** @deprecated This was part of the legacy catalog engine */
export type LocationResponse = {
data: Location;
currentStatus: LocationUpdateStatus;
};
/** @deprecated This was part of the legacy catalog engine */
export type LocationsCatalog = {
addLocation(location: Location): Promise<Location>;
removeLocation(id: string): Promise<void>;
locations(): Promise<LocationResponse[]>;
location(id: string): Promise<LocationResponse>;
locationHistory(id: string): Promise<LocationUpdateLogEvent[]>;
logUpdateSuccess(
locationId: string,
entityName?: string | string[],
): Promise<void>;
logUpdateFailure(
locationId: string,
error?: Error,
entityName?: string,
): Promise<void>;
};
plugins/catalog-backend/src/legacy/database/CommonDatabase.test.ts
-789
View File
@@ -1,789 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, Location, parseEntityRef } from '@backstage/catalog-model';
import { ConflictError } from '@backstage/errors';
import { basicEntityFilter } from '../../service/request';
import { DatabaseManager } from './DatabaseManager';
import type {
DbEntityRequest,
DbEntityResponse,
DbLocationsRowWithStatus,
} from './types';
import { Database, DatabaseLocationUpdateLogStatus } from './types';
const bootstrapLocation = {
id: expect.any(String),
type: 'bootstrap',
target: 'bootstrap',
message: null,
status: null,
timestamp: null,
};
describe('CommonDatabase', () => {
let db: Database;
let entityRequest: DbEntityRequest;
let entityResponse: DbEntityResponse;
beforeEach(async () => {
db = await DatabaseManager.createTestDatabase();
entityRequest = {
entity: {
apiVersion: 'a',
kind: 'b',
metadata: {
name: 'c',
namespace: 'd',
labels: { e: 'f' },
annotations: { g: 'h' },
},
spec: { i: 'j' },
},
relations: [],
};
entityResponse = {
locationId: undefined,
entity: {
apiVersion: 'a',
kind: 'b',
metadata: {
uid: expect.anything(),
etag: expect.anything(),
generation: expect.anything(),
name: 'c',
namespace: 'd',
labels: { e: 'f' },
annotations: {
g: 'h',
},
},
spec: { i: 'j' },
},
};
});
it('manages locations', async () => {
const input: Location = {
id: 'dd12620d-0436-422f-93bd-929aa0788123',
type: 'a',
target: 'b',
};
const output: DbLocationsRowWithStatus = {
id: 'dd12620d-0436-422f-93bd-929aa0788123',
type: 'a',
target: 'b',
message: null,
status: null,
timestamp: null,
};
await db.transaction(async tx => await db.addLocation(tx, input));
const locations = await db.locations();
expect(locations).toEqual(
expect.arrayContaining([output, bootstrapLocation]),
);
const location = await db.location(
locations.find(l => l.type !== 'bootstrap')!.id,
);
expect(location).toEqual(output);
// If we add 2 new update log events,
// this should not result in location duplication
// due to incorrect join in DB
await db.addLocationUpdateLogEvent(
'dd12620d-0436-422f-93bd-929aa0788123',
DatabaseLocationUpdateLogStatus.SUCCESS,
);
// Have a second in-between
// To avoid having same timestamp on event
await new Promise(res => setTimeout(res, 1000));
await db.addLocationUpdateLogEvent(
'dd12620d-0436-422f-93bd-929aa0788123',
DatabaseLocationUpdateLogStatus.FAIL,
);
await expect(db.locations()).resolves.toEqual(
expect.arrayContaining([
bootstrapLocation,
{
...output,
status: DatabaseLocationUpdateLogStatus.FAIL,
timestamp: expect.anything(),
},
]),
);
await db.transaction(tx => db.removeLocation(tx, location.id));
await expect(db.locations()).resolves.toEqual([bootstrapLocation]);
await expect(db.location(location.id)).rejects.toThrow(/Found no location/);
});
it('refuses to remove the bootstrap location', async () => {
const input: Location = {
id: 'dd12620d-0436-422f-93bd-929aa0788123',
type: 'bootstrap',
target: 'bootstrap',
};
const output = await db.transaction(
async tx => await db.addLocation(tx, input),
);
await expect(
db.transaction(async tx => await db.removeLocation(tx, output.id)),
).rejects.toThrow(ConflictError);
});
describe('addEntities', () => {
it('happy path: adds entities to empty database', async () => {
const result = await db.transaction(tx =>
db.addEntities(tx, [entityRequest]),
);
expect(result).toEqual([entityResponse]);
});
it('rejects adding the same-named entity twice', async () => {
const req: DbEntityRequest[] = [
{
entity: {
apiVersion: 'av1',
kind: 'k1',
metadata: { name: 'n1', namespace: 'ns1' },
},
relations: [],
},
{
entity: {
apiVersion: 'av1',
kind: 'k1',
metadata: { name: 'n1', namespace: 'ns1' },
},
relations: [],
},
];
await expect(
db.transaction(tx => db.addEntities(tx, req)),
).rejects.toThrow(ConflictError);
});
it('rejects adding the almost-same-namespace entity twice', async () => {
const req: DbEntityRequest[] = [
{
entity: {
apiVersion: 'av1',
kind: 'k1',
metadata: { name: 'n1', namespace: 'ns1' },
},
relations: [],
},
{
entity: {
apiVersion: 'av1',
kind: 'k1',
metadata: { name: 'n1', namespace: 'nS1' },
},
relations: [],
},
];
await expect(
db.transaction(tx => db.addEntities(tx, req)),
).rejects.toThrow(ConflictError);
});
it('accepts adding the same-named entity twice if on different namespaces', async () => {
const req: DbEntityRequest[] = [
{
entity: {
apiVersion: 'av1',
kind: 'k1',
metadata: { name: 'n1', namespace: 'ns1' },
},
relations: [],
},
{
entity: {
apiVersion: 'av1',
kind: 'k1',
metadata: { name: 'n1', namespace: 'ns2' },
},
relations: [],
},
];
await expect(
db.transaction(tx => db.addEntities(tx, req)),
).resolves.toEqual([
{
entity: expect.objectContaining({
metadata: expect.objectContaining({
namespace: 'ns1',
uid: expect.any(String),
etag: expect.any(String),
generation: expect.any(Number),
}),
}),
},
{
entity: expect.objectContaining({
metadata: expect.objectContaining({
namespace: 'ns2',
uid: expect.any(String),
etag: expect.any(String),
generation: expect.any(Number),
}),
}),
},
]);
});
});
describe('locationHistory', () => {
it('outputs the history correctly', async () => {
const location: Location = {
id: 'dd12620d-0436-422f-93bd-929aa0788123',
type: 'a',
target: 'b',
};
await db.transaction(async tx => await db.addLocation(tx, location));
await db.addLocationUpdateLogEvent(
'dd12620d-0436-422f-93bd-929aa0788123',
DatabaseLocationUpdateLogStatus.SUCCESS,
);
await db.addLocationUpdateLogEvent(
'dd12620d-0436-422f-93bd-929aa0788123',
DatabaseLocationUpdateLogStatus.FAIL,
undefined,
'Something went wrong',
);
const result = await db.locationHistory(
'dd12620d-0436-422f-93bd-929aa0788123',
);
expect(result).toEqual(
expect.arrayContaining([
{
created_at: expect.anything(),
entity_name: null,
id: expect.anything(),
location_id: 'dd12620d-0436-422f-93bd-929aa0788123',
message: null,
status: DatabaseLocationUpdateLogStatus.SUCCESS,
},
{
created_at: expect.anything(),
entity_name: null,
id: expect.anything(),
location_id: 'dd12620d-0436-422f-93bd-929aa0788123',
message: 'Something went wrong',
status: DatabaseLocationUpdateLogStatus.FAIL,
},
]),
);
});
});
describe('updateEntity', () => {
it('can read and no-op-update an entity', async () => {
const [added] = await db.transaction(tx =>
db.addEntities(tx, [entityRequest]),
);
const updated = await db.transaction(tx =>
db.updateEntity(tx, { entity: added.entity, relations: [] }),
);
expect(updated.entity.apiVersion).toEqual(added.entity.apiVersion);
expect(updated.entity.kind).toEqual(added.entity.kind);
expect(updated.entity.metadata.etag).toEqual(added.entity.metadata.etag);
expect(updated.entity.metadata.generation).toEqual(
added.entity.metadata.generation,
);
expect(updated.entity.metadata.name).toEqual(added.entity.metadata.name);
expect(updated.entity.metadata.namespace).toEqual(
added.entity.metadata.namespace,
);
});
it('can update name if uid matches', async () => {
const [added] = await db.transaction(tx =>
db.addEntities(tx, [entityRequest]),
);
added.entity.metadata.name! = 'new!';
const updated = await db.transaction(tx =>
db.updateEntity(tx, { entity: added.entity, relations: [] }),
);
expect(updated.entity.metadata.name).toEqual('new!');
});
it('fails to update an entity if etag does not match', async () => {
const [added] = await db.transaction(tx =>
db.addEntities(tx, [entityRequest]),
);
await expect(
db.transaction(tx =>
db.updateEntity(
tx,
{ entity: added.entity, relations: [] },
'garbage',
),
),
).rejects.toThrow(ConflictError);
});
it('fails to update an entity if generation does not match', async () => {
const [added] = await db.transaction(tx =>
db.addEntities(tx, [entityRequest]),
);
await expect(
db.transaction(tx =>
db.updateEntity(
tx,
{ entity: added.entity, relations: [] },
undefined,
1e20,
),
),
).rejects.toThrow(ConflictError);
});
});
describe('entities', () => {
it('can get all entities with empty filters list', async () => {
const e1: Entity = {
apiVersion: 'a',
kind: 'k1',
metadata: { name: 'n' },
};
const e2: Entity = {
apiVersion: 'c',
kind: 'k2',
metadata: { name: 'n' },
spec: { c: null },
};
await db.transaction(async tx => {
await db.addEntities(tx, [
{ entity: e1, relations: [] },
{ entity: e2, relations: [] },
]);
});
const result = await db.transaction(async tx => db.entities(tx));
expect(result.entities.length).toEqual(2);
expect(result.entities).toEqual(
expect.arrayContaining([
{
locationId: undefined,
entity: expect.objectContaining({ kind: 'k1' }),
},
{
locationId: undefined,
entity: expect.objectContaining({ kind: 'k2' }),
},
]),
);
});
it('can get all specific entities for matching filters (naive case)', async () => {
const entities: Entity[] = [
{ apiVersion: 'a', kind: 'k1', metadata: { name: 'n' } },
{
apiVersion: 'a',
kind: 'k2',
metadata: { name: 'n' },
spec: { c: 'some' },
},
{
apiVersion: 'a',
kind: 'k3',
metadata: { name: 'n' },
spec: { c: null },
},
];
await db.transaction(async tx => {
await db.addEntities(
tx,
entities.map(entity => ({ entity, relations: [] })),
);
});
const response = await db.transaction(async tx =>
db.entities(tx, {
filter: basicEntityFilter({ kind: 'k2', 'spec.c': 'some' }),
}),
);
expect(response.entities).toEqual([
{
locationId: undefined,
entity: expect.objectContaining({ kind: 'k2' }),
},
]);
});
it('can get all specific entities for matching filters case insensitively', async () => {
const entities: Entity[] = [
{
apiVersion: 'A',
kind: 'K1',
metadata: { name: 'N' },
spec: { c: 'SOME' },
},
{
apiVersion: 'a',
kind: 'k2',
metadata: { name: 'n' },
spec: { c: 'Some' },
},
{
apiVersion: 'a',
kind: 'k3',
metadata: { name: 'n' },
spec: { c: 'somE' },
},
];
await db.transaction(async tx => {
await db.addEntities(
tx,
entities.map(entity => ({ entity, relations: [] })),
);
});
const rows = await db.transaction(async tx =>
db.entities(tx, {
filter: basicEntityFilter({ ApiVersioN: 'A', 'spEc.C': 'some' }),
}),
);
expect(rows.entities.length).toEqual(3);
expect(rows.entities).toEqual(
expect.arrayContaining([
{
locationId: undefined,
entity: expect.objectContaining({ kind: 'K1' }),
},
{
locationId: undefined,
entity: expect.objectContaining({ kind: 'k2' }),
},
{
locationId: undefined,
entity: expect.objectContaining({ kind: 'k3' }),
},
]),
);
});
it('can get all specific entities for matching existence filters', async () => {
const entities: Entity[] = [
{
apiVersion: 'A',
kind: 'K1',
metadata: {
name: 'N',
annotations: {
foo: 'bar',
},
},
spec: { c: 'SOME' },
},
{
apiVersion: 'a',
kind: 'k2',
metadata: {
name: 'N',
annotations: {
foo: 'bar',
},
},
spec: { c: 'Some' },
},
{
apiVersion: 'a',
kind: 'k3',
metadata: { name: 'n' },
spec: { c: 'somE' },
},
];
await db.transaction(async tx => {
await db.addEntities(
tx,
entities.map(entity => ({ entity, relations: [] })),
);
});
const existRows = await db.transaction(async tx =>
db.entities(tx, {
filter: {
anyOf: [
{
allOf: [{ key: 'metadata.annotations.foo' }],
},
],
},
}),
);
expect(existRows.entities.length).toEqual(2);
expect(existRows.entities).toEqual(
expect.arrayContaining([
{
locationId: undefined,
entity: expect.objectContaining({ kind: 'K1' }),
},
{
locationId: undefined,
entity: expect.objectContaining({ kind: 'k2' }),
},
]),
);
});
});
describe('setRelations', () => {
it('adds a relation for an entity', async () => {
const mockRelations = [
{
source: {
kind: entityRequest.entity.kind,
namespace: entityRequest.entity.metadata.namespace!,
name: entityRequest.entity.metadata.name,
},
target: {
kind: 'component',
namespace: 'asd',
name: 'bleb',
},
type: 'child',
},
];
const entityId = await db.transaction(async tx => {
const [{ entity }] = await db.addEntities(tx, [entityRequest]);
await db.setRelations(tx, entity?.metadata?.uid!, mockRelations);
return entity.metadata.uid;
});
const returnedEntity1 = await db.transaction(tx =>
db.entityByUid(tx, entityId!),
);
expect(returnedEntity1?.entity.relations).toEqual([
{ target: mockRelations[0].target, type: 'child' },
]);
const returnedEntity2 = await db.transaction(tx =>
db.entityByName(tx, mockRelations[0].source),
);
expect(returnedEntity2?.entity.relations).toEqual([
{ target: mockRelations[0].target, type: 'child' },
]);
const { entities } = await db.transaction(tx => db.entities(tx));
const [returnedEntity3] = entities;
expect(returnedEntity3?.entity.relations).toEqual([
{ target: mockRelations[0].target, type: 'child' },
]);
});
function makeRelation(source: string, type: string, target: string) {
return {
source: parseEntityRef(source, {
defaultKind: 'x',
defaultNamespace: 'x',
}),
type,
target: parseEntityRef(target, {
defaultKind: 'x',
defaultNamespace: 'x',
}),
};
}
it('should not allow setting relations on nonexistent entities', async () => {
await expect(
db.transaction(async tx => {
await db.setRelations(tx, 'nonexistent', [
makeRelation('a:b/c', 'rel1', 'x:y/z'),
]);
}),
).rejects.toThrow(/constraint failed/);
});
it('should allow setting relations on nonexistent entities without any relations', async () => {
await expect(
db.transaction(async tx => {
await db.setRelations(tx, 'nonexistent', []);
}),
).resolves.toBeUndefined();
});
it('adds multiple relations for entities', async () => {
const entity1 = {
apiVersion: 'v1',
kind: 'a',
metadata: {
name: 'c',
namespace: 'b',
},
};
const entity2 = {
apiVersion: 'v1',
kind: 'x',
metadata: {
name: 'z',
namespace: 'y',
},
};
const fromEntity1 = [
makeRelation('a:b/c', 'rel1', 'x:y/z'),
makeRelation('x:y/z', 'rel2', 'a:b/c'),
makeRelation('a:b/c', 'rel2', 'x:y/z'),
];
const fromEntity2 = [
makeRelation('a:b/c', 'rel4', 'x:y/z'),
makeRelation('a:b/c', 'rel5', 'x:y/z'),
makeRelation('x:y/z', 'rel6', 'a:b/c'),
// relations don't have to reference the originating entity, so this should be fine, but not show up
makeRelation('g:h/i', 'rel8', 'd:e/f'),
];
const { id2: secondEntityId } = await db.transaction(async tx => {
const [{ entity: e1 }, { entity: e2 }] = await db.addEntities(tx, [
{ entity: entity1, relations: [] },
{ entity: entity2, relations: [] },
]);
const id1 = e1?.metadata?.uid!;
const id2 = e2?.metadata?.uid!;
await db.setRelations(tx, id1, fromEntity1);
await db.setRelations(tx, id2, fromEntity2);
return { id1, id2 };
});
const res = await db.transaction(tx => db.entities(tx));
expect(
res.entities.map(r => ({
name: r.entity.metadata.name,
relations: r.entity.relations,
})),
).toEqual([
{
name: 'c',
relations: [
{
type: 'rel1',
target: { kind: 'x', namespace: 'y', name: 'z' },
},
{
type: 'rel2',
target: { kind: 'x', namespace: 'y', name: 'z' },
},
{
type: 'rel4',
target: { kind: 'x', namespace: 'y', name: 'z' },
},
{
type: 'rel5',
target: { kind: 'x', namespace: 'y', name: 'z' },
},
],
},
{
name: 'z',
relations: [
{
type: 'rel2',
target: { kind: 'a', namespace: 'b', name: 'c' },
},
{
type: 'rel6',
target: { kind: 'a', namespace: 'b', name: 'c' },
},
],
},
]);
await db.transaction(tx => db.removeEntityByUid(tx, secondEntityId));
const res2 = await db.transaction(tx => db.entities(tx));
expect(
res2.entities.map(r => ({
name: r.entity.metadata.name,
relations: r.entity.relations,
})),
).toEqual([
{
name: 'c',
relations: [
{
type: 'rel1',
target: { kind: 'x', namespace: 'y', name: 'z' },
},
{
type: 'rel2',
target: { kind: 'x', namespace: 'y', name: 'z' },
},
],
},
]);
});
});
describe('entityByName', () => {
it('can get entities case insensitively', async () => {
const entities: Entity[] = [
{
apiVersion: 'a',
kind: 'k1',
metadata: { name: 'n' },
},
{
apiVersion: 'B',
kind: 'K2',
metadata: { name: 'N', namespace: 'NS' },
},
];
await db.transaction(async tx => {
await db.addEntities(
tx,
entities.map(entity => ({ entity, relations: [] })),
);
});
const e1 = await db.transaction(async tx =>
db.entityByName(tx, { kind: 'k1', namespace: 'default', name: 'n' }),
);
expect(e1!.entity.metadata.name).toEqual('n');
const e2 = await db.transaction(async tx =>
db.entityByName(tx, { kind: 'k2', namespace: 'nS', name: 'n' }),
);
expect(e2!.entity.metadata.name).toEqual('N');
const e3 = await db.transaction(async tx =>
db.entityByName(tx, { kind: 'unknown', namespace: 'nS', name: 'n' }),
);
expect(e3).toBeUndefined();
});
});
});
plugins/catalog-backend/src/legacy/database/CommonDatabase.ts
-624
View File
@@ -1,624 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConflictError, InputError, NotFoundError } from '@backstage/errors';
import {
Entity,
EntityName,
EntityRelationSpec,
ENTITY_DEFAULT_NAMESPACE,
ENTITY_META_GENERATED_FIELDS,
generateEntityEtag,
generateEntityUid,
Location,
parseEntityName,
} from '@backstage/catalog-model';
import { Knex } from 'knex';
import lodash from 'lodash';
import type { Logger } from 'winston';
import { buildEntitySearch } from './search';
import {
Database,
DatabaseLocationUpdateLogEvent,
DatabaseLocationUpdateLogStatus,
DbEntitiesRelationsRow,
DbEntitiesRequest,
DbEntitiesResponse,
DbEntitiesRow,
DbEntitiesSearchRow,
DbEntityRequest,
DbEntityResponse,
DbLocationsRow,
DbLocationsRowWithStatus,
DbPageInfo,
Transaction,
} from './types';
import { EntityPagination, EntitiesSearchFilter } from '../../catalog/types';
type LegacyEntityFilter = {
anyOf: { allOf: EntitiesSearchFilter[] }[];
};
// The number of items that are sent per batch to the database layer, when
// doing .batchInsert calls to knex. This needs to be low enough to not cause
// errors in the underlying engine due to exceeding query limits, but large
// enough to get the speed benefits.
const BATCH_SIZE = 50;
/**
* The core database implementation..
* @deprecated This was part of the legacy catalog engin
*/
export class CommonDatabase implements Database {
constructor(
private readonly database: Knex,
private readonly logger: Logger,
) {}
async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {
try {
let result: T | undefined = undefined;
await this.database.transaction(
async tx => {
// We can't return here, as knex swallows the return type in case the transaction is rolled back:
// https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136
result = await fn(tx);
},
{
// If we explicitly trigger a rollback, don't fail.
doNotRejectOnRollback: true,
},
);
return result!;
} catch (e) {
this.logger.debug(`Error during transaction, ${e}`);
if (
/SQLITE_CONSTRAINT: UNIQUE/.test(e.message) ||
/unique constraint/.test(e.message)
) {
throw new ConflictError(`Rejected due to a conflicting entity`, e);
}
throw e;
}
}
async addEntities(
txOpaque: Transaction,
request: DbEntityRequest[],
): Promise<DbEntityResponse[]> {
const tx = txOpaque as Knex.Transaction;
const result: DbEntityResponse[] = [];
const entityRows: DbEntitiesRow[] = [];
const relationRows: DbEntitiesRelationsRow[] = [];
const searchRows: DbEntitiesSearchRow[] = [];
for (const { entity, relations, locationId } of request) {
if (entity.metadata.uid !== undefined) {
throw new InputError('May not specify uid for new entities');
} else if (entity.metadata.etag !== undefined) {
throw new InputError('May not specify etag for new entities');
} else if (entity.metadata.generation !== undefined) {
throw new InputError('May not specify generation for new entities');
} else if (entity.relations !== undefined) {
throw new InputError('May not specify relations for new entities');
}
const uid = generateEntityUid();
const etag = generateEntityEtag();
const generation = 1;
const newEntity = {
...entity,
metadata: {
...entity.metadata,
uid,
etag,
generation,
},
};
result.push({ entity: newEntity, locationId });
entityRows.push(this.toEntityRow(locationId, newEntity));
relationRows.push(...this.toRelationRows(uid, relations));
searchRows.push(...buildEntitySearch(uid, newEntity));
}
await tx.batchInsert('entities', entityRows, BATCH_SIZE);
await tx.batchInsert('entities_relations', relationRows, BATCH_SIZE);
await tx.batchInsert('entities_search', searchRows, BATCH_SIZE);
return result;
}
async updateEntity(
txOpaque: Transaction,
request: DbEntityRequest,
matchingEtag?: string,
matchingGeneration?: number,
): Promise<DbEntityResponse> {
const tx = txOpaque as Knex.Transaction;
const { uid } = request.entity.metadata;
if (!uid) {
throw new InputError('Must specify uid when updating entities');
}
// Find existing entity
const oldRows = await tx<DbEntitiesRow>('entities')
.where({ id: uid })
.select();
if (oldRows.length !== 1) {
throw new NotFoundError('No matching entity found');
}
const etag = oldRows[0].etag;
const generation = Number(oldRows[0].generation);
// Validate the old entity. The Number cast is here because sqlite reads it
// as a string, no matter what the table actually says.
if (matchingEtag && matchingEtag !== etag) {
throw new ConflictError(
`Etag mismatch, expected="${matchingEtag}" found="${etag}"`,
);
}
if (matchingGeneration && matchingGeneration !== generation) {
throw new ConflictError(
`Generation mismatch, expected="${matchingGeneration}" found="${generation}"`,
);
}
// Store the updated entity; select on the old etag to ensure that we do
// not lose to another writer
const newRow = this.toEntityRow(request.locationId, request.entity);
const updatedRows = await tx<DbEntitiesRow>('entities')
.where({ id: uid, etag })
.update(newRow);
if (updatedRows !== 1) {
throw new ConflictError(`Failed to update entity`);
}
const relationRows = this.toRelationRows(uid, request.relations);
await tx<DbEntitiesRelationsRow>('entities_relations')
.where({ originating_entity_id: uid })
.del();
await tx.batchInsert('entities_relations', relationRows, BATCH_SIZE);
try {
const entries = buildEntitySearch(uid, request.entity);
await tx<DbEntitiesSearchRow>('entities_search')
.where({ entity_id: uid })
.del();
await tx.batchInsert('entities_search', entries, BATCH_SIZE);
} catch {
// ignore intentionally - if this happens, the entity was deleted before
// we got around to writing the entries
}
return request;
}
async entities(
txOpaque: Transaction,
request?: DbEntitiesRequest,
): Promise<DbEntitiesResponse> {
const tx = txOpaque as Knex.Transaction;
let entitiesQuery = tx<DbEntitiesRow>('entities');
if (
request?.filter &&
(request.filter.hasOwnProperty('key') ||
request.filter.hasOwnProperty('allOf') ||
request.filter.hasOwnProperty('not'))
) {
throw new Error(
'Filters for the legacy CommonDatabase must obey the { anyOf: [{ allOf: [] }] } format.',
);
}
for (const singleFilter of (request?.filter as LegacyEntityFilter)?.anyOf ??
[]) {
entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() {
for (const filter of singleFilter.allOf) {
if (
filter.hasOwnProperty('anyOf') ||
filter.hasOwnProperty('allOf') ||
filter.hasOwnProperty('not')
) {
throw new Error(
'Nested filters are not supported in the legacy CommonDatabase',
);
}
const { key, values } = filter;
// NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
// make a lot of sense. However, it had abysmal performance on sqlite
// when datasets grew large, so we're using IN instead.
const matchQuery = tx<DbEntitiesSearchRow>('entities_search')
.select('entity_id')
.where(function keyFilter() {
this.andWhere({ key: key.toLowerCase() });
if (values) {
if (values.length === 1) {
this.andWhere({ value: values[0].toLowerCase() });
} else if (values.length > 1) {
this.andWhere(
'value',
'in',
values.map(v => v.toLowerCase()),
);
}
}
});
this.andWhere('id', 'in', matchQuery);
}
});
}
entitiesQuery = entitiesQuery
.select('entities.*')
.orderBy('full_name', 'asc');
const { limit, offset } = parsePagination(request?.pagination);
if (limit !== undefined) {
entitiesQuery = entitiesQuery.limit(limit + 1);
}
if (offset !== undefined) {
entitiesQuery = entitiesQuery.offset(offset);
}
let rows = await entitiesQuery;
let pageInfo: DbPageInfo;
if (limit === undefined || rows.length <= limit) {
pageInfo = { hasNextPage: false };
} else {
rows = rows.slice(0, -1);
pageInfo = {
hasNextPage: true,
endCursor: stringifyPagination({
limit,
offset: (offset ?? 0) + limit,
}),
};
}
return {
entities: await this.toEntityResponses(tx, rows),
pageInfo,
};
}
async entityByName(
txOpaque: Transaction,
name: EntityName,
): Promise<DbEntityResponse | undefined> {
const tx = txOpaque as Knex.Transaction;
const rows = await tx<DbEntitiesRow>('entities')
.where({
full_name: `${name.kind}:${name.namespace}/${name.name}`.toLowerCase(),
})
.select();
if (rows.length !== 1) {
return undefined;
}
return this.toEntityResponses(tx, rows).then(r => r[0]);
}
async entityByUid(
txOpaque: Transaction,
uid: string,
): Promise<DbEntityResponse | undefined> {
const tx = txOpaque as Knex.Transaction;
const rows = await tx<DbEntitiesRow>('entities')
.where({ id: uid })
.select();
if (rows.length !== 1) {
return undefined;
}
return this.toEntityResponses(tx, rows).then(r => r[0]);
}
async removeEntityByUid(txOpaque: Transaction, uid: string): Promise<void> {
const tx = txOpaque as Knex.Transaction;
const result = await tx<DbEntitiesRow>('entities').where({ id: uid }).del();
if (!result) {
throw new NotFoundError(`Found no entity with ID ${uid}`);
}
}
async setRelations(
txOpaque: Transaction,
originatingEntityId: string,
relations: EntityRelationSpec[],
): Promise<void> {
const tx = txOpaque as Knex.Transaction;
const relationRows = this.toRelationRows(originatingEntityId, relations);
await tx<DbEntitiesRelationsRow>('entities_relations')
.where({ originating_entity_id: originatingEntityId })
.del();
await tx.batchInsert('entities_relations', relationRows, BATCH_SIZE);
}
async addLocation(
txOpaque: Transaction,
location: Location,
): Promise<DbLocationsRow> {
const tx = txOpaque as Knex.Transaction;
const row: DbLocationsRow = {
id: location.id,
type: location.type,
target: location.target,
};
await tx<DbLocationsRow>('locations').insert(row);
return row;
}
async removeLocation(txOpaque: Transaction, id: string): Promise<void> {
const tx = txOpaque as Knex.Transaction;
const locations = await tx<DbLocationsRow>('locations')
.where({ id })
.select();
if (!locations.length) {
throw new NotFoundError(`Found no location with ID ${id}`);
}
if (locations[0].type === 'bootstrap') {
throw new ConflictError('You may not delete the bootstrap location.');
}
await tx<DbEntitiesRow>('entities')
.where({ location_id: id })
.update({ location_id: null });
await tx<DbLocationsRow>('locations').where({ id }).del();
}
async location(id: string): Promise<DbLocationsRowWithStatus> {
const items = await this.database<DbLocationsRowWithStatus>('locations')
.where('locations.id', id)
.leftOuterJoin(
'location_update_log_latest',
'locations.id',
'location_update_log_latest.location_id',
)
.select('locations.*', {
status: 'location_update_log_latest.status',
timestamp: 'location_update_log_latest.created_at',
message: 'location_update_log_latest.message',
});
if (!items.length) {
throw new NotFoundError(`Found no location with ID ${id}`);
}
return items[0];
}
async locations(): Promise<DbLocationsRowWithStatus[]> {
const locations = await this.database('locations')
.leftOuterJoin(
'location_update_log_latest',
'locations.id',
'location_update_log_latest.location_id',
)
.select('locations.*', {
status: 'location_update_log_latest.status',
timestamp: 'location_update_log_latest.created_at',
message: 'location_update_log_latest.message',
});
return locations;
}
async locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]> {
const result = await this.database<DatabaseLocationUpdateLogEvent>(
'location_update_log',
)
.where('location_id', id)
.orderBy('created_at', 'desc')
.limit(10)
.select();
return result;
}
async addLocationUpdateLogEvent(
locationId: string,
status: DatabaseLocationUpdateLogStatus,
entityName?: string | string[],
message?: string,
): Promise<void> {
// Remove log entries older than a day
const cutoff = new Date();
cutoff.setDate(cutoff.getDate() - 1);
await this.database<DatabaseLocationUpdateLogEvent>('location_update_log')
.where('created_at', '<', cutoff.toISOString())
.del();
const items: Partial<DatabaseLocationUpdateLogEvent>[] = [entityName]
.flat()
.map(n => ({
status,
location_id: locationId,
entity_name: n,
message,
}));
for (const chunk of lodash.chunk(items, BATCH_SIZE)) {
await this.database<DatabaseLocationUpdateLogEvent>(
'location_update_log',
).insert(chunk);
}
}
private toEntityRow(
locationId: string | undefined,
entity: Entity,
): DbEntitiesRow {
const lowerKind = entity.kind.toLowerCase();
const lowerNamespace = (
entity.metadata.namespace || ENTITY_DEFAULT_NAMESPACE
).toLowerCase();
const lowerName = entity.metadata.name.toLowerCase();
const data = {
...entity,
metadata: lodash.omit(entity.metadata, ...ENTITY_META_GENERATED_FIELDS),
};
return {
id: entity.metadata.uid!,
location_id: locationId || null,
etag: entity.metadata.etag!,
generation: entity.metadata.generation!,
full_name: `${lowerKind}:${lowerNamespace}/${lowerName}`,
data: JSON.stringify(data),
};
}
private toRelationRows(
originatingEntityId: string,
relations: EntityRelationSpec[],
): DbEntitiesRelationsRow[] {
const serializeName = (e: EntityName) =>
`${e.kind}:${e.namespace}/${e.name}`.toLowerCase();
const rows = relations.map(({ source, target, type }) => ({
originating_entity_id: originatingEntityId,
source_full_name: serializeName(source),
target_full_name: serializeName(target),
type,
}));
return deduplicateRelations(rows);
}
private async toEntityResponses(
tx: Knex.Transaction,
rows: DbEntitiesRow[],
): Promise<DbEntityResponse[]> {
// TODO(Rugvip): This is here because it's simple for now, but we likely
// need to refactor this to be more efficient or introduce pagination.
const relations = await this.getRelationsPerFullName(
tx,
rows.map(r => r.full_name),
);
const result = new Array<DbEntityResponse>();
for (const row of rows) {
const entity = JSON.parse(row.data) as Entity;
entity.metadata.uid = row.id;
entity.metadata.etag = row.etag;
entity.metadata.generation = Number(row.generation); // cast due to sqlite
entity.relations = (relations[row.full_name] ?? []).map(r => ({
target: parseEntityName(r.target_full_name),
type: r.type,
}));
result.push({
locationId: row.location_id || undefined,
entity,
});
}
return result;
}
// Returns a mapping from e.g. component:default/foo to the relations whose
// source_full_name matches that.
private async getRelationsPerFullName(
tx: Knex.Transaction,
sourceFullNames: string[],
): Promise<Record<string, DbEntitiesRelationsRow[]>> {
const batches = lodash.chunk(lodash.uniq(sourceFullNames), 500);
const relations = new Array<DbEntitiesRelationsRow>();
for (const batch of batches) {
relations.push(
...(await tx<DbEntitiesRelationsRow>('entities_relations')
.whereIn('source_full_name', batch)
.orderBy(['type', 'target_full_name'])
.select()),
);
}
return lodash.groupBy(
deduplicateRelations(relations),
r => r.source_full_name,
);
}
}
function parsePagination(input?: EntityPagination): {
limit?: number;
offset?: number;
} {
if (!input) {
return {};
}
let { limit, offset } = input;
if (input.after !== undefined) {
let cursor;
try {
const json = Buffer.from(input.after, 'base64').toString('utf8');
cursor = JSON.parse(json);
} catch {
throw new InputError('Malformed after cursor, could not be parsed');
}
if (cursor.limit !== undefined) {
if (!Number.isInteger(cursor.limit)) {
throw new InputError('Malformed after cursor, limit was not an number');
}
limit = cursor.limit;
}
if (cursor.offset !== undefined) {
if (!Number.isInteger(cursor.offset)) {
throw new InputError('Malformed after cursor, offset was not a number');
}
offset = cursor.offset;
}
}
return { limit, offset };
}
function stringifyPagination(input: { limit: number; offset: number }) {
const json = JSON.stringify({ limit: input.limit, offset: input.offset });
const base64 = Buffer.from(json, 'utf8').toString('base64');
return base64;
}
function deduplicateRelations(
rows: DbEntitiesRelationsRow[],
): DbEntitiesRelationsRow[] {
return lodash.uniqBy(
rows,
r => `${r.source_full_name}:${r.target_full_name}:${r.type}`,
);
}
plugins/catalog-backend/src/legacy/database/DatabaseManager.ts
-109
View File
@@ -1,109 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common';
import knexFactory, { Knex } from 'knex';
import { v4 as uuidv4 } from 'uuid';
import { Logger } from 'winston';
import { CommonDatabase } from './CommonDatabase';
import { Database } from './types';
const migrationsDir = resolvePackagePath(
'@backstage/plugin-catalog-backend',
'migrations',
);
/** @deprecated This was part of the legacy catalog engine */
export type CreateDatabaseOptions = {
logger: Logger;
};
const defaultOptions: CreateDatabaseOptions = {
logger: getVoidLogger(),
};
/** @deprecated This was part of the legacy catalog engine */
export class DatabaseManager {
public static async createDatabase(
knex: Knex,
options: Partial<CreateDatabaseOptions> = {},
): Promise<Database> {
await knex.migrate.latest({
directory: migrationsDir,
});
const { logger } = { ...defaultOptions, ...options };
return new CommonDatabase(knex, logger);
}
public static async createInMemoryDatabase(): Promise<Database> {
const knex = await this.createInMemoryDatabaseConnection();
return await this.createDatabase(knex);
}
public static async createInMemoryDatabaseConnection(): Promise<Knex> {
const knex = knexFactory({
client: 'sqlite3',
connection: ':memory:',
useNullAsDefault: true,
});
knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
resource.run('PRAGMA foreign_keys = ON', () => {});
});
return knex;
}
public static async createTestDatabase(): Promise<Database> {
const knex = await this.createTestDatabaseConnection();
return await this.createDatabase(knex);
}
public static async createTestDatabaseConnection(): Promise<Knex> {
const config: Knex.Config<any> = {
/*
client: 'pg',
connection: {
host: 'localhost',
user: 'postgres',
password: 'postgres',
},
*/
client: 'sqlite3',
connection: ':memory:',
useNullAsDefault: true,
};
let knex = knexFactory(config);
if (typeof config.connection !== 'string') {
const tempDbName = `d${uuidv4().replace(/-/g, '')}`;
await knex.raw(`CREATE DATABASE ${tempDbName};`);
knex = knexFactory({
...config,
connection: {
...config.connection,
database: tempDbName,
},
});
}
knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
resource.run('PRAGMA foreign_keys = ON', () => {});
});
return knex;
}
}
plugins/catalog-backend/src/legacy/database/index.ts
-32
View File
@@ -1,32 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { CommonDatabase } from './CommonDatabase';
export { DatabaseManager } from './DatabaseManager';
export type { CreateDatabaseOptions } from './DatabaseManager';
export type {
Database,
DbEntityRequest,
DbEntityResponse,
Transaction,
DbEntitiesRequest,
DbEntitiesResponse,
DbLocationsRowWithStatus,
DatabaseLocationUpdateLogEvent,
DbLocationsRow,
DatabaseLocationUpdateLogStatus,
DbPageInfo,
} from './types';
plugins/catalog-backend/src/legacy/database/search.test.ts
-138
View File
@@ -1,138 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
import { buildEntitySearch, mapToRows, traverse } from './search';
describe('search', () => {
describe('traverse', () => {
it('expands lists of strings to several rows', () => {
const input = { a: ['b', 'c', 'd'] };
const output = traverse(input);
expect(output).toEqual([
{ key: 'a', value: 'b' },
{ key: 'a.b', value: true },
{ key: 'a', value: 'c' },
{ key: 'a.c', value: true },
{ key: 'a', value: 'd' },
{ key: 'a.d', value: true },
]);
});
it('expands objects', () => {
const input = { a: { b: { c: 'd' }, e: 'f' } };
const output = traverse(input);
expect(output).toEqual([
{ key: 'a.b.c', value: 'd' },
{ key: 'a.e', value: 'f' },
]);
});
it('expands list of objects', () => {
const input = { root: { list: [{ a: 1 }, { a: 2 }] } };
const output = traverse(input);
expect(output).toEqual([
{ key: 'root.list.a', value: 1 },
{ key: 'root.list.a', value: 2 },
]);
});
it('skips over special keys', () => {
const input = {
a: 'a',
metadata: {
b: 'b',
name: 'name',
namespace: 'namespace',
uid: 'uid',
etag: 'etag',
generation: 'generation',
c: 'c',
},
d: 'd',
};
const output = traverse(input);
expect(output).toEqual([
{ key: 'a', value: 'a' },
{ key: 'metadata.b', value: 'b' },
{ key: 'metadata.c', value: 'c' },
{ key: 'd', value: 'd' },
]);
});
});
describe('mapToRows', () => {
it('converts base types to strings or null', () => {
const input = [
{ key: 'a', value: true },
{ key: 'b', value: false },
{ key: 'c', value: 7 },
{ key: 'd', value: 'string' },
{ key: 'e', value: null },
{ key: 'f', value: undefined },
];
const output = mapToRows(input, 'eid');
expect(output).toEqual([
{ entity_id: 'eid', key: 'a', value: 'true' },
{ entity_id: 'eid', key: 'b', value: 'false' },
{ entity_id: 'eid', key: 'c', value: '7' },
{ entity_id: 'eid', key: 'd', value: 'string' },
{ entity_id: 'eid', key: 'e', value: null },
{ entity_id: 'eid', key: 'f', value: null },
]);
});
it('emits lowercase version of keys and values', () => {
const input = [{ key: 'fOo', value: 'BaR' }];
const output = mapToRows(input, 'eid');
expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: 'bar' }]);
});
it('skips very large keys', () => {
const input = [{ key: 'a'.repeat(10000), value: 'foo' }];
const output = mapToRows(input, 'eid');
expect(output).toEqual([]);
});
it('skips very large values', () => {
const input = [{ key: 'foo', value: 'a'.repeat(10000) }];
const output = mapToRows(input, 'eid');
expect(output).toEqual([]);
});
});
describe('buildEntitySearch', () => {
it('adds special keys even if missing', () => {
const input: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: { name: 'n' },
};
expect(buildEntitySearch('eid', input)).toEqual([
{ entity_id: 'eid', key: 'apiversion', value: 'a' },
{ entity_id: 'eid', key: 'kind', value: 'b' },
{ entity_id: 'eid', key: 'metadata.name', value: 'n' },
{ entity_id: 'eid', key: 'metadata.namespace', value: null },
{ entity_id: 'eid', key: 'metadata.uid', value: null },
{
entity_id: 'eid',
key: 'metadata.namespace',
value: ENTITY_DEFAULT_NAMESPACE,
},
]);
});
});
});
plugins/catalog-backend/src/legacy/database/search.ts
-176
View File
@@ -1,176 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
import type { DbEntitiesSearchRow } from './types';
// These are excluded in the generic loop, either because they do not make sense
// to index, or because they are special-case always inserted whether they are
// null or not
const SPECIAL_KEYS = [
'metadata.name',
'metadata.namespace',
'metadata.uid',
'metadata.etag',
'metadata.generation',
];
// The maximum length allowed for search values. These columns are indexed, and
// database engines do not like to index on massive values. For example,
// postgres will balk after 8191 byte line sizes.
const MAX_KEY_LENGTH = 200;
const MAX_VALUE_LENGTH = 200;
type Kv = {
key: string;
value: unknown;
};
// Helper for traversing through a nested structure and outputting a list of
// path->value entries of the leaves.
//
// For example, this yaml structure
//
// a: 1
// b:
// c: null
// e: [f, g]
// h:
// - i: 1
// j: k
// - i: 2
// j: l
//
// will result in
//
// "a", 1
// "b.c", null
// "b.e": "f"
// "b.e.f": true
// "b.e": "g"
// "b.e.g": true
// "h.i": 1
// "h.j": "k"
// "h.i": 2
// "h.j": "l"
export function traverse(root: unknown): Kv[] {
const output: Kv[] = [];
function visit(path: string, current: unknown) {
if (SPECIAL_KEYS.includes(path)) {
return;
}
// empty or scalar
if (
current === undefined ||
current === null ||
['string', 'number', 'boolean'].includes(typeof current)
) {
output.push({ key: path, value: current });
return;
}
// unknown
if (typeof current !== 'object') {
return;
}
// array
if (Array.isArray(current)) {
for (const item of current) {
// NOTE(freben): The reason that these are output in two different ways,
// is to support use cases where you want to express that MORE than one
// tag is present in a list. Since the EntityFilters structure is a
// record, you can't have several entries of the same key. Therefore
// you will have to match on
//
// { "a.b": ["true"], "a.c": ["true"] }
//
// rather than
//
// { "a": ["b", "c"] }
//
// because the latter means EITHER b or c has to be present.
visit(path, item);
if (typeof item === 'string') {
output.push({ key: `${path}.${item}`, value: true });
}
}
return;
}
// object
for (const [key, value] of Object.entries(current!)) {
visit(path ? `${path}.${key}` : key, value);
}
}
visit('', root);
return output;
}
// Translates a number of raw data rows to search table rows
export function mapToRows(
input: Kv[],
entityId: string,
): DbEntitiesSearchRow[] {
const result: DbEntitiesSearchRow[] = [];
for (const { key: rawKey, value: rawValue } of input) {
const key = rawKey.toLowerCase();
if (rawValue === undefined || rawValue === null) {
result.push({ entity_id: entityId, key, value: null });
} else {
const value = String(rawValue).toLowerCase();
if (key.length <= MAX_KEY_LENGTH && value.length <= MAX_VALUE_LENGTH) {
result.push({ entity_id: entityId, key, value });
}
}
}
return result;
}
/**
* Generates all of the search rows that are relevant for this entity.
*
* @param entityId - The uid of the entity
* @param entity - The entity
* @returns A list of entity search rows
*/
export function buildEntitySearch(
entityId: string,
entity: Entity,
): DbEntitiesSearchRow[] {
// Visit the entire structure recursively
const raw = traverse(entity);
// Start with some special keys that are always present because you want to
// be able to easily search for null specifically
raw.push({ key: 'metadata.name', value: entity.metadata.name });
raw.push({ key: 'metadata.namespace', value: entity.metadata.namespace });
raw.push({ key: 'metadata.uid', value: entity.metadata.uid });
// Namespace not specified has the default value "default", so we want to
// match on that as well
if (!entity.metadata.namespace) {
raw.push({ key: 'metadata.namespace', value: ENTITY_DEFAULT_NAMESPACE });
}
return mapToRows(raw, entityId);
}
plugins/catalog-backend/src/legacy/database/types.ts
-223
View File
@@ -1,223 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type {
Entity,
EntityName,
EntityRelationSpec,
Location,
} from '@backstage/catalog-model';
import { EntityFilter, EntityPagination } from '../../catalog/types';
/** @deprecated This was part of the legacy catalog engine */
export type DbEntitiesRow = {
id: string;
location_id: string | null;
etag: string;
generation: number;
full_name: string;
data: string;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntityRequest = {
locationId?: string;
entity: Entity;
relations: EntityRelationSpec[];
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntitiesRequest = {
filter?: EntityFilter;
pagination?: EntityPagination;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntitiesResponse = {
entities: DbEntityResponse[];
pageInfo: DbPageInfo;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbPageInfo =
| {
hasNextPage: false;
}
| {
hasNextPage: true;
endCursor: string;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntityResponse = {
locationId?: string;
entity: Entity;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntitiesRelationsRow = {
originating_entity_id: string;
source_full_name: string;
type: string;
target_full_name: string;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntitiesSearchRow = {
entity_id: string;
key: string;
value: string | null;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbLocationsRow = {
id: string;
type: string;
target: string;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbLocationsRowWithStatus = DbLocationsRow & {
status: string | null;
timestamp: string | null;
message: string | null;
};
export enum DatabaseLocationUpdateLogStatus {
FAIL = 'fail',
SUCCESS = 'success',
}
/** @deprecated This was part of the legacy catalog engine */
export type DatabaseLocationUpdateLogEvent = {
id: string;
status: DatabaseLocationUpdateLogStatus;
location_id: string;
entity_name: string;
created_at?: string;
message?: string;
};
/**
* An abstraction for transactions of the underlying database technology.
*
* @deprecated This was part of the legacy catalog engine
*/
export type Transaction = {
rollback(): Promise<unknown>;
};
/**
* An abstraction on top of the underlying database, wrapping the basic CRUD
* needs.
* @deprecated This was part of the legacy catalog engine
*/
export type Database = {
/**
* Runs a transaction.
*
* The callback is expected to make calls back into this class. When it
* completes, the transaction is closed.
*
* @param fn - The callback that implements the transaction
*/
transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T>;
/**
* Adds a set of new entities to the catalog.
*
* @param tx - An ongoing transaction
* @param request - The entities being added
*/
addEntities(
tx: Transaction,
request: DbEntityRequest[],
): Promise<DbEntityResponse[]>;
/**
* Updates an existing entity in the catalog.
*
* The given entity must contain an uid to identify an already stored entity
* in the catalog. If it is missing or if no matching entity is found, the
* operation fails.
*
* If matchingEtag or matchingGeneration are given, they are taken into
* account. Attempts to update a matching entity, but where the etag and/or
* generation are not equal to the passed values, will fail.
*
* @param tx - An ongoing transaction
* @param request - The entity being updated
* @param matchingEtag - If specified, reject with ConflictError if not
* matching the entry in the database
* @param matchingGeneration - If specified, reject with ConflictError if not
* matching the entry in the database
* @returns The updated entity
*/
updateEntity(
tx: Transaction,
request: DbEntityRequest,
matchingEtag?: string,
matchingGeneration?: number,
): Promise<DbEntityResponse>;
entities(
tx: Transaction,
request?: DbEntitiesRequest,
): Promise<DbEntitiesResponse>;
entityByName(
tx: Transaction,
name: EntityName,
): Promise<DbEntityResponse | undefined>;
entityByUid(
tx: Transaction,
uid: string,
): Promise<DbEntityResponse | undefined>;
removeEntityByUid(tx: Transaction, uid: string): Promise<void>;
/**
* Remove current relations for the entity and replace them with the new
* relations array.
*
* @param tx - An ongoing transaction
* @param entityUid - The entity uid
* @param relations - The relationships to be set
*/
setRelations(
tx: Transaction,
entityUid: string,
relations: EntityRelationSpec[],
): Promise<void>;
addLocation(tx: Transaction, location: Location): Promise<DbLocationsRow>;
removeLocation(tx: Transaction, id: string): Promise<void>;
location(id: string): Promise<DbLocationsRowWithStatus>;
locations(): Promise<DbLocationsRowWithStatus[]>;
locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]>;
addLocationUpdateLogEvent(
locationId: string,
status: DatabaseLocationUpdateLogStatus,
entityName?: string | string[],
message?: string,
): Promise<void>;
};
plugins/catalog-backend/src/legacy/index.ts
-20
View File
@@ -1,20 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './catalog';
export * from './ingestion';
export * from './service';
export * from './database';
plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts
-415
View File
@@ -1,415 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, Location, LocationSpec } from '@backstage/catalog-model';
import { EntitiesCatalog } from '../../catalog';
import { LocationsCatalog } from '../catalog';
import { LocationUpdateStatus } from '../catalog/types';
import { DatabaseLocationUpdateLogStatus } from '../database/types';
import { HigherOrderOperations } from './HigherOrderOperations';
import { LocationReader } from './types';
describe('HigherOrderOperations', () => {
let entitiesCatalog: jest.Mocked<Required<EntitiesCatalog>>;
let locationsCatalog: jest.Mocked<LocationsCatalog>;
let locationReader: jest.Mocked<LocationReader>;
let higherOrderOperation: HigherOrderOperations;
beforeAll(() => {
entitiesCatalog = {
entities: jest.fn(),
removeEntityByUid: jest.fn(),
batchAddOrUpdateEntities: jest.fn(),
entityAncestry: jest.fn(),
};
locationsCatalog = {
addLocation: jest.fn(),
removeLocation: jest.fn(),
locations: jest.fn(),
location: jest.fn(),
locationHistory: jest.fn(),
logUpdateSuccess: jest.fn(),
logUpdateFailure: jest.fn(),
};
locationReader = {
read: jest.fn(),
};
higherOrderOperation = new HigherOrderOperations(
entitiesCatalog,
locationsCatalog,
locationReader,
getVoidLogger(),
);
});
beforeEach(() => {
jest.resetAllMocks();
});
describe('addLocation', () => {
it('just inserts the location when there are no entities to read', async () => {
const spec = {
type: 'a',
target: 'b',
};
locationsCatalog.addLocation.mockImplementation(x => Promise.resolve(x));
locationsCatalog.locations.mockResolvedValue([]);
locationReader.read.mockResolvedValue({
entities: [],
errors: [],
});
const result = await higherOrderOperation.addLocation(spec);
expect(result.location).toEqual(
expect.objectContaining({
id: expect.anything(),
...spec,
}),
);
expect(result.entities).toEqual([]);
expect(locationsCatalog.locations).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled();
expect(locationsCatalog.addLocation).toBeCalledTimes(1);
expect(locationsCatalog.addLocation).toBeCalledWith(
expect.objectContaining({
id: expect.anything(),
...spec,
}),
);
});
it('insert the location and its entities', async () => {
const spec = {
type: 'a',
target: 'b',
};
const location: LocationSpec = { type: '', target: '' };
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: { name: 'n' },
};
locationsCatalog.addLocation.mockImplementation(x => Promise.resolve(x));
locationsCatalog.locations.mockResolvedValue([]);
locationsCatalog.locations.mockResolvedValue([]);
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
{
entityId: 'id',
entity,
},
]);
locationReader.read.mockResolvedValue({
entities: [
{
location,
entity,
relations: [],
},
],
errors: [],
});
const result = await higherOrderOperation.addLocation(spec);
expect(result.location).toEqual(
expect.objectContaining({
id: expect.anything(),
...spec,
}),
);
expect(result.entities).toEqual([entity]);
expect(locationsCatalog.locations).toBeCalledTimes(1);
expect(locationsCatalog.addLocation).toBeCalledTimes(1);
expect(locationsCatalog.addLocation).toBeCalledWith(
expect.objectContaining({
id: expect.anything(),
...spec,
}),
);
expect(locationReader.read).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledTimes(1);
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledWith(
expect.anything(),
expect.objectContaining({
locationId: expect.anything(),
dryRun: false,
outputEntities: true,
}),
);
});
it('reuses the location if a match already existed', async () => {
const spec = {
type: 'a',
target: 'b',
};
const location = {
id: 'dd12620d-0436-422f-93bd-929aa0788123',
...spec,
};
locationsCatalog.locations.mockResolvedValue([
{
currentStatus: { timestamp: '', status: '', message: '' },
data: location,
},
]);
locationReader.read.mockResolvedValue({
entities: [],
errors: [],
});
const result = await higherOrderOperation.addLocation(spec);
expect(result.location).toEqual(location);
expect(result.entities).toEqual([]);
expect(locationsCatalog.locations).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled();
expect(locationsCatalog.addLocation).not.toBeCalled();
});
it('rejects the whole operation if any entity could not be read', async () => {
const spec = {
type: 'a',
target: 'b',
};
const location: LocationSpec = { type: '', target: '' };
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: { name: 'n' },
};
locationsCatalog.locations.mockResolvedValue([]);
locationReader.read.mockResolvedValue({
entities: [{ entity, location, relations: [] }],
errors: [{ error: new Error('abcd'), location }],
});
await expect(higherOrderOperation.addLocation(spec)).rejects.toThrow(
/abcd/,
);
expect(locationsCatalog.locations).toBeCalledTimes(1);
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled();
expect(locationsCatalog.addLocation).not.toBeCalled();
});
it('rollback everything after a dry run', async () => {
const spec = {
type: 'a',
target: 'b',
};
const location: LocationSpec = { type: '', target: '' };
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: { name: 'n' },
};
locationsCatalog.locations.mockResolvedValue([]);
locationsCatalog.locations.mockResolvedValue([]);
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
{
entityId: 'id',
entity,
},
]);
locationReader.read.mockResolvedValue({
entities: [
{
location,
entity,
relations: [],
},
],
errors: [],
});
const result = await higherOrderOperation.addLocation(spec, {
dryRun: true,
});
expect(result.location).toEqual(
expect.objectContaining({
id: expect.anything(),
...spec,
}),
);
expect(result.entities).toEqual([entity]);
expect(locationsCatalog.locations).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledTimes(1);
expect(entitiesCatalog.batchAddOrUpdateEntities).toBeCalledWith(
expect.anything(),
expect.objectContaining({
dryRun: true,
outputEntities: true,
}),
);
});
});
describe('refreshLocations', () => {
it('works with no locations added', async () => {
locationsCatalog.locations.mockResolvedValue([]);
await expect(
higherOrderOperation.refreshAllLocations(),
).resolves.toBeUndefined();
expect(locationsCatalog.locations).toHaveBeenCalledTimes(1);
expect(locationReader.read).not.toHaveBeenCalled();
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toHaveBeenCalled();
});
it('can update a single location where a matching entity did not exist', async () => {
const locationStatus: LocationUpdateStatus = {
message: '',
status: DatabaseLocationUpdateLogStatus.SUCCESS,
timestamp: new Date(314159265).toISOString(),
};
const location: Location = {
id: '123',
type: 'some',
target: 'thing',
};
const desc: Entity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const entityId = 'xyz123';
locationsCatalog.locations.mockResolvedValue([
{ currentStatus: locationStatus, data: location },
]);
locationReader.read.mockResolvedValue({
entities: [{ entity: desc, location, relations: [] }],
errors: [],
});
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
{ entityId },
]);
await expect(
higherOrderOperation.refreshAllLocations(),
).resolves.toBeUndefined();
expect(locationsCatalog.locations).toHaveBeenCalledTimes(1);
expect(locationReader.read).toHaveBeenCalledTimes(1);
expect(locationReader.read).toHaveBeenNthCalledWith(1, {
type: 'some',
target: 'thing',
});
expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledWith(
[
expect.objectContaining({
entity: expect.objectContaining({ metadata: { name: 'c1' } }),
relations: [],
}),
],
{
locationId: '123',
},
);
});
it('logs successful updates', async () => {
const locationStatus: LocationUpdateStatus = {
message: '',
status: DatabaseLocationUpdateLogStatus.SUCCESS,
timestamp: new Date(314159265).toISOString(),
};
const location: Location = {
id: '123',
type: 'some',
target: 'thing',
};
const desc: Entity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
locationsCatalog.locations.mockResolvedValue([
{ currentStatus: locationStatus, data: location },
]);
locationReader.read.mockResolvedValue({
entities: [{ entity: desc, location, relations: [] }],
errors: [],
});
entitiesCatalog.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNextPage: false },
});
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([]);
await expect(
higherOrderOperation.refreshAllLocations(),
).resolves.toBeUndefined();
expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledTimes(2);
expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith(
'123',
undefined,
);
expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith('123', [
'c1',
]);
});
it('logs unsuccessful updates when reader fails', async () => {
const locationStatus: LocationUpdateStatus = {
message: '',
status: DatabaseLocationUpdateLogStatus.SUCCESS,
timestamp: new Date(314159265).toISOString(),
};
const location: Location = {
id: '123',
type: 'some',
target: 'thing',
};
locationsCatalog.locations.mockResolvedValue([
{ currentStatus: locationStatus, data: location },
]);
locationReader.read.mockRejectedValue(new Error('reader error message'));
await expect(
higherOrderOperation.refreshAllLocations(),
).resolves.toBeUndefined();
expect(locationReader.read).toHaveBeenCalledTimes(1);
expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledTimes(1);
expect(locationsCatalog.logUpdateSuccess).not.toHaveBeenCalled();
expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledWith(
'123',
expect.objectContaining({ message: 'reader error message' }),
);
});
});
});
plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.ts
-218
View File
@@ -1,218 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Location,
LocationSpec,
stringifyLocationReference,
} from '@backstage/catalog-model';
import { v4 as uuidv4 } from 'uuid';
import { Logger } from 'winston';
import { EntitiesCatalog } from '../../catalog';
import { LocationsCatalog } from '../catalog';
import { durationText } from '../../util';
import {
AddLocationResult,
HigherOrderOperation,
LocationReader,
} from './types';
/**
* Placeholder for operations that span several catalogs and/or stretches out
* in time.
*
* @deprecated This was part of the legacy catalog engine
*/
export class HigherOrderOperations implements HigherOrderOperation {
constructor(
private readonly entitiesCatalog: EntitiesCatalog,
private readonly locationsCatalog: LocationsCatalog,
private readonly locationReader: LocationReader,
private readonly logger: Logger,
) {}
/**
* Adds a single location to the catalog.
*
* The location is inspected and fetched, and all of the resulting data is
* validated. If everything goes well, the location and entities are stored
* in the catalog.
*
* If the location already existed, the old location is returned instead and
* the catalog is left unchanged.
*
* @param spec - The location to add
*/
async addLocation(
spec: LocationSpec,
options?: { dryRun?: boolean },
): Promise<AddLocationResult> {
const dryRun = options?.dryRun || false;
// Attempt to find a previous location matching the spec
const previousLocations = await this.locationsCatalog.locations();
const previousLocation = previousLocations.find(
l => spec.type === l.data.type && spec.target === l.data.target,
);
const location: Location = previousLocation
? previousLocation.data
: {
id: uuidv4(),
type: spec.type,
target: spec.target,
};
// Read the location fully, bailing on any errors
const readerOutput = await this.locationReader.read(spec);
if (!(spec.presence === 'optional') && readerOutput.errors.length) {
const item = readerOutput.errors[0];
throw item.error;
}
// TODO(freben): At this point, we could detect orphaned entities, by way
// of having a location annotation pointing to the location but not being
// in the entities list. But we aren't sure what to do about those yet.
// Write
if (!previousLocation && !dryRun) {
// TODO: We do not include location operations in the dryRun. We might perform
// this operation as a separate dry run.
await this.locationsCatalog.addLocation(location);
}
if (readerOutput.entities.length === 0) {
return { location, entities: [] };
}
const writtenEntities = await this.entitiesCatalog
.batchAddOrUpdateEntities!(readerOutput.entities, {
locationId: dryRun ? undefined : location.id,
dryRun,
outputEntities: true,
});
const entities = writtenEntities.map(e => e.entity!);
return { location, entities };
}
/**
* Goes through all registered locations, and performs a refresh of each one.
*
* Entities are read from their respective sources, are parsed and validated
* according to the entity policy, and get inserted or updated in the catalog.
* Entities that have disappeared from their location are left orphaned,
* without changes.
*/
async refreshAllLocations(): Promise<void> {
const startTimestamp = process.hrtime();
const logger = this.logger.child({
component: 'catalog-all-locations-refresh',
});
logger.info('Locations Refresh: Beginning locations refresh');
const locations = await this.locationsCatalog.locations();
logger.info(`Locations Refresh: Visiting ${locations.length} locations`);
for (const { data: location } of locations) {
logger.info(
`Locations Refresh: Refreshing location ${stringifyLocationReference(
location,
)}`,
);
try {
await this.refreshSingleLocation(location, logger);
await this.locationsCatalog.logUpdateSuccess(location.id, undefined);
} catch (e) {
logger.warn(
`Locations Refresh: Failed to refresh location ${stringifyLocationReference(
location,
)}, ${e.stack}`,
);
await this.locationsCatalog.logUpdateFailure(location.id, e);
}
}
logger.info(
`Locations Refresh: Completed locations refresh in ${durationText(
startTimestamp,
)}`,
);
}
// Performs a full refresh of a single location
private async refreshSingleLocation(
location: Location,
optionalLogger?: Logger,
) {
let startTimestamp = process.hrtime();
const logger = optionalLogger || this.logger;
const readerOutput = await this.locationReader.read({
type: location.type,
target: location.target,
});
for (const item of readerOutput.errors) {
logger.warn(
`Failed item in location ${stringifyLocationReference(
item.location,
)}, ${item.error.stack}`,
);
}
logger.info(
`Read ${
readerOutput.entities.length
} entities from location ${stringifyLocationReference(
location,
)} in ${durationText(startTimestamp)}`,
);
startTimestamp = process.hrtime();
try {
await this.entitiesCatalog.batchAddOrUpdateEntities!(
readerOutput.entities,
{ locationId: location.id },
);
} catch (e) {
for (const entity of readerOutput.entities) {
await this.locationsCatalog.logUpdateFailure(
location.id,
e,
entity.entity.metadata.name,
);
}
throw e;
}
logger.debug(`Posting update success markers`);
await this.locationsCatalog.logUpdateSuccess(
location.id,
readerOutput.entities.map(e => e.entity.metadata.name),
);
logger.info(
`Wrote ${
readerOutput.entities.length
} entities from location ${stringifyLocationReference(
location,
)} in ${durationText(startTimestamp)}`,
);
}
}
plugins/catalog-backend/src/legacy/ingestion/LocationReaders.ts
-353
View File
@@ -1,353 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { NotAllowedError } from '@backstage/errors';
import { UrlReader } from '@backstage/backend-common';
import {
Entity,
EntityPolicy,
EntityRelationSpec,
ENTITY_DEFAULT_NAMESPACE,
LocationSpec,
stringifyLocationReference,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { Logger } from 'winston';
import { CatalogRulesEnforcer } from '../../ingestion/CatalogRules';
import * as result from '../../ingestion/processors/results';
import {
CatalogProcessor,
CatalogProcessorEmit,
CatalogProcessorEntityResult,
CatalogProcessorErrorResult,
CatalogProcessorLocationResult,
CatalogProcessorParser,
CatalogProcessorResult,
} from '../../ingestion/processors/types';
import { LocationReader, ReadLocationResult } from './types';
// The max amount of nesting depth of generated work items
const MAX_DEPTH = 10;
type Options = {
reader: UrlReader;
parser: CatalogProcessorParser;
logger: Logger;
config: Config;
processors: CatalogProcessor[];
rulesEnforcer: CatalogRulesEnforcer;
policy: EntityPolicy;
};
const noopCache = {
async get() {
return undefined;
},
async set() {},
};
/**
* Implements the reading of a location through a series of processor tasks.
*
* @deprecated This was part of the legacy catalog engine
*/
export class LocationReaders implements LocationReader {
private readonly options: Options;
constructor(options: Options) {
this.options = options;
}
async read(location: LocationSpec): Promise<ReadLocationResult> {
const { rulesEnforcer, logger } = this.options;
const output: ReadLocationResult = {
entities: [],
errors: [],
};
let items: CatalogProcessorResult[] = [result.location(location, false)];
for (let depth = 0; depth < MAX_DEPTH; ++depth) {
const newItems: CatalogProcessorResult[] = [];
const emit: CatalogProcessorEmit = i => newItems.push(i);
for (const item of items) {
if (item.type === 'location') {
await this.handleLocation(item, emit);
} else if (item.type === 'entity') {
if (rulesEnforcer.isAllowed(item.entity, item.location)) {
const relations = Array<EntityRelationSpec>();
const entity = await this.handleEntity(
item,
emitResult => {
if (emitResult.type === 'relation') {
relations.push(emitResult.relation);
return;
}
emit(emitResult);
},
location,
);
if (entity) {
output.entities.push({
entity,
location: item.location,
relations,
});
}
} else {
output.errors.push({
location: item.location,
error: new NotAllowedError(
`Entity of kind ${
item.entity.kind
} is not allowed from location ${stringifyLocationReference(
item.location,
)}`,
),
});
}
} else if (item.type === 'error') {
await this.handleError(item, emit);
output.errors.push({
location: item.location,
error: item.error,
});
}
}
if (newItems.length === 0) {
return output;
}
items = newItems;
}
const message = `Max recursion depth ${MAX_DEPTH} reached for location ${location.type} ${location.target}`;
logger.warn(message);
output.errors.push({ location, error: new Error(message) });
return output;
}
private async handleLocation(
item: CatalogProcessorLocationResult,
emit: CatalogProcessorEmit,
) {
const { processors, logger } = this.options;
const validatedEmit: CatalogProcessorEmit = emitResult => {
if (emitResult.type === 'relation') {
throw new Error('readLocation may not emit entity relations');
}
if (
emitResult.type === 'location' &&
emitResult.location.type === item.location.type &&
emitResult.location.target === item.location.target
) {
// Ignore self-referential locations silently (this can happen for
// example if you use a glob target like "**/*.yaml" in a Location
// entity)
return;
}
emit(emitResult);
};
for (const processor of processors) {
if (processor.readLocation) {
try {
if (
await processor.readLocation(
item.location,
item.optional,
validatedEmit,
this.options.parser,
noopCache,
)
) {
return;
}
} catch (e) {
const message = `Processor ${
processor.constructor.name
} threw an error while reading location ${stringifyLocationReference(
item.location,
)}, ${e}`;
emit(result.generalError(item.location, message));
logger.warn(message);
}
}
}
const message = `No processor was able to read location ${stringifyLocationReference(
item.location,
)}`;
emit(result.inputError(item.location, message));
logger.warn(message);
}
private async handleEntity(
item: CatalogProcessorEntityResult,
emit: CatalogProcessorEmit,
originLocation: LocationSpec,
): Promise<Entity | undefined> {
const { processors, logger } = this.options;
let current = item.entity;
// Construct the name carefully, this happens before validation below
// so we do not want to crash here due to missing metadata or so
const kind = current.kind || '';
const namespace = !current.metadata
? ''
: current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE;
const name = !current.metadata ? '' : current.metadata.name;
for (const processor of processors) {
if (processor.preProcessEntity) {
try {
current = await processor.preProcessEntity(
current,
item.location,
emit,
originLocation,
noopCache,
);
} catch (e) {
const message = `Processor ${
processor.constructor.name
} threw an error while preprocessing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
item.location,
)}, ${e}`;
emit(result.generalError(item.location, e.message));
logger.warn(message);
return undefined;
}
}
}
try {
const next = await this.options.policy.enforce(current);
if (!next) {
const message = `Policy unexpectedly returned no data while analyzing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
item.location,
)}`;
emit(result.generalError(item.location, message));
logger.warn(message);
return undefined;
}
current = next;
} catch (e) {
const message = `Policy check failed while analyzing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
item.location,
)}, ${e}`;
emit(result.inputError(item.location, message));
logger.warn(message);
return undefined;
}
let handled = false;
for (const processor of processors) {
if (processor.validateEntityKind) {
try {
handled = await processor.validateEntityKind(current);
if (handled) {
break;
}
} catch (e) {
const message = `Processor ${
processor.constructor.name
} threw an error while validating the entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
item.location,
)}, ${e}`;
emit(result.inputError(item.location, message));
logger.warn(message);
return undefined;
}
}
}
if (!handled) {
const message = `No processor recognized the entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
item.location,
)}`;
emit(result.inputError(item.location, message));
logger.warn(message);
return undefined;
}
for (const processor of processors) {
if (processor.postProcessEntity) {
try {
current = await processor.postProcessEntity(
current,
item.location,
emit,
noopCache,
);
} catch (e) {
const message = `Processor ${
processor.constructor.name
} threw an error while postprocessing entity ${kind}:${namespace}/${name} at ${stringifyLocationReference(
item.location,
)}, ${e}`;
emit(result.generalError(item.location, message));
logger.warn(message);
return undefined;
}
}
}
return current;
}
private async handleError(
item: CatalogProcessorErrorResult,
emit: CatalogProcessorEmit,
) {
const { processors, logger } = this.options;
logger.debug(
`Encountered error at location ${stringifyLocationReference(
item.location,
)}, ${item.error}`,
);
const validatedEmit: CatalogProcessorEmit = emitResult => {
if (emitResult.type === 'relation') {
throw new Error('handleError may not emit entity relations');
}
emit(emitResult);
};
for (const processor of processors) {
if (processor.handleError) {
try {
await processor.handleError(item.error, item.location, validatedEmit);
} catch (e) {
const message = `Processor ${
processor.constructor.name
} threw an error while handling another error at ${stringifyLocationReference(
item.location,
)}, ${e}`;
emit(result.generalError(item.location, message));
logger.warn(message);
}
}
}
}
}
plugins/catalog-backend/src/legacy/ingestion/index.ts
-26
View File
@@ -1,26 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { HigherOrderOperations } from './HigherOrderOperations';
export { LocationReaders } from './LocationReaders';
export type {
AddLocationResult,
HigherOrderOperation,
LocationReader,
ReadLocationEntity,
ReadLocationError,
ReadLocationResult,
} from './types';
plugins/catalog-backend/src/legacy/ingestion/types.ts
-76
View File
@@ -1,76 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Entity,
EntityRelationSpec,
Location,
LocationSpec,
} from '@backstage/catalog-model';
//
// LocationReader
//
/** @deprecated This was part of the legacy catalog engine */
export type HigherOrderOperation = {
addLocation(
spec: LocationSpec,
options?: { dryRun?: boolean },
): Promise<AddLocationResult>;
refreshAllLocations(): Promise<void>;
};
/** @deprecated This was part of the legacy catalog engine */
export type AddLocationResult = {
location: Location;
entities: Entity[];
};
//
// LocationReader
//
/** @deprecated This was part of the legacy catalog engine */
export type LocationReader = {
/**
* Reads the contents of a location.
*
* @param location - The location to read
* @throws An error if the location was handled by this reader, but could not
* be read
*/
read(location: LocationSpec): Promise<ReadLocationResult>;
};
/** @deprecated This was part of the legacy catalog engine */
export type ReadLocationResult = {
entities: ReadLocationEntity[];
errors: ReadLocationError[];
};
/** @deprecated This was part of the legacy catalog engine */
export type ReadLocationEntity = {
location: LocationSpec;
entity: Entity;
relations: EntityRelationSpec[];
};
/** @deprecated This was part of the legacy catalog engine */
export type ReadLocationError = {
location: LocationSpec;
error: Error;
};
plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts
-260
View File
@@ -1,260 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
getVoidLogger,
PluginEndpointDiscovery,
ServerTokenManager,
UrlReader,
} from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import { Knex } from 'knex';
import yaml from 'yaml';
import { DatabaseManager } from '../database';
import { CatalogProcessorParser } from '../../ingestion';
import * as result from '../../ingestion/processors/results';
import { CatalogBuilder } from './CatalogBuilder';
import { CatalogEnvironment } from '../../service';
import { ServerPermissionClient } from '@backstage/plugin-permission-node';
const dummyEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'n',
},
spec: {
type: 't',
owner: 'o',
lifecycle: 'l',
},
};
const dummyEntityYaml = yaml.stringify(dummyEntity);
describe('CatalogBuilder', () => {
let db: Knex<any, unknown[]>;
const reader: jest.Mocked<UrlReader> = {
read: jest.fn(),
readTree: jest.fn(),
search: jest.fn(),
};
const config = new ConfigReader({});
const mockBaseUrl = 'http://backstage:9191/i-am-a-mock-base';
const discovery: PluginEndpointDiscovery = {
async getBaseUrl() {
return mockBaseUrl;
},
async getExternalBaseUrl() {
return mockBaseUrl;
},
};
const env: CatalogEnvironment = {
logger: getVoidLogger(),
database: { getClient: async () => db },
config,
reader,
permissions: ServerPermissionClient.fromConfig(config, {
discovery,
tokenManager: ServerTokenManager.noop(),
}),
};
beforeEach(async () => {
db = await DatabaseManager.createTestDatabaseConnection();
jest.resetAllMocks();
});
it('works with no changes', async () => {
const builder = new CatalogBuilder(env);
const built = await builder.build();
await expect(built.entitiesCatalog.entities()).resolves.toEqual({
entities: [],
pageInfo: { hasNextPage: false },
});
await expect(built.locationsCatalog.locations()).resolves.toEqual([
expect.objectContaining({
data: expect.objectContaining({ type: 'bootstrap' }),
}),
]);
});
it('works with everything replaced', async () => {
reader.read.mockResolvedValueOnce(Buffer.from('junk'));
const builder = new CatalogBuilder(env)
.replaceEntityPolicies([
{
async enforce(entity: Entity) {
expect(entity.metadata.namespace).toBe('ns');
return entity;
},
},
])
.setPlaceholderResolver('t', async ({ value }) => {
expect(value).toBe('tt');
return 'tt2';
})
.setFieldFormatValidators({
isValidEntityName: n => {
expect(n).toBe('n');
return true;
},
})
.replaceProcessors([
{
async readLocation(location, _optional, emit) {
expect(location.type).toBe('test');
emit(
result.entity(location, {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: { name: 'n', replaced: { $t: 'tt' } },
spec: { type: 't', owner: 'o', lifecycle: 'l' },
}),
);
return true;
},
async preProcessEntity(entity) {
expect(entity.apiVersion).toBe('backstage.io/v1alpha1');
return {
...entity,
metadata: { ...entity.metadata, namespace: 'ns' },
};
},
async postProcessEntity(entity) {
expect(entity.metadata.namespace).toBe('ns');
return {
...entity,
metadata: { ...entity.metadata, post: 'p' },
};
},
},
]);
const out = await builder.build();
const added = await out.higherOrderOperation.addLocation({
type: 'test',
target: '',
});
expect.assertions(6);
expect(added.entities).toEqual([
{
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'n',
namespace: 'ns',
post: 'p',
replaced: 'tt2',
uid: expect.any(String),
etag: expect.any(String),
generation: expect.any(Number),
},
spec: {
type: 't',
owner: 'o',
lifecycle: 'l',
},
relations: expect.anything(),
},
]);
});
it('addProcessor works', async () => {
reader.read.mockResolvedValueOnce(Buffer.from(dummyEntityYaml));
const builder = new CatalogBuilder(env);
builder.addProcessor({
async preProcessEntity(e) {
return { ...e, metadata: { ...e.metadata, foo: 7 } };
},
});
const { entitiesCatalog, higherOrderOperation } = await builder.build();
await higherOrderOperation.addLocation({
type: 'url',
target: 'https://github.com/a/b/x.yaml',
});
const { entities } = await entitiesCatalog.entities();
expect(entities).toEqual([
expect.objectContaining({
metadata: expect.objectContaining({
foo: 7,
}),
}),
]);
});
it('replaceProcessors works', async () => {
reader.read.mockResolvedValueOnce(Buffer.from(dummyEntityYaml));
const builder = new CatalogBuilder(env);
builder.replaceProcessors([
{
async readLocation(location, _optional, emit) {
expect(location.type).toBe('x');
emit(result.entity(location, dummyEntity));
return true;
},
async preProcessEntity(e) {
expect(e.metadata.name).toBe('n');
return { ...e, metadata: { ...e.metadata, foo: 7 } };
},
},
]);
const { entitiesCatalog, higherOrderOperation } = await builder.build();
await higherOrderOperation.addLocation({
type: 'x',
target: 'y',
});
const { entities } = await entitiesCatalog.entities();
expect.assertions(3);
expect(entities).toEqual([
expect.objectContaining({
metadata: expect.objectContaining({
foo: 7,
}),
}),
]);
});
it('setEntityDataParser works', async () => {
const mockParser: CatalogProcessorParser = jest
.fn()
.mockImplementation(() => {});
const builder = new CatalogBuilder(env)
.setEntityDataParser(mockParser)
.replaceProcessors([
{
async readLocation(_location, _optional, _emit, parser) {
expect(parser).toBe(mockParser);
return true;
},
},
]);
const { higherOrderOperation } = await builder.build();
await higherOrderOperation.addLocation({ type: 'x', target: 'y' });
expect.assertions(1);
});
});
plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts
-373
View File
@@ -1,373 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
DefaultNamespaceEntityPolicy,
EntityPolicies,
EntityPolicy,
FieldFormatEntityPolicy,
makeValidator,
NoForeignRootFieldsEntityPolicy,
SchemaValidEntityPolicy,
Validators,
} from '@backstage/catalog-model';
import {
ScmIntegrations,
DefaultGithubCredentialsProvider,
} from '@backstage/integration';
import lodash from 'lodash';
import { EntitiesCatalog } from '../../catalog';
import {
DatabaseEntitiesCatalog,
DatabaseLocationsCatalog,
LocationsCatalog,
} from '../catalog';
import { DatabaseManager } from '../database';
import {
AnnotateLocationEntityProcessor,
BitbucketDiscoveryProcessor,
BuiltinKindsEntityProcessor,
CatalogProcessor,
CatalogProcessorParser,
CodeOwnersProcessor,
FileReaderProcessor,
GithubDiscoveryProcessor,
AzureDevOpsDiscoveryProcessor,
GithubOrgReaderProcessor,
GitLabDiscoveryProcessor,
LocationEntityProcessor,
PlaceholderProcessor,
PlaceholderResolver,
StaticLocationProcessor,
UrlReaderProcessor,
} from '../../ingestion';
import {
HigherOrderOperation,
HigherOrderOperations,
LocationReaders,
} from '../ingestion';
import { DefaultCatalogRulesEnforcer } from '../../ingestion/CatalogRules';
import { RepoLocationAnalyzer } from '../../ingestion/LocationAnalyzer';
import {
jsonPlaceholderResolver,
textPlaceholderResolver,
yamlPlaceholderResolver,
} from '../../ingestion/processors/PlaceholderProcessor';
import { defaultEntityDataParser } from '../../ingestion/processors/util/parse';
import { LocationAnalyzer } from '../../ingestion/types';
import { CatalogEnvironment, NextCatalogBuilder } from '../../service';
/**
* A builder that helps wire up all of the component parts of the catalog.
*
* The touch points where you can replace or extend behavior are as follows:
*
* - Entity policies can be added or replaced. These are automatically run
* after the processors' pre-processing steps. All policies are given the
* chance to inspect the entity, and all of them have to pass in order for
* the entity to be considered valid from an overall point of view.
* - Placeholder resolvers can be replaced or added. These run on the raw
* structured data between the parsing and pre-processing steps, to replace
* dollar-prefixed entries with their actual values (like $file).
* - Field format validators can be replaced. These check the format of
* individual core fields such as metadata.name, to ensure that they adhere
* to certain rules.
* - Processors can be added or replaced. These implement the functionality of
* reading, parsing, validating, and processing the entity data before it is
* persisted in the catalog.
*
* NOTE(freben): Not actually marking the class as deprecated formally, since
* it would appear to end users that even using `create` is deprecated. We will
* instead hot-swap the entire exported class when we are ready.
*/
export class CatalogBuilder {
private readonly env: CatalogEnvironment;
private entityPolicies: EntityPolicy[];
private entityPoliciesReplace: boolean;
private placeholderResolvers: Record<string, PlaceholderResolver>;
private fieldFormatValidators: Partial<Validators>;
private processors: CatalogProcessor[];
private processorsReplace: boolean;
private parser: CatalogProcessorParser | undefined;
static async create(env: CatalogEnvironment): Promise<NextCatalogBuilder> {
return new NextCatalogBuilder(env);
}
/** @deprecated Please use CatalogBuilder.create() instead */
constructor(env: CatalogEnvironment) {
this.env = env;
this.entityPolicies = [];
this.entityPoliciesReplace = false;
this.placeholderResolvers = {};
this.fieldFormatValidators = {};
this.processors = [];
this.processorsReplace = false;
this.parser = undefined;
env.logger.warn(
"Creating the catalog with 'new CatalogBuilder(env)' is deprecated! Use CatalogBuilder.create(env) instead",
);
}
/**
* Adds policies that are used to validate entities between the pre-
* processing and post-processing stages. All such policies must pass for the
* entity to be considered valid.
*
* If what you want to do is to replace the rules for what format is allowed
* in various core entity fields (such as metadata.name), you may want to use
* {@link CatalogBuilder#setFieldFormatValidators} instead.
*
* @param policies - One or more policies
*/
addEntityPolicy(...policies: EntityPolicy[]): CatalogBuilder {
this.entityPolicies.push(...policies);
return this;
}
/**
* Sets what policies to use for validation of entities between the pre-
* processing and post-processing stages. All such policies must pass for the
* entity to be considered valid.
*
* If what you want to do is to replace the rules for what format is allowed
* in various core entity fields (such as metadata.name), you may want to use
* {@link CatalogBuilder#setFieldFormatValidators} instead.
*
* This function replaces the default set of policies; use with care.
*
* @param policies - One or more policies
*/
replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder {
this.entityPolicies = [...policies];
this.entityPoliciesReplace = true;
return this;
}
/**
* Adds, or overwrites, a handler for placeholders (e.g. $file) in entity
* definition files.
*
* @param key - The key that identifies the placeholder, e.g. "file"
* @param resolver - The resolver that gets values for this placeholder
*/
setPlaceholderResolver(
key: string,
resolver: PlaceholderResolver,
): CatalogBuilder {
this.placeholderResolvers[key] = resolver;
return this;
}
/**
* Sets the validator function to use for one or more special fields of an
* entity. This is useful if the default rules for formatting of fields are
* not sufficient.
*
* This function has no effect if used together with
* {@link CatalogBuilder#replaceEntityPolicies}.
*
* @param validators - The (subset of) validators to set
*/
setFieldFormatValidators(validators: Partial<Validators>): CatalogBuilder {
lodash.merge(this.fieldFormatValidators, validators);
return this;
}
/**
* Adds entity processors. These are responsible for reading, parsing, and
* processing entities before they are persisted in the catalog.
*
* @param processors - One or more processors
*/
addProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.processors.push(...processors);
return this;
}
/**
* Sets what entity processors to use. These are responsible for reading,
* parsing, and processing entities before they are persisted in the catalog.
*
* This function replaces the default set of processors; use with care.
*
* @param processors - One or more processors
*/
replaceProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.processors = [...processors];
this.processorsReplace = true;
return this;
}
/**
* Sets up the catalog to use a custom parser for entity data.
*
* This is the function that gets called immediately after some raw entity
* specification data has been read from a remote source, and needs to be
* parsed and emitted as structured data.
*
* @param parser - The custom parser
*/
setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder {
this.parser = parser;
return this;
}
/**
* Wires up and returns all of the component parts of the catalog
*/
async build(): Promise<{
entitiesCatalog: EntitiesCatalog;
locationsCatalog: LocationsCatalog;
higherOrderOperation: HigherOrderOperation;
locationAnalyzer: LocationAnalyzer;
}> {
const { config, database, logger } = this.env;
const integrations = ScmIntegrations.fromConfig(config);
const policy = this.buildEntityPolicy();
const processors = this.buildProcessors();
const rulesEnforcer = DefaultCatalogRulesEnforcer.fromConfig(config);
const parser = this.parser || defaultEntityDataParser;
const locationReader = new LocationReaders({
...this.env,
parser,
processors,
rulesEnforcer,
policy,
});
const db = await DatabaseManager.createDatabase(
await database.getClient(),
{ logger },
);
const entitiesCatalog = new DatabaseEntitiesCatalog(db, this.env.logger);
const locationsCatalog = new DatabaseLocationsCatalog(db);
const higherOrderOperation = new HigherOrderOperations(
entitiesCatalog,
locationsCatalog,
locationReader,
logger,
);
const locationAnalyzer = new RepoLocationAnalyzer(logger, integrations);
return {
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
locationAnalyzer,
};
}
private buildEntityPolicy(): EntityPolicy {
const entityPolicies: EntityPolicy[] = this.entityPoliciesReplace
? [new SchemaValidEntityPolicy(), ...this.entityPolicies]
: [
new SchemaValidEntityPolicy(),
new DefaultNamespaceEntityPolicy(),
new NoForeignRootFieldsEntityPolicy(),
new FieldFormatEntityPolicy(
makeValidator(this.fieldFormatValidators),
),
...this.entityPolicies,
];
return EntityPolicies.allOf(entityPolicies);
}
private buildProcessors(): CatalogProcessor[] {
const { config, logger, reader } = this.env;
const integrations = ScmIntegrations.fromConfig(config);
const githubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
this.checkDeprecatedReaderProcessors();
const placeholderResolvers: Record<string, PlaceholderResolver> = {
json: jsonPlaceholderResolver,
yaml: yamlPlaceholderResolver,
text: textPlaceholderResolver,
...this.placeholderResolvers,
};
// These are always there no matter what
const processors: CatalogProcessor[] = [
StaticLocationProcessor.fromConfig(config),
new PlaceholderProcessor({
resolvers: placeholderResolvers,
reader,
integrations,
}),
new BuiltinKindsEntityProcessor(),
];
// These are only added unless the user replaced them all
if (!this.processorsReplace) {
processors.push(
new FileReaderProcessor(),
BitbucketDiscoveryProcessor.fromConfig(config, { logger }),
GithubDiscoveryProcessor.fromConfig(config, {
logger,
githubCredentialsProvider,
}),
AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }),
GithubOrgReaderProcessor.fromConfig(config, {
logger,
githubCredentialsProvider,
}),
GitLabDiscoveryProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor({ reader, logger }),
CodeOwnersProcessor.fromConfig(config, { logger, reader }),
new LocationEntityProcessor({ integrations }),
new AnnotateLocationEntityProcessor({ integrations }),
);
}
// Add the ones (if any) that the user added
processors.push(...this.processors);
return processors;
}
// TODO(Rugvip): These old processors are removed, for a while we'll be throwing
// errors here to make sure people know where to move the config
private checkDeprecatedReaderProcessors() {
const pc = this.env.config.getOptionalConfig('catalog.processors');
if (pc?.has('github')) {
throw new Error(
`Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`,
);
}
if (pc?.has('gitlabApi')) {
throw new Error(
`Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`,
);
}
if (pc?.has('bitbucketApi')) {
throw new Error(
`Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`,
);
}
if (pc?.has('azureApi')) {
throw new Error(
`Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`,
);
}
}
}
plugins/catalog-backend/src/legacy/service/router.test.ts
-529
View File
@@ -1,529 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { NotFoundError } from '@backstage/errors';
import type { Entity, LocationSpec } from '@backstage/catalog-model';
import express from 'express';
import request from 'supertest';
import { EntitiesCatalog } from '../../catalog';
import { LocationResponse, LocationsCatalog } from '../catalog/types';
import { HigherOrderOperation } from '../ingestion/types';
import { createRouter } from './router';
import { basicEntityFilter } from '../../service/request';
import { RefreshService } from '../../service';
describe('createRouter readonly disabled', () => {
let entitiesCatalog: jest.Mocked<Required<EntitiesCatalog>>;
let locationsCatalog: jest.Mocked<LocationsCatalog>;
let higherOrderOperation: jest.Mocked<HigherOrderOperation>;
let app: express.Express;
let refreshService: RefreshService;
beforeAll(async () => {
entitiesCatalog = {
entities: jest.fn(),
removeEntityByUid: jest.fn(),
batchAddOrUpdateEntities: jest.fn(),
entityAncestry: jest.fn(),
};
locationsCatalog = {
addLocation: jest.fn(),
removeLocation: jest.fn(),
locations: jest.fn(),
location: jest.fn(),
locationHistory: jest.fn(),
logUpdateSuccess: jest.fn(),
logUpdateFailure: jest.fn(),
};
higherOrderOperation = {
addLocation: jest.fn(),
refreshAllLocations: jest.fn(),
};
refreshService = { refresh: jest.fn() };
const router = await createRouter({
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
logger: getVoidLogger(),
refreshService,
config: new ConfigReader(undefined),
});
app = express().use(router);
});
beforeEach(() => {
jest.resetAllMocks();
});
describe('POST /refresh', () => {
it('refreshes an entity using the refresh service', async () => {
const response = await request(app)
.post('/refresh')
.set('Content-Type', 'application/json')
.send({ entityRef: 'Component/default:foo' });
expect(response.status).toBe(200);
expect(refreshService.refresh).toHaveBeenCalledWith({
entityRef: 'Component/default:foo',
});
});
});
describe('GET /entities', () => {
it('happy path: lists entities', async () => {
const entities: Entity[] = [
{ apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
];
entitiesCatalog.entities.mockResolvedValueOnce({
entities: [entities[0]],
pageInfo: { hasNextPage: false },
});
const response = await request(app).get('/entities');
expect(response.status).toEqual(200);
expect(response.body).toEqual(entities);
});
it('parses single and multiple request parameters and passes them down', async () => {
entitiesCatalog.entities.mockResolvedValueOnce({
entities: [],
pageInfo: { hasNextPage: false },
});
const response = await request(app).get(
'/entities?filter=a=1,a=2,b=3&filter=c=4',
);
expect(response.status).toEqual(200);
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: {
anyOf: [
{
allOf: [
{ key: 'a', values: ['1', '2'] },
{ key: 'b', values: ['3'] },
],
},
{ allOf: [{ key: 'c', values: ['4'] }] },
],
},
});
});
});
describe('GET /entities/by-uid/:uid', () => {
it('can fetch entity by uid', async () => {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
name: 'c',
},
};
entitiesCatalog.entities.mockResolvedValue({
entities: [entity],
pageInfo: { hasNextPage: false },
});
const response = await request(app).get('/entities/by-uid/zzz');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({ 'metadata.uid': 'zzz' }),
});
expect(response.status).toEqual(200);
expect(response.body).toEqual(expect.objectContaining(entity));
});
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNextPage: false },
});
const response = await request(app).get('/entities/by-uid/zzz');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({ 'metadata.uid': 'zzz' }),
});
expect(response.status).toEqual(404);
expect(response.text).toMatch(/uid/);
});
});
describe('GET /entities/by-name/:kind/:namespace/:name', () => {
it('can fetch entity by name', async () => {
const entity: Entity = {
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
},
};
entitiesCatalog.entities.mockResolvedValue({
entities: [entity],
pageInfo: { hasNextPage: false },
});
const response = await request(app).get('/entities/by-name/k/ns/n');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({
kind: 'k',
'metadata.namespace': 'ns',
'metadata.name': 'n',
}),
});
expect(response.status).toEqual(200);
expect(response.body).toEqual(expect.objectContaining(entity));
});
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNextPage: false },
});
const response = await request(app).get('/entities/by-name/b/d/c');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(response.status).toEqual(404);
expect(response.text).toMatch(/name/);
});
});
describe('POST /entities', () => {
it('requires a body', async () => {
const response = await request(app)
.post('/entities')
.set('Content-Type', 'application/json')
.send();
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toHaveBeenCalled();
expect(response.status).toEqual(400);
expect(response.text).toMatch(/body/);
});
it('passes the body down', async () => {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
name: 'c',
namespace: 'd',
},
};
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
{ entityId: 'u' },
]);
entitiesCatalog.entities.mockResolvedValue({
entities: [entity],
pageInfo: { hasNextPage: false },
});
const response = await request(app)
.post('/entities')
.send(entity)
.set('Content-Type', 'application/json');
expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledWith([
{ entity, relations: [] },
]);
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({ 'metadata.uid': 'u' }),
});
expect(response.status).toEqual(200);
expect(response.body).toEqual(entity);
});
});
describe('DELETE /entities/by-uid/:uid', () => {
it('can remove', async () => {
entitiesCatalog.removeEntityByUid.mockResolvedValue(undefined);
const response = await request(app).delete('/entities/by-uid/apa');
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledWith('apa');
expect(response.status).toEqual(204);
});
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.removeEntityByUid.mockRejectedValue(
new NotFoundError('nope'),
);
const response = await request(app).delete('/entities/by-uid/apa');
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledWith('apa');
expect(response.status).toEqual(404);
});
});
describe('GET /locations', () => {
it('happy path: lists locations', async () => {
const locations: LocationResponse[] = [
{
currentStatus: { timestamp: '', status: '', message: '' },
data: { id: 'a', type: 'b', target: 'c' },
},
];
locationsCatalog.locations.mockResolvedValueOnce(locations);
const response = await request(app).get('/locations');
expect(response.status).toEqual(200);
expect(response.body).toEqual(locations);
});
});
describe('POST /locations', () => {
it('rejects malformed locations', async () => {
const spec = {
typez: 'b',
target: 'c',
} as unknown as LocationSpec;
const response = await request(app).post('/locations').send(spec);
expect(higherOrderOperation.addLocation).not.toHaveBeenCalled();
expect(response.status).toEqual(400);
});
it('passes the body down', async () => {
const spec: LocationSpec = {
type: 'b',
target: 'c',
};
higherOrderOperation.addLocation.mockResolvedValue({
location: { id: 'a', ...spec },
entities: [],
});
const response = await request(app).post('/locations').send(spec);
expect(higherOrderOperation.addLocation).toHaveBeenCalledTimes(1);
expect(higherOrderOperation.addLocation).toHaveBeenCalledWith(spec, {
dryRun: false,
});
expect(response.status).toEqual(201);
expect(response.body).toEqual(
expect.objectContaining({
location: { id: 'a', ...spec },
}),
);
});
it('supports dry run', async () => {
const spec: LocationSpec = {
type: 'b',
target: 'c',
};
higherOrderOperation.addLocation.mockResolvedValue({
location: { id: 'a', ...spec },
entities: [],
});
const response = await request(app)
.post('/locations?dryRun=true')
.send(spec);
expect(higherOrderOperation.addLocation).toHaveBeenCalledTimes(1);
expect(higherOrderOperation.addLocation).toHaveBeenCalledWith(spec, {
dryRun: true,
});
expect(response.status).toEqual(201);
expect(response.body).toEqual(
expect.objectContaining({
location: { id: 'a', ...spec },
}),
);
});
});
});
describe('createRouter readonly enabled', () => {
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
let locationsCatalog: jest.Mocked<LocationsCatalog>;
let higherOrderOperation: jest.Mocked<HigherOrderOperation>;
let app: express.Express;
beforeAll(async () => {
entitiesCatalog = {
entities: jest.fn(),
removeEntityByUid: jest.fn(),
batchAddOrUpdateEntities: jest.fn(),
entityAncestry: jest.fn(),
};
locationsCatalog = {
addLocation: jest.fn(),
removeLocation: jest.fn(),
locations: jest.fn(),
location: jest.fn(),
locationHistory: jest.fn(),
logUpdateSuccess: jest.fn(),
logUpdateFailure: jest.fn(),
};
higherOrderOperation = {
addLocation: jest.fn(),
refreshAllLocations: jest.fn(),
};
const router = await createRouter({
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
logger: getVoidLogger(),
config: new ConfigReader({
catalog: {
readonly: true,
},
}),
});
app = express().use(router);
});
beforeEach(() => {
jest.resetAllMocks();
});
describe('GET /entities', () => {
it('happy path: lists entities', async () => {
const entities: Entity[] = [
{ apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
];
entitiesCatalog.entities.mockResolvedValueOnce({
entities: [entities[0]],
pageInfo: { hasNextPage: false },
});
const response = await request(app).get('/entities');
expect(response.status).toEqual(200);
expect(response.body).toEqual(entities);
});
});
describe('POST /entities', () => {
it('is not allowed', async () => {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
name: 'c',
namespace: 'd',
},
};
const response = await request(app)
.post('/entities')
.set('Content-Type', 'application/json')
.send(entity);
expect(entitiesCatalog.batchAddOrUpdateEntities).not.toHaveBeenCalled();
expect(response.status).toEqual(403);
expect(response.text).toMatch(/not allowed in readonly/);
});
});
describe('DELETE /entities/by-uid/:uid', () => {
// this delete is allowed as there is no other way to remove entities
it('is allowed', async () => {
const response = await request(app).delete('/entities/by-uid/apa');
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledWith('apa');
expect(response.status).toEqual(204);
});
});
describe('GET /locations', () => {
it('happy path: lists locations', async () => {
const locations: LocationResponse[] = [
{
currentStatus: { timestamp: '', status: '', message: '' },
data: { id: 'a', type: 'b', target: 'c' },
},
];
locationsCatalog.locations.mockResolvedValueOnce(locations);
const response = await request(app).get('/locations');
expect(response.status).toEqual(200);
expect(response.body).toEqual(locations);
});
});
describe('POST /locations', () => {
it('is not allowed', async () => {
const spec: LocationSpec = {
type: 'b',
target: 'c',
};
const response = await request(app).post('/locations').send(spec);
expect(higherOrderOperation.addLocation).not.toHaveBeenCalled();
expect(response.status).toEqual(403);
expect(response.text).toMatch(/not allowed in readonly/);
});
it('supports dry run', async () => {
const spec: LocationSpec = {
type: 'b',
target: 'c',
};
higherOrderOperation.addLocation.mockResolvedValue({
location: { id: 'a', ...spec },
entities: [],
});
const response = await request(app)
.post('/locations?dryRun=true')
.send(spec);
expect(higherOrderOperation.addLocation).toHaveBeenCalledTimes(1);
expect(higherOrderOperation.addLocation).toHaveBeenCalledWith(spec, {
dryRun: true,
});
expect(response.status).toEqual(201);
expect(response.body).toEqual(
expect.objectContaining({
location: { id: 'a', ...spec },
}),
);
});
});
});
plugins/catalog-backend/src/legacy/service/router.ts
-254
View File
@@ -1,254 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { errorHandler } from '@backstage/backend-common';
import type { Entity } from '@backstage/catalog-model';
import {
analyzeLocationSchema,
locationSpecSchema,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { NotFoundError } from '@backstage/errors';
import express from 'express';
import Router from 'express-promise-router';
import { Logger } from 'winston';
import yn from 'yn';
import { EntitiesCatalog } from '../../catalog';
import { LocationsCatalog } from '../catalog';
import { LocationAnalyzer } from '../../ingestion/types';
import { HigherOrderOperation } from '../ingestion/types';
import {
RefreshService,
LocationService,
RefreshOptions,
} from '../../service/types';
import {
basicEntityFilter,
parseEntityFilterParams,
parseEntityPaginationParams,
parseEntityTransformParams,
} from '../../service/request';
import {
disallowReadonlyMode,
requireRequestBody,
validateRequestBody,
} from '../../service/util';
/** @deprecated This was part of the legacy catalog engine */
export interface RouterOptions {
entitiesCatalog?: EntitiesCatalog;
locationsCatalog?: LocationsCatalog;
higherOrderOperation?: HigherOrderOperation;
locationAnalyzer?: LocationAnalyzer;
locationService?: LocationService;
refreshService?: RefreshService;
logger: Logger;
config: Config;
}
/** @deprecated This was part of the legacy catalog engine */
export async function createRouter(
options: RouterOptions,
): Promise<express.Router> {
const {
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
locationAnalyzer,
locationService,
refreshService,
config,
logger,
} = options;
const router = Router();
router.use(express.json());
const readonlyEnabled =
config.getOptionalBoolean('catalog.readonly') || false;
if (readonlyEnabled) {
logger.info('Catalog is running in readonly mode');
}
if (refreshService) {
router.post('/refresh', async (req, res) => {
const refreshOptions: RefreshOptions = req.body;
await refreshService.refresh(refreshOptions);
res.status(200).send();
});
}
if (entitiesCatalog) {
router
.get('/entities', async (req, res) => {
const { entities, pageInfo } = await entitiesCatalog.entities({
filter: parseEntityFilterParams(req.query),
fields: parseEntityTransformParams(req.query),
pagination: parseEntityPaginationParams(req.query),
});
// Add a Link header to the next page
if (pageInfo.hasNextPage) {
const url = new URL(`http://ignored${req.url}`);
url.searchParams.delete('offset');
url.searchParams.set('after', pageInfo.endCursor);
res.setHeader('link', `<${url.pathname}${url.search}>; rel="next"`);
}
// TODO(freben): encode the pageInfo in the response
res.json(entities);
})
.post('/entities', async (req, res) => {
/*
* NOTE: THIS METHOD IS DEPRECATED AND NOT RECOMMENDED TO USE
*
* Posting entities to this method has unclear semantics and will not
* properly subject them to limitations, processing, or resolution of
* relations.
*
* It stays around in the service for the time being, but may be
* removed or change semantics at any time without prior notice.
*/
disallowReadonlyMode(readonlyEnabled);
const body = await requireRequestBody(req);
const [result] = await entitiesCatalog.batchAddOrUpdateEntities!([
{ entity: body as Entity, relations: [] },
]);
const response = await entitiesCatalog.entities({
filter: basicEntityFilter({ 'metadata.uid': result.entityId }),
});
res.status(200).json(response.entities[0]);
})
.get('/entities/by-uid/:uid', async (req, res) => {
const { uid } = req.params;
const { entities } = await entitiesCatalog.entities({
filter: basicEntityFilter({ 'metadata.uid': uid }),
});
if (!entities.length) {
throw new NotFoundError(`No entity with uid ${uid}`);
}
res.status(200).json(entities[0]);
})
.delete('/entities/by-uid/:uid', async (req, res) => {
const { uid } = req.params;
await entitiesCatalog.removeEntityByUid(uid);
res.status(204).end();
})
.get('/entities/by-name/:kind/:namespace/:name', async (req, res) => {
const { kind, namespace, name } = req.params;
const { entities } = await entitiesCatalog.entities({
filter: basicEntityFilter({
kind: kind,
'metadata.namespace': namespace,
'metadata.name': name,
}),
});
if (!entities.length) {
throw new NotFoundError(
`No entity named '${name}' found, with kind '${kind}' in namespace '${namespace}'`,
);
}
res.status(200).json(entities[0]);
});
}
if (locationService) {
router
.post('/locations', async (req, res) => {
const input = await validateRequestBody(req, locationSpecSchema);
const dryRun = yn(req.query.dryRun, { default: false });
// when in dryRun addLocation is effectively a read operation so we don't
// need to disallow readonly
if (!dryRun) {
disallowReadonlyMode(readonlyEnabled);
}
const output = await locationService.createLocation(input, dryRun);
res.status(201).json(output);
})
.get('/locations', async (_req, res) => {
const locations = await locationService.listLocations();
res.status(200).json(locations.map(l => ({ data: l })));
})
.get('/locations/:id', async (req, res) => {
const { id } = req.params;
const output = await locationService.getLocation(id);
res.status(200).json(output);
})
.delete('/locations/:id', async (req, res) => {
disallowReadonlyMode(readonlyEnabled);
const { id } = req.params;
await locationService.deleteLocation(id);
res.status(204).end();
});
}
if (higherOrderOperation) {
router.post('/locations', async (req, res) => {
const input = await validateRequestBody(req, locationSpecSchema);
const dryRun = yn(req.query.dryRun, { default: false });
// when in dryRun addLocation is effectively a read operation so we don't
// need to disallow readonly
if (!dryRun) {
disallowReadonlyMode(readonlyEnabled);
}
const output = await higherOrderOperation.addLocation(input, { dryRun });
res.status(201).json(output);
});
}
if (locationsCatalog) {
router
.get('/locations', async (_req, res) => {
const output = await locationsCatalog.locations();
res.status(200).json(output);
})
.get('/locations/:id/history', async (req, res) => {
const { id } = req.params;
const output = await locationsCatalog.locationHistory(id);
res.status(200).json(output);
})
.get('/locations/:id', async (req, res) => {
const { id } = req.params;
const output = await locationsCatalog.location(id);
res.status(200).json(output);
})
.delete('/locations/:id', async (req, res) => {
disallowReadonlyMode(readonlyEnabled);
const { id } = req.params;
await locationsCatalog.removeLocation(id);
res.status(204).end();
});
}
if (locationAnalyzer) {
router.post('/analyze-location', async (req, res) => {
const input = await validateRequestBody(req, analyzeLocationSchema);
const output = await locationAnalyzer.analyzeLocation(input);
res.status(200).json(output);
});
}
router.use(errorHandler());
return router;
}
plugins/catalog-backend/src/service/NextCatalogBuilder.ts → plugins/catalog-backend/src/service/CatalogBuilder.ts
+31 -37
View File
@@ -37,11 +37,6 @@ import { createHash } from 'crypto';
import { Router } from 'express';
import lodash, { keyBy } from 'lodash';
import { EntitiesCatalog, EntitiesSearchFilter } from '../catalog';
import {
DatabaseLocationsCatalog,
LocationsCatalog,
CommonDatabase,
} from '../legacy';
import {
AnnotateLocationEntityProcessor,
@@ -75,14 +70,14 @@ import { applyDatabaseMigrations } from '../database/migrations';
import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine';
import { DefaultLocationService } from './DefaultLocationService';
import { DefaultLocationStore } from '../providers/DefaultLocationStore';
import { NextEntitiesCatalog } from './NextEntitiesCatalog';
import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog';
import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator';
import { Stitcher } from '../stitching/Stitcher';
import {
createRandomRefreshInterval,
RefreshIntervalFunction,
} from '../processing/refresh';
import { createNextRouter } from './NextRouter';
import { createRouter } from './createRouter';
import { DefaultRefreshService } from './DefaultRefreshService';
import { AuthorizedRefreshService } from './AuthorizedRefreshService';
import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules';
@@ -127,8 +122,10 @@ export type CatalogEnvironment = {
* - Processors can be added or replaced. These implement the functionality of
* reading, parsing, validating, and processing the entity data before it is
* persisted in the catalog.
*
* @public
*/
export class NextCatalogBuilder {
export class CatalogBuilder {
private readonly env: CatalogEnvironment;
private entityPolicies: EntityPolicy[];
private entityPoliciesReplace: boolean;
@@ -150,7 +147,14 @@ export class NextCatalogBuilder {
unknown[]
>[];
constructor(env: CatalogEnvironment) {
/**
* Creates a catalog builder.
*/
static create(env: CatalogEnvironment): CatalogBuilder {
return new CatalogBuilder(env);
}
private constructor(env: CatalogEnvironment) {
this.env = env;
this.entityPolicies = [];
this.entityPoliciesReplace = false;
@@ -170,11 +174,11 @@ export class NextCatalogBuilder {
*
* If what you want to do is to replace the rules for what format is allowed
* in various core entity fields (such as metadata.name), you may want to use
* {@link NextCatalogBuilder#setFieldFormatValidators} instead.
* {@link CatalogBuilder#setFieldFormatValidators} instead.
*
* @param policies - One or more policies
*/
addEntityPolicy(...policies: EntityPolicy[]): NextCatalogBuilder {
addEntityPolicy(...policies: EntityPolicy[]): CatalogBuilder {
this.entityPolicies.push(...policies);
return this;
}
@@ -185,7 +189,7 @@ export class NextCatalogBuilder {
* The default refresh duration is 100-150 seconds.
* setting this too low will potentially deplete request quotas to upstream services.
*/
setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder {
setRefreshIntervalSeconds(seconds: number): CatalogBuilder {
this.refreshInterval = createRandomRefreshInterval({
minSeconds: seconds,
maxSeconds: seconds * 1.5,
@@ -197,9 +201,7 @@ export class NextCatalogBuilder {
* Overwrites the default refresh interval function used to spread
* entity updates in the catalog.
*/
setRefreshInterval(
refreshInterval: RefreshIntervalFunction,
): NextCatalogBuilder {
setRefreshInterval(refreshInterval: RefreshIntervalFunction): CatalogBuilder {
this.refreshInterval = refreshInterval;
return this;
}
@@ -207,7 +209,7 @@ export class NextCatalogBuilder {
/**
* Overwrites the default location analyzer.
*/
setLocationAnalyzer(locationAnalyzer: LocationAnalyzer): NextCatalogBuilder {
setLocationAnalyzer(locationAnalyzer: LocationAnalyzer): CatalogBuilder {
this.locationAnalyzer = locationAnalyzer;
return this;
}
@@ -219,13 +221,13 @@ export class NextCatalogBuilder {
*
* If what you want to do is to replace the rules for what format is allowed
* in various core entity fields (such as metadata.name), you may want to use
* {@link NextCatalogBuilder#setFieldFormatValidators} instead.
* {@link CatalogBuilder#setFieldFormatValidators} instead.
*
* This function replaces the default set of policies; use with care.
*
* @param policies - One or more policies
*/
replaceEntityPolicies(policies: EntityPolicy[]): NextCatalogBuilder {
replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder {
this.entityPolicies = [...policies];
this.entityPoliciesReplace = true;
return this;
@@ -241,7 +243,7 @@ export class NextCatalogBuilder {
setPlaceholderResolver(
key: string,
resolver: PlaceholderResolver,
): NextCatalogBuilder {
): CatalogBuilder {
this.placeholderResolvers[key] = resolver;
return this;
}
@@ -252,13 +254,11 @@ export class NextCatalogBuilder {
* not sufficient.
*
* This function has no effect if used together with
* {@link NextCatalogBuilder#replaceEntityPolicies}.
* {@link CatalogBuilder#replaceEntityPolicies}.
*
* @param validators - The (subset of) validators to set
*/
setFieldFormatValidators(
validators: Partial<Validators>,
): NextCatalogBuilder {
setFieldFormatValidators(validators: Partial<Validators>): CatalogBuilder {
lodash.merge(this.fieldFormatValidators, validators);
return this;
}
@@ -272,7 +272,7 @@ export class NextCatalogBuilder {
*
* @param providers - One or more entity providers
*/
addEntityProvider(...providers: EntityProvider[]): NextCatalogBuilder {
addEntityProvider(...providers: EntityProvider[]): CatalogBuilder {
this.entityProviders.push(...providers);
return this;
}
@@ -283,7 +283,7 @@ export class NextCatalogBuilder {
*
* @param processors - One or more processors
*/
addProcessor(...processors: CatalogProcessor[]): NextCatalogBuilder {
addProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.processors.push(...processors);
return this;
}
@@ -293,11 +293,11 @@ export class NextCatalogBuilder {
* parsing, and processing entities before they are persisted in the catalog.
*
* This function replaces the default set of processors, consider using with
* {@link NextCatalogBuilder#getDefaultProcessors}; use with care.
* {@link CatalogBuilder#getDefaultProcessors}; use with care.
*
* @param processors - One or more processors
*/
replaceProcessors(processors: CatalogProcessor[]): NextCatalogBuilder {
replaceProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.processors = [...processors];
this.processorsReplace = true;
return this;
@@ -308,7 +308,7 @@ export class NextCatalogBuilder {
* parsing, and processing entities before they are persisted in the catalog. Changing
* the order of processing can give more control to custom processors.
*
* Consider using with {@link NextCatalogBuilder#replaceProcessors}
* Consider using with {@link CatalogBuilder#replaceProcessors}
*
*/
getDefaultProcessors(): CatalogProcessor[] {
@@ -345,7 +345,7 @@ export class NextCatalogBuilder {
*
* @param parser - The custom parser
*/
setEntityDataParser(parser: CatalogProcessorParser): NextCatalogBuilder {
setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder {
this.parser = parser;
return this;
}
@@ -372,8 +372,6 @@ export class NextCatalogBuilder {
*/
async build(): Promise<{
entitiesCatalog: EntitiesCatalog;
/** @deprecated This will be removed */
locationsCatalog: LocationsCatalog;
locationAnalyzer: LocationAnalyzer;
processingEngine: CatalogProcessingEngine;
locationService: LocationService;
@@ -391,8 +389,6 @@ export class NextCatalogBuilder {
await applyDatabaseMigrations(dbClient);
}
const db = new CommonDatabase(dbClient, logger);
const processingDatabase = new DefaultProcessingDatabase({
database: dbClient,
logger,
@@ -408,7 +404,7 @@ export class NextCatalogBuilder {
parser,
policy,
});
const unauthorizedEntitiesCatalog = new NextEntitiesCatalog(dbClient);
const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog(dbClient);
const entitiesCatalog = new AuthorizedEntitiesCatalog(
unauthorizedEntitiesCatalog,
permissions,
@@ -457,7 +453,6 @@ export class NextCatalogBuilder {
() => createHash('sha1'),
);
const locationsCatalog = new DatabaseLocationsCatalog(db);
const locationAnalyzer =
this.locationAnalyzer ?? new RepoLocationAnalyzer(logger, integrations);
const locationService = new DefaultLocationService(
@@ -468,7 +463,7 @@ export class NextCatalogBuilder {
new DefaultRefreshService({ database: processingDatabase }),
permissions,
);
const router = await createNextRouter({
const router = await createRouter({
entitiesCatalog,
locationAnalyzer,
locationService,
@@ -482,7 +477,6 @@ export class NextCatalogBuilder {
return {
entitiesCatalog,
locationsCatalog,
locationAnalyzer,
processingEngine,
locationService,
plugins/catalog-backend/src/service/NextEntitiesCatalog.test.ts → plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts
+11 -11
View File
@@ -25,9 +25,9 @@ import {
DbRefreshStateRow,
DbSearchRow,
} from '../database/tables';
import { NextEntitiesCatalog } from './NextEntitiesCatalog';
import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog';
describe('NextEntitiesCatalog', () => {
describe('DefaultEntitiesCatalog', () => {
const databases = TestDatabases.create({
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
});
@@ -151,7 +151,7 @@ describe('NextEntitiesCatalog', () => {
await addEntity(knex, parent, [{ entity: grandparent }]);
await addEntity(knex, root, [{ entity: parent }]);
const catalog = new NextEntitiesCatalog(knex);
const catalog = new DefaultEntitiesCatalog(knex);
const result = await catalog.entityAncestry('k:default/root');
expect(result.rootEntityRef).toEqual('k:default/root');
@@ -181,7 +181,7 @@ describe('NextEntitiesCatalog', () => {
'should throw error if the entity does not exist, %p',
async databaseId => {
const { knex } = await createDatabase(databaseId);
const catalog = new NextEntitiesCatalog(knex);
const catalog = new DefaultEntitiesCatalog(knex);
await expect(() =>
catalog.entityAncestry('k:default/root'),
).rejects.toThrow('No such entity k:default/root');
@@ -224,7 +224,7 @@ describe('NextEntitiesCatalog', () => {
await addEntity(knex, parent2, [{ entity: grandparent }]);
await addEntity(knex, root, [{ entity: parent1 }, { entity: parent2 }]);
const catalog = new NextEntitiesCatalog(knex);
const catalog = new DefaultEntitiesCatalog(knex);
const result = await catalog.entityAncestry('k:default/root');
expect(result.rootEntityRef).toEqual('k:default/root');
@@ -280,7 +280,7 @@ describe('NextEntitiesCatalog', () => {
};
await addEntityToSearch(knex, entity1);
await addEntityToSearch(knex, entity2);
const catalog = new NextEntitiesCatalog(knex);
const catalog = new DefaultEntitiesCatalog(knex);
const testFilter = {
key: 'spec.test',
@@ -313,7 +313,7 @@ describe('NextEntitiesCatalog', () => {
};
await addEntityToSearch(knex, entity1);
await addEntityToSearch(knex, entity2);
const catalog = new NextEntitiesCatalog(knex);
const catalog = new DefaultEntitiesCatalog(knex);
const testFilter = {
not: {
@@ -360,7 +360,7 @@ describe('NextEntitiesCatalog', () => {
await addEntityToSearch(knex, entity2);
await addEntityToSearch(knex, entity3);
await addEntityToSearch(knex, entity4);
const catalog = new NextEntitiesCatalog(knex);
const catalog = new DefaultEntitiesCatalog(knex);
const testFilter1 = {
key: 'metadata.org',
@@ -415,7 +415,7 @@ describe('NextEntitiesCatalog', () => {
};
await addEntityToSearch(knex, entity1);
await addEntityToSearch(knex, entity2);
const catalog = new NextEntitiesCatalog(knex);
const catalog = new DefaultEntitiesCatalog(knex);
const testFilter1 = {
key: 'metadata.org',
@@ -457,7 +457,7 @@ describe('NextEntitiesCatalog', () => {
};
await addEntityToSearch(knex, entity1);
await addEntityToSearch(knex, entity2);
const catalog = new NextEntitiesCatalog(knex);
const catalog = new DefaultEntitiesCatalog(knex);
const testFilter = {
key: 'kind',
@@ -518,7 +518,7 @@ describe('NextEntitiesCatalog', () => {
await addEntity(knex, unrelated, []);
await knex('refresh_state').update({ result_hash: 'not-changed' });
const catalog = new NextEntitiesCatalog(knex);
const catalog = new DefaultEntitiesCatalog(knex);
await catalog.removeEntityByUid(uid);
await expect(
plugins/catalog-backend/src/service/NextEntitiesCatalog.ts → plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts
+1 -5
View File
@@ -150,7 +150,7 @@ function parseFilter(
});
}
export class NextEntitiesCatalog implements EntitiesCatalog {
export class DefaultEntitiesCatalog implements EntitiesCatalog {
constructor(private readonly database: Knex) {}
async entities(request?: EntitiesRequest): Promise<EntitiesResponse> {
@@ -295,8 +295,4 @@ export class NextEntitiesCatalog implements EntitiesCatalog {
items,
};
}
async batchAddOrUpdateEntities(): Promise<never> {
throw new Error('Not implemented');
}
}
plugins/catalog-backend/src/service/NextRouter.test.ts → plugins/catalog-backend/src/service/createRouter.test.ts
+6 -9
View File
@@ -23,12 +23,12 @@ import request from 'supertest';
import { EntitiesCatalog } from '../catalog';
import { LocationService, RefreshService } from './types';
import { basicEntityFilter } from './request';
import { createNextRouter } from './NextRouter';
import { createRouter } from './createRouter';
import { AuthorizeResult } from '@backstage/plugin-permission-common';
import { createPermissionIntegrationRouter } from '@backstage/plugin-permission-node';
import { RESOURCE_TYPE_CATALOG_ENTITY } from '@backstage/plugin-catalog-common';
describe('createNextRouter readonly disabled', () => {
describe('createRouter readonly disabled', () => {
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
let locationService: jest.Mocked<LocationService>;
let app: express.Express;
@@ -38,7 +38,6 @@ describe('createNextRouter readonly disabled', () => {
entitiesCatalog = {
entities: jest.fn(),
removeEntityByUid: jest.fn(),
batchAddOrUpdateEntities: jest.fn(),
entityAncestry: jest.fn(),
};
locationService = {
@@ -48,7 +47,7 @@ describe('createNextRouter readonly disabled', () => {
deleteLocation: jest.fn(),
};
refreshService = { refresh: jest.fn() };
const router = await createNextRouter({
const router = await createRouter({
entitiesCatalog,
locationService,
logger: getVoidLogger(),
@@ -323,7 +322,7 @@ describe('createNextRouter readonly disabled', () => {
});
});
describe('createNextRouter readonly enabled', () => {
describe('createRouter readonly enabled', () => {
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
let app: express.Express;
let locationService: jest.Mocked<LocationService>;
@@ -332,7 +331,6 @@ describe('createNextRouter readonly enabled', () => {
entitiesCatalog = {
entities: jest.fn(),
removeEntityByUid: jest.fn(),
batchAddOrUpdateEntities: jest.fn(),
entityAncestry: jest.fn(),
};
locationService = {
@@ -341,7 +339,7 @@ describe('createNextRouter readonly enabled', () => {
listLocations: jest.fn(),
deleteLocation: jest.fn(),
};
const router = await createNextRouter({
const router = await createRouter({
entitiesCatalog,
locationService,
logger: getVoidLogger(),
@@ -466,7 +464,6 @@ describe('NextRouter permissioning', () => {
entitiesCatalog = {
entities: jest.fn(),
removeEntityByUid: jest.fn(),
batchAddOrUpdateEntities: jest.fn(),
entityAncestry: jest.fn(),
};
locationService = {
@@ -476,7 +473,7 @@ describe('NextRouter permissioning', () => {
deleteLocation: jest.fn(),
};
refreshService = { refresh: jest.fn() };
const router = await createNextRouter({
const router = await createRouter({
entitiesCatalog,
locationService,
logger: getVoidLogger(),
plugins/catalog-backend/src/service/NextRouter.ts → plugins/catalog-backend/src/service/createRouter.ts
+15 -5
View File
@@ -33,11 +33,16 @@ import {
parseEntityFilterParams,
parseEntityPaginationParams,
parseEntityTransformParams,
} from '../service/request';
import { disallowReadonlyMode, validateRequestBody } from '../service/util';
} from './request';
import { disallowReadonlyMode, validateRequestBody } from './util';
import { RefreshOptions, LocationService, RefreshService } from './types';
export interface NextRouterOptions {
/**
* Options used by {@link createRouter}.
*
* @public
*/
export interface RouterOptions {
entitiesCatalog?: EntitiesCatalog;
locationAnalyzer?: LocationAnalyzer;
locationService: LocationService;
@@ -47,8 +52,13 @@ export interface NextRouterOptions {
permissionIntegrationRouter?: express.Router;
}
export async function createNextRouter(
options: NextRouterOptions,
/**
* Creates a catalog router.
*
* @public
*/
export async function createRouter(
options: RouterOptions,
): Promise<express.Router> {
const {
entitiesCatalog,
plugins/catalog-backend/src/service/index.ts
+4 -4
View File
@@ -20,7 +20,7 @@ export type {
RefreshOptions,
LocationStore,
} from './types';
export { createNextRouter } from './NextRouter';
export type { NextRouterOptions } from './NextRouter';
export type { CatalogEnvironment } from './NextCatalogBuilder';
export { NextCatalogBuilder } from './NextCatalogBuilder';
export { createRouter } from './createRouter';
export type { RouterOptions } from './createRouter';
export type { CatalogEnvironment } from './CatalogBuilder';
export { CatalogBuilder } from './CatalogBuilder';
plugins/catalog-backend/src/service/standaloneServer.ts
+21 -20
View File
@@ -16,18 +16,19 @@
import {
createServiceBuilder,
DatabaseManager,
loadBackendConfig,
ServerTokenManager,
SingleHostDiscovery,
UrlReaders,
useHotMemoize,
} from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { ServerPermissionClient } from '@backstage/plugin-permission-node';
import { Server } from 'http';
import { Logger } from 'winston';
import { DatabaseManager } from '../legacy/database';
import { CatalogBuilder } from '../legacy/service/CatalogBuilder';
import { createRouter } from '../legacy/service';
import { ServerPermissionClient } from '@backstage/plugin-permission-node';
import { applyDatabaseMigrations } from '../database/migrations';
import { CatalogBuilder } from './CatalogBuilder';
export interface ServerOptions {
port: number;
@@ -42,38 +43,38 @@ export async function startStandaloneServer(
const logger = options.logger.child({ service: 'catalog-backend' });
const config = await loadBackendConfig({ logger, argv: process.argv });
const reader = UrlReaders.default({ logger, config });
const db = useHotMemoize(module, () =>
DatabaseManager.createInMemoryDatabaseConnection(),
);
const database = useHotMemoize(module, () => {
const manager = DatabaseManager.fromConfig(
new ConfigReader({
backend: { database: { client: 'sqlite3', connection: ':memory:' } },
}),
);
return manager.forPlugin('catalog');
});
const discovery = SingleHostDiscovery.fromConfig(config);
const tokenManager = ServerTokenManager.fromConfig(config, { logger });
const tokenManager = ServerTokenManager.fromConfig(config, {
logger,
});
const permissions = ServerPermissionClient.fromConfig(config, {
discovery,
tokenManager,
});
logger.debug('Creating application...');
const builder = new CatalogBuilder({
await applyDatabaseMigrations(await database.getClient());
const builder = CatalogBuilder.create({
logger,
database: { getClient: () => db },
database,
config,
reader,
permissions,
});
const { entitiesCatalog, locationsCatalog, higherOrderOperation } =
await builder.build();
const catalog = await builder.build();
logger.debug('Starting application server...');
const router = await createRouter({
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
logger,
config,
});
let service = createServiceBuilder(module)
.setPort(options.port)
.addRouter('/catalog', router);
.addRouter('/catalog', catalog.router);
if (options.enableCors) {
service = service.enableCors({ origin: 'http://localhost:3000' });
}