Merge pull request #7566 from backstage/mob/catalog-cleaning

Catalog: Remove next folder from catalog internals
This commit is contained in:
Johan Haals
2021-10-12 14:51:25 +02:00
committed by GitHub
57 changed files with 112 additions and 263 deletions
@@ -28,7 +28,7 @@ import {
DbRefreshStateRow,
DbRelationsRow,
} from './tables';
import { createRandomRefreshInterval } from '../refresh';
import { createRandomRefreshInterval } from '../processing/refresh';
import { timestampToDateTime } from './conversion';
import { generateStableHash } from './util';
@@ -33,7 +33,7 @@ import {
UpdateEntityCacheOptions,
} from './types';
import { DeferredEntity } from '../processing/types';
import { RefreshIntervalFunction } from '../refresh';
import { RefreshIntervalFunction } from '../processing/refresh';
import { rethrowError, timestampToDateTime } from './conversion';
import { initDatabaseMetrics } from './metrics';
import {
@@ -15,7 +15,7 @@
*/
import { Knex } from 'knex';
import { createGaugeMetric } from '../metrics';
import { createGaugeMetric } from '../util/metrics';
import { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables';
export function initDatabaseMetrics(knex: Knex) {
+3 -1
View File
@@ -25,4 +25,6 @@ export * from './ingestion';
export * from './legacy';
export * from './search';
export * from './util';
export * from './next';
export * from './processing';
export * from './providers';
export * from './service';
@@ -23,7 +23,7 @@ import { DatabaseManager } from '../database';
import { CatalogProcessorParser } from '../../ingestion';
import * as result from '../../ingestion/processors/results';
import { CatalogBuilder } from './CatalogBuilder';
import { CatalogEnvironment } from '../../next';
import { CatalogEnvironment } from '../../service';
const dummyEntity = {
apiVersion: 'backstage.io/v1alpha1',
@@ -64,7 +64,7 @@ import {
} from '../../ingestion/processors/PlaceholderProcessor';
import { defaultEntityDataParser } from '../../ingestion/processors/util/parse';
import { LocationAnalyzer } from '../../ingestion/types';
import { CatalogEnvironment, NextCatalogBuilder } from '../../next';
import { CatalogEnvironment, NextCatalogBuilder } from '../../service';
/**
* A builder that helps wire up all of the component parts of the catalog.
@@ -25,7 +25,7 @@ import { LocationResponse, LocationsCatalog } from '../catalog/types';
import { HigherOrderOperation } from '../ingestion/types';
import { createRouter } from './router';
import { basicEntityFilter } from '../../service/request';
import { RefreshService } from '../../next';
import { RefreshService } from '../../service';
describe('createRouter readonly disabled', () => {
let entitiesCatalog: jest.Mocked<Required<EntitiesCatalog>>;
@@ -34,7 +34,7 @@ import {
RefreshService,
LocationService,
RefreshOptions,
} from '../../next/types';
} from '../../service/types';
import {
basicEntityFilter,
parseEntityFilterParams,
@@ -1,39 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Context, ContextKey } from './types';
/**
* A Context implementation that holds a single value, optionally extending an existing context.
*/
export class ContextWithValue implements Context {
static create(parent: Context, key: ContextKey<unknown>, value: unknown) {
return new ContextWithValue(parent, key, value);
}
private constructor(
private readonly parent: Context,
private readonly key: ContextKey<unknown>,
private readonly value: unknown,
) {}
getContextValue<T>(key: ContextKey<T>): T {
if (this.key === key) {
return this.value as T;
}
return this.parent.getContextValue(key);
}
}
@@ -1,37 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TransactionValue } from './TransactionValue';
import { Knex } from 'knex';
import { BackgroundContext } from './BackgroundContext';
describe('TransactionValue Context', () => {
it('should be able to store tx values and retrieve them from a context', () => {
const tx = {} as Knex.Transaction;
const ctx = new BackgroundContext();
const nextCtx = TransactionValue.in(ctx, tx);
expect(TransactionValue.from(nextCtx)).toBe(tx);
});
it('should throw when there is no tx value in the context', () => {
const ctx = new BackgroundContext();
expect(() => TransactionValue.from(ctx)).toThrow(
/No transaction available in context/,
);
});
});
@@ -1,40 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Context, ContextKey } from './types';
import { Knex } from 'knex';
import { ContextWithValue } from './ContextWithValue';
const transactionContextKey = new ContextKey<Knex.Transaction | undefined>(
undefined,
);
/**
* TransactionValue handles the wrapping of a knex transaction in a Context.
*/
export class TransactionValue {
static in(parent: Context, tx: Knex.Transaction) {
return ContextWithValue.create(parent, transactionContextKey, tx);
}
static from(context: Context): Knex.Transaction {
const transaction = context.getContextValue(transactionContextKey);
if (!transaction) {
throw new Error(`No transaction available in context`);
}
return transaction;
}
}
@@ -1,21 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { BackgroundContext } from './BackgroundContext';
export { ContextWithValue } from './ContextWithValue';
export { TransactionValue } from './TransactionValue';
export { ContextKey } from './types';
export type { Context } from './types';
@@ -18,10 +18,10 @@ import { getVoidLogger } from '@backstage/backend-common';
import { Hash } from 'crypto';
import { DateTime } from 'luxon';
import waitForExpect from 'wait-for-expect';
import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase';
import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';
import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine';
import { CatalogProcessingOrchestrator } from './processing/types';
import { Stitcher } from './stitching/Stitcher';
import { CatalogProcessingOrchestrator } from './types';
import { Stitcher } from '../stitching/Stitcher';
describe('DefaultCatalogProcessingEngine', () => {
const db = {
@@ -23,20 +23,20 @@ import { serializeError } from '@backstage/errors';
import { Hash } from 'crypto';
import stableStringify from 'fast-json-stable-stringify';
import { Logger } from 'winston';
import { ProcessingDatabase, RefreshStateItem } from './database/types';
import { createCounterMetric, createSummaryMetric } from './metrics';
import {
CatalogProcessingOrchestrator,
EntityProcessingResult,
} from './processing/types';
import { Stitcher } from './stitching/Stitcher';
import { startTaskPipeline } from './TaskPipeline';
import { ProcessingDatabase, RefreshStateItem } from '../database/types';
import { createCounterMetric, createSummaryMetric } from '../util/metrics';
import {
CatalogProcessingEngine,
CatalogProcessingOrchestrator,
EntityProcessingResult,
} from '../processing/types';
import { Stitcher } from '../stitching/Stitcher';
import { startTaskPipeline } from './TaskPipeline';
import {
EntityProvider,
EntityProviderConnection,
EntityProviderMutation,
} from './types';
} from '../providers/types';
const CACHE_TTL = 5;
@@ -30,10 +30,10 @@ import {
CatalogProcessorEmit,
CatalogProcessorParser,
results,
} from '../../ingestion';
import { CatalogRulesEnforcer } from '../../ingestion/CatalogRules';
} from '../ingestion';
import { CatalogRulesEnforcer } from '../ingestion/CatalogRules';
import { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator';
import { defaultEntityDataParser } from '../../ingestion/processors/util/parse';
import { defaultEntityDataParser } from '../ingestion/processors/util/parse';
import { ConfigReader } from '@backstage/config';
class FooBarProcessor implements CatalogProcessor {
@@ -31,8 +31,8 @@ import { Logger } from 'winston';
import {
CatalogProcessor,
CatalogProcessorParser,
} from '../../ingestion/processors';
import * as results from '../../ingestion/processors/results';
} from '../ingestion/processors';
import * as results from '../ingestion/processors/results';
import {
CatalogProcessingOrchestrator,
EntityProcessingRequest,
@@ -48,7 +48,7 @@ import {
validateEntityEnvelope,
isObject,
} from './util';
import { CatalogRulesEnforcer } from '../../ingestion/CatalogRules';
import { CatalogRulesEnforcer } from '../ingestion/CatalogRules';
import { ProcessorCacheManager } from './ProcessorCacheManager';
type Context = {
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { CatalogProcessor } from '../../ingestion/processors';
import { CatalogProcessor } from '../ingestion/processors';
import { ProcessorCacheManager } from './ProcessorCacheManager';
class MyProcessor implements CatalogProcessor {
@@ -15,8 +15,8 @@
*/
import { JsonObject, JsonValue } from '@backstage/config';
import { CatalogProcessor } from '../../ingestion/processors';
import { CatalogProcessorCache } from '../../ingestion/processors/types';
import { CatalogProcessor } from '../ingestion/processors';
import { CatalogProcessorCache } from '../ingestion/processors/types';
import { isObject } from './util';
class SingleProcessorSubCache implements CatalogProcessorCache {
@@ -22,8 +22,8 @@ import {
stringifyLocationReference,
} from '@backstage/catalog-model';
import { Logger } from 'winston';
import { CatalogProcessorResult } from '../../ingestion';
import { locationSpecToLocationEntity } from '../util';
import { CatalogProcessorResult } from '../ingestion';
import { locationSpecToLocationEntity } from '../util/conversion';
import { DeferredEntity } from './types';
import {
getEntityLocationRef,
@@ -16,8 +16,12 @@
export type {
CatalogProcessingOrchestrator,
CatalogProcessingEngine,
EntityProcessingRequest,
EntityProcessingResult,
DeferredEntity,
} from './types';
export { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator';
export { createRandomRefreshInterval } from './refresh';
export type { RefreshIntervalFunction } from './refresh';
@@ -44,3 +44,8 @@ export type DeferredEntity = {
entity: Entity;
locationKey?: string;
};
export interface CatalogProcessingEngine {
start(): Promise<void>;
stop(): Promise<void>;
}
@@ -16,9 +16,9 @@
import { Config } from '@backstage/config';
import path from 'path';
import { getEntityLocationRef } from './processing/util';
import { getEntityLocationRef } from '../processing/util';
import { EntityProvider, EntityProviderConnection } from './types';
import { locationSpecToLocationEntity } from './util';
import { locationSpecToLocationEntity } from '../util/conversion';
export class ConfigLocationEntityProvider implements EntityProvider {
constructor(private readonly config: Config) {}
@@ -15,7 +15,7 @@
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { v4 as uuid } from 'uuid';
import { applyDatabaseMigrations } from './database/migrations';
import { applyDatabaseMigrations } from '../database/migrations';
import { DefaultLocationStore } from './DefaultLocationStore';
describe('DefaultLocationStore', () => {
@@ -18,14 +18,11 @@ import { Location, LocationSpec } from '@backstage/catalog-model';
import { ConflictError, NotFoundError } from '@backstage/errors';
import { Knex } from 'knex';
import { v4 as uuid } from 'uuid';
import { DbLocationsRow } from './database/tables';
import { getEntityLocationRef } from './processing/util';
import {
EntityProvider,
EntityProviderConnection,
LocationStore,
} from './types';
import { locationSpecToLocationEntity } from './util';
import { DbLocationsRow } from '../database/tables';
import { getEntityLocationRef } from '../processing/util';
import { EntityProvider, EntityProviderConnection } from './types';
import { locationSpecToLocationEntity } from '../util/conversion';
import { LocationStore } from '../service';
export class DefaultLocationStore implements LocationStore, EntityProvider {
private _connection: EntityProviderConnection | undefined;
@@ -14,10 +14,8 @@
* limitations under the License.
*/
export class ContextKey<T> {
constructor(readonly defaultValue: T) {}
}
export interface Context {
getContextValue<T>(key: ContextKey<T>): T;
}
export type {
EntityProvider,
EntityProviderConnection,
EntityProviderMutation,
} from './types';
@@ -14,13 +14,17 @@
* limitations under the License.
*/
import { Context, ContextKey } from './types';
import { DeferredEntity } from '../processing';
/**
* A base Context implementation that does not hold any value.
*/
export class BackgroundContext implements Context {
getContextValue<T>(key: ContextKey<T>): T {
return key.defaultValue;
}
export type EntityProviderMutation =
| { type: 'full'; entities: DeferredEntity[] }
| { type: 'delta'; added: DeferredEntity[]; removed: DeferredEntity[] };
export interface EntityProviderConnection {
applyMutation(mutation: EntityProviderMutation): Promise<void>;
}
export interface EntityProvider {
getProviderName(): string;
connect(connection: EntityProviderConnection): Promise<void>;
}
@@ -15,7 +15,7 @@
*/
import { DefaultLocationService } from './DefaultLocationService';
import { CatalogProcessingOrchestrator } from './processing/types';
import { CatalogProcessingOrchestrator } from '../processing/types';
import { LocationStore } from './types';
describe('DefaultLocationServiceTest', () => {
@@ -23,9 +23,9 @@ import {
import {
CatalogProcessingOrchestrator,
DeferredEntity,
} from './processing/types';
} from '../processing/types';
import { LocationService, LocationStore } from './types';
import { locationSpecToMetadataName } from './util';
import { locationSpecToMetadataName } from '../util/conversion';
export class DefaultLocationService implements LocationService {
constructor(
@@ -19,16 +19,16 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { createHash } from 'crypto';
import { Knex } from 'knex';
import { Logger } from 'winston';
import { applyDatabaseMigrations } from './database/migrations';
import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase';
import { applyDatabaseMigrations } from '../database/migrations';
import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';
import {
DbRefreshStateReferencesRow,
DbRefreshStateRow,
} from './database/tables';
import { ProcessingDatabase } from './database/types';
import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine';
import { EntityProcessingRequest } from './processing/types';
import { Stitcher } from './stitching/Stitcher';
} from '../database/tables';
import { ProcessingDatabase } from '../database/types';
import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine';
import { EntityProcessingRequest } from '../processing/types';
import { Stitcher } from '../stitching/Stitcher';
import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
import { v4 as uuid } from 'uuid';
import { DefaultRefreshService } from './DefaultRefreshService';
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase';
import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';
import { RefreshOptions, RefreshService } from './types';
export class DefaultRefreshService implements RefreshService {
@@ -59,29 +59,27 @@ import {
} from '../ingestion/processors/PlaceholderProcessor';
import { defaultEntityDataParser } from '../ingestion/processors/util/parse';
import { LocationAnalyzer } from '../ingestion/types';
import {
CatalogProcessingEngine,
EntityProvider,
LocationService,
} from '../next/types';
import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider';
import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase';
import { applyDatabaseMigrations } from './database/migrations';
import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine';
import { EntityProvider } from '../providers/types';
import { CatalogProcessingEngine } from '../processing/types';
import { ConfigLocationEntityProvider } from '../providers/ConfigLocationEntityProvider';
import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';
import { applyDatabaseMigrations } from '../database/migrations';
import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine';
import { DefaultLocationService } from './DefaultLocationService';
import { DefaultLocationStore } from './DefaultLocationStore';
import { DefaultLocationStore } from '../providers/DefaultLocationStore';
import { NextEntitiesCatalog } from './NextEntitiesCatalog';
import { DefaultCatalogProcessingOrchestrator } from './processing/DefaultCatalogProcessingOrchestrator';
import { Stitcher } from './stitching/Stitcher';
import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator';
import { Stitcher } from '../stitching/Stitcher';
import {
createRandomRefreshInterval,
RefreshIntervalFunction,
} from './refresh';
} from '../processing/refresh';
import { createNextRouter } from './NextRouter';
import { DefaultRefreshService } from './DefaultRefreshService';
import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules';
import { Config } from '@backstage/config';
import { Logger } from 'winston';
import { LocationService } from './types';
export type CatalogEnvironment = {
logger: Logger;
@@ -18,12 +18,12 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
import { Knex } from 'knex';
import { v4 as uuid } from 'uuid';
import { applyDatabaseMigrations } from './database/migrations';
import { applyDatabaseMigrations } from '../database/migrations';
import {
DbFinalEntitiesRow,
DbRefreshStateReferencesRow,
DbRefreshStateRow,
} from './database/tables';
} from '../database/tables';
import { NextEntitiesCatalog } from './NextEntitiesCatalog';
describe('NextEntitiesCatalog', () => {
@@ -30,7 +30,7 @@ import {
DbRefreshStateRow,
DbSearchRow,
DbPageInfo,
} from './database/tables';
} from '../database/tables';
function parsePagination(input?: EntityPagination): {
limit?: number;
@@ -22,7 +22,7 @@ import express from 'express';
import request from 'supertest';
import { EntitiesCatalog } from '../catalog';
import { LocationService, RefreshService } from './types';
import { basicEntityFilter } from '../service/request';
import { basicEntityFilter } from './request';
import { createNextRouter } from './NextRouter';
describe('createNextRouter readonly disabled', () => {
@@ -14,21 +14,13 @@
* limitations under the License.
*/
export type { CatalogEnvironment } from './NextCatalogBuilder';
export { NextCatalogBuilder } from './NextCatalogBuilder';
export type {
LocationService,
RefreshService,
RefreshOptions,
LocationStore,
} from './types';
export { createNextRouter } from './NextRouter';
export type { NextRouterOptions } from './NextRouter';
export * from './processing';
export { createRandomRefreshInterval } from './refresh';
export type { RefreshIntervalFunction } from './refresh';
export * from './stitching';
export type {
EntityProvider,
EntityProviderConnection,
EntityProviderMutation,
CatalogProcessingEngine,
LocationService,
LocationStore,
RefreshOptions,
RefreshService,
} from './types';
export type { CatalogEnvironment } from './NextCatalogBuilder';
export { NextCatalogBuilder } from './NextCatalogBuilder';
@@ -14,8 +14,7 @@
* limitations under the License.
*/
import { Entity, Location, LocationSpec } from '@backstage/catalog-model';
import { DeferredEntity } from './processing/types';
import { Entity, LocationSpec, Location } from '@backstage/catalog-model';
export interface LocationService {
createLocation(
@@ -27,18 +26,6 @@ export interface LocationService {
deleteLocation(id: string): Promise<void>;
}
export interface LocationStore {
createLocation(spec: LocationSpec): Promise<Location>;
listLocations(): Promise<Location[]>;
getLocation(id: string): Promise<Location>;
deleteLocation(id: string): Promise<void>;
}
export interface CatalogProcessingEngine {
start(): Promise<void>;
stop(): Promise<void>;
}
/**
* Options for requesting a refresh of entities in the catalog.
*
@@ -61,15 +48,9 @@ export interface RefreshService {
refresh(options: RefreshOptions): Promise<void>;
}
export type EntityProviderMutation =
| { type: 'full'; entities: DeferredEntity[] }
| { type: 'delta'; added: DeferredEntity[]; removed: DeferredEntity[] };
export interface EntityProviderConnection {
applyMutation(mutation: EntityProviderMutation): Promise<void>;
}
export interface EntityProvider {
getProviderName(): string;
connect(connection: EntityProviderConnection): Promise<void>;
export interface LocationStore {
createLocation(spec: LocationSpec): Promise<Location>;
listLocations(): Promise<Location[]>;
getLocation(id: string): Promise<Location>;
deleteLocation(id: string): Promise<void>;
}