From ac7f177291a7169c4e4de9be242146d227cde6ad Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 12 Oct 2021 10:27:28 +0200 Subject: [PATCH 1/8] catalog: Move database, processing and stitching out of next Signed-off-by: Johan Haals --- .../database/DefaultProcessingDatabase.test.ts | 2 +- .../{next => }/database/DefaultProcessingDatabase.ts | 2 +- .../src/{next => }/database/conversion.test.ts | 0 .../src/{next => }/database/conversion.ts | 0 .../src/{next => }/database/metrics.ts | 2 +- .../src/{next => }/database/migrations.ts | 0 .../src/{next => }/database/tables.ts | 0 .../catalog-backend/src/{next => }/database/types.ts | 0 .../catalog-backend/src/{next => }/database/util.ts | 0 .../src/next/ConfigLocationEntityProvider.ts | 2 +- .../src/next/DefaultCatalogProcessingEngine.test.ts | 6 +++--- .../src/next/DefaultCatalogProcessingEngine.ts | 6 +++--- .../src/next/DefaultLocationService.test.ts | 2 +- .../src/next/DefaultLocationService.ts | 2 +- .../src/next/DefaultLocationStore.test.ts | 2 +- .../catalog-backend/src/next/DefaultLocationStore.ts | 4 ++-- .../src/next/DefaultRefreshService.test.ts | 12 ++++++------ .../src/next/DefaultRefreshService.ts | 2 +- .../catalog-backend/src/next/NextCatalogBuilder.ts | 8 ++++---- .../src/next/NextEntitiesCatalog.test.ts | 4 ++-- .../catalog-backend/src/next/NextEntitiesCatalog.ts | 2 +- plugins/catalog-backend/src/next/index.ts | 4 ++-- plugins/catalog-backend/src/next/types.ts | 2 +- .../DefaultCatalogProcessingOrchestrator.test.ts | 6 +++--- .../DefaultCatalogProcessingOrchestrator.ts | 6 +++--- .../processing/ProcessorCacheManager.test.ts | 2 +- .../{next => }/processing/ProcessorCacheManager.ts | 4 ++-- .../processing/ProcessorOutputCollector.ts | 4 ++-- .../src/{next => }/processing/index.ts | 0 .../src/{next => }/processing/types.ts | 0 .../src/{next => }/processing/util.ts | 0 .../src/{next => }/stitching/Stitcher.test.ts | 0 .../src/{next => }/stitching/Stitcher.ts | 0 .../{next => }/stitching/buildEntitySearch.test.ts | 0 .../src/{next => }/stitching/buildEntitySearch.ts | 0 .../src/{next => }/stitching/index.ts | 0 .../catalog-backend/src/{next => }/stitching/util.ts | 0 37 files changed, 43 insertions(+), 43 deletions(-) rename plugins/catalog-backend/src/{next => }/database/DefaultProcessingDatabase.test.ts (99%) rename plugins/catalog-backend/src/{next => }/database/DefaultProcessingDatabase.ts (99%) rename plugins/catalog-backend/src/{next => }/database/conversion.test.ts (100%) rename plugins/catalog-backend/src/{next => }/database/conversion.ts (100%) rename plugins/catalog-backend/src/{next => }/database/metrics.ts (97%) rename plugins/catalog-backend/src/{next => }/database/migrations.ts (100%) rename plugins/catalog-backend/src/{next => }/database/tables.ts (100%) rename plugins/catalog-backend/src/{next => }/database/types.ts (100%) rename plugins/catalog-backend/src/{next => }/database/util.ts (100%) rename plugins/catalog-backend/src/{next => }/processing/DefaultCatalogProcessingOrchestrator.test.ts (97%) rename plugins/catalog-backend/src/{next => }/processing/DefaultCatalogProcessingOrchestrator.ts (98%) rename plugins/catalog-backend/src/{next => }/processing/ProcessorCacheManager.test.ts (98%) rename plugins/catalog-backend/src/{next => }/processing/ProcessorCacheManager.ts (96%) rename plugins/catalog-backend/src/{next => }/processing/ProcessorOutputCollector.ts (96%) rename plugins/catalog-backend/src/{next => }/processing/index.ts (100%) rename plugins/catalog-backend/src/{next => }/processing/types.ts (100%) rename plugins/catalog-backend/src/{next => }/processing/util.ts (100%) rename plugins/catalog-backend/src/{next => }/stitching/Stitcher.test.ts (100%) rename plugins/catalog-backend/src/{next => }/stitching/Stitcher.ts (100%) rename plugins/catalog-backend/src/{next => }/stitching/buildEntitySearch.test.ts (100%) rename plugins/catalog-backend/src/{next => }/stitching/buildEntitySearch.ts (100%) rename plugins/catalog-backend/src/{next => }/stitching/index.ts (100%) rename plugins/catalog-backend/src/{next => }/stitching/util.ts (100%) diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts similarity index 99% rename from plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts rename to plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 5f509747f3..e77d2cde7b 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -28,7 +28,7 @@ import { DbRefreshStateRow, DbRelationsRow, } from './tables'; -import { createRandomRefreshInterval } from '../refresh'; +import { createRandomRefreshInterval } from '../next/refresh'; import { timestampToDateTime } from './conversion'; import { generateStableHash } from './util'; diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts similarity index 99% rename from plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts rename to plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index b47154a46f..fd16ca5384 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -33,7 +33,7 @@ import { UpdateEntityCacheOptions, } from './types'; import { DeferredEntity } from '../processing/types'; -import { RefreshIntervalFunction } from '../refresh'; +import { RefreshIntervalFunction } from '../next/refresh'; import { rethrowError, timestampToDateTime } from './conversion'; import { initDatabaseMetrics } from './metrics'; import { diff --git a/plugins/catalog-backend/src/next/database/conversion.test.ts b/plugins/catalog-backend/src/database/conversion.test.ts similarity index 100% rename from plugins/catalog-backend/src/next/database/conversion.test.ts rename to plugins/catalog-backend/src/database/conversion.test.ts diff --git a/plugins/catalog-backend/src/next/database/conversion.ts b/plugins/catalog-backend/src/database/conversion.ts similarity index 100% rename from plugins/catalog-backend/src/next/database/conversion.ts rename to plugins/catalog-backend/src/database/conversion.ts diff --git a/plugins/catalog-backend/src/next/database/metrics.ts b/plugins/catalog-backend/src/database/metrics.ts similarity index 97% rename from plugins/catalog-backend/src/next/database/metrics.ts rename to plugins/catalog-backend/src/database/metrics.ts index 4ad319ed10..0f1ad54820 100644 --- a/plugins/catalog-backend/src/next/database/metrics.ts +++ b/plugins/catalog-backend/src/database/metrics.ts @@ -15,7 +15,7 @@ */ import { Knex } from 'knex'; -import { createGaugeMetric } from '../metrics'; +import { createGaugeMetric } from '../next/metrics'; import { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables'; export function initDatabaseMetrics(knex: Knex) { diff --git a/plugins/catalog-backend/src/next/database/migrations.ts b/plugins/catalog-backend/src/database/migrations.ts similarity index 100% rename from plugins/catalog-backend/src/next/database/migrations.ts rename to plugins/catalog-backend/src/database/migrations.ts diff --git a/plugins/catalog-backend/src/next/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts similarity index 100% rename from plugins/catalog-backend/src/next/database/tables.ts rename to plugins/catalog-backend/src/database/tables.ts diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/database/types.ts similarity index 100% rename from plugins/catalog-backend/src/next/database/types.ts rename to plugins/catalog-backend/src/database/types.ts diff --git a/plugins/catalog-backend/src/next/database/util.ts b/plugins/catalog-backend/src/database/util.ts similarity index 100% rename from plugins/catalog-backend/src/next/database/util.ts rename to plugins/catalog-backend/src/database/util.ts diff --git a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts b/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts index 5deca09944..4f1c93da3e 100644 --- a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts +++ b/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts @@ -16,7 +16,7 @@ import { Config } from '@backstage/config'; import path from 'path'; -import { getEntityLocationRef } from './processing/util'; +import { getEntityLocationRef } from '../processing/util'; import { EntityProvider, EntityProviderConnection } from './types'; import { locationSpecToLocationEntity } from './util'; diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts index 430d7f251f..2bae899514 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts @@ -18,10 +18,10 @@ import { getVoidLogger } from '@backstage/backend-common'; import { Hash } from 'crypto'; import { DateTime } from 'luxon'; import waitForExpect from 'wait-for-expect'; -import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; +import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; -import { CatalogProcessingOrchestrator } from './processing/types'; -import { Stitcher } from './stitching/Stitcher'; +import { CatalogProcessingOrchestrator } from '../processing/types'; +import { Stitcher } from '../stitching/Stitcher'; describe('DefaultCatalogProcessingEngine', () => { const db = { diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index 346a912009..3fb870230b 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -23,13 +23,13 @@ import { serializeError } from '@backstage/errors'; import { Hash } from 'crypto'; import stableStringify from 'fast-json-stable-stringify'; import { Logger } from 'winston'; -import { ProcessingDatabase, RefreshStateItem } from './database/types'; +import { ProcessingDatabase, RefreshStateItem } from '../database/types'; import { createCounterMetric, createSummaryMetric } from './metrics'; import { CatalogProcessingOrchestrator, EntityProcessingResult, -} from './processing/types'; -import { Stitcher } from './stitching/Stitcher'; +} from '../processing/types'; +import { Stitcher } from '../stitching/Stitcher'; import { startTaskPipeline } from './TaskPipeline'; import { CatalogProcessingEngine, diff --git a/plugins/catalog-backend/src/next/DefaultLocationService.test.ts b/plugins/catalog-backend/src/next/DefaultLocationService.test.ts index 78dd9c1009..a089989d20 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationService.test.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationService.test.ts @@ -15,7 +15,7 @@ */ import { DefaultLocationService } from './DefaultLocationService'; -import { CatalogProcessingOrchestrator } from './processing/types'; +import { CatalogProcessingOrchestrator } from '../processing/types'; import { LocationStore } from './types'; describe('DefaultLocationServiceTest', () => { diff --git a/plugins/catalog-backend/src/next/DefaultLocationService.ts b/plugins/catalog-backend/src/next/DefaultLocationService.ts index 678ce9791c..325a2a3612 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationService.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationService.ts @@ -23,7 +23,7 @@ import { import { CatalogProcessingOrchestrator, DeferredEntity, -} from './processing/types'; +} from '../processing/types'; import { LocationService, LocationStore } from './types'; import { locationSpecToMetadataName } from './util'; diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts index f3cefeb14f..b9d129de5f 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts @@ -15,7 +15,7 @@ */ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { v4 as uuid } from 'uuid'; -import { applyDatabaseMigrations } from './database/migrations'; +import { applyDatabaseMigrations } from '../database/migrations'; import { DefaultLocationStore } from './DefaultLocationStore'; describe('DefaultLocationStore', () => { diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.ts index c9ffe9a682..fdada36737 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.ts @@ -18,8 +18,8 @@ import { Location, LocationSpec } from '@backstage/catalog-model'; import { ConflictError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; -import { DbLocationsRow } from './database/tables'; -import { getEntityLocationRef } from './processing/util'; +import { DbLocationsRow } from '../database/tables'; +import { getEntityLocationRef } from '../processing/util'; import { EntityProvider, EntityProviderConnection, diff --git a/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts b/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts index 7b6a181375..42838f39da 100644 --- a/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts +++ b/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts @@ -19,16 +19,16 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { createHash } from 'crypto'; import { Knex } from 'knex'; import { Logger } from 'winston'; -import { applyDatabaseMigrations } from './database/migrations'; -import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; +import { applyDatabaseMigrations } from '../database/migrations'; +import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { DbRefreshStateReferencesRow, DbRefreshStateRow, -} from './database/tables'; -import { ProcessingDatabase } from './database/types'; +} from '../database/tables'; +import { ProcessingDatabase } from '../database/types'; import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; -import { EntityProcessingRequest } from './processing/types'; -import { Stitcher } from './stitching/Stitcher'; +import { EntityProcessingRequest } from '../processing/types'; +import { Stitcher } from '../stitching/Stitcher'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { v4 as uuid } from 'uuid'; import { DefaultRefreshService } from './DefaultRefreshService'; diff --git a/plugins/catalog-backend/src/next/DefaultRefreshService.ts b/plugins/catalog-backend/src/next/DefaultRefreshService.ts index cfc7fa220b..3b982a0e46 100644 --- a/plugins/catalog-backend/src/next/DefaultRefreshService.ts +++ b/plugins/catalog-backend/src/next/DefaultRefreshService.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; +import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { RefreshOptions, RefreshService } from './types'; export class DefaultRefreshService implements RefreshService { diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 8ca93d4bd7..79f6a75143 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -65,14 +65,14 @@ import { LocationService, } from '../next/types'; import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider'; -import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; -import { applyDatabaseMigrations } from './database/migrations'; +import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; +import { applyDatabaseMigrations } from '../database/migrations'; import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; import { DefaultLocationService } from './DefaultLocationService'; import { DefaultLocationStore } from './DefaultLocationStore'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; -import { DefaultCatalogProcessingOrchestrator } from './processing/DefaultCatalogProcessingOrchestrator'; -import { Stitcher } from './stitching/Stitcher'; +import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; +import { Stitcher } from '../stitching/Stitcher'; import { createRandomRefreshInterval, RefreshIntervalFunction, diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts b/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts index 47e92445e9..082aa47df5 100644 --- a/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts @@ -18,12 +18,12 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; -import { applyDatabaseMigrations } from './database/migrations'; +import { applyDatabaseMigrations } from '../database/migrations'; import { DbFinalEntitiesRow, DbRefreshStateReferencesRow, DbRefreshStateRow, -} from './database/tables'; +} from '../database/tables'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; describe('NextEntitiesCatalog', () => { diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts index 4fde078a8b..0891be045d 100644 --- a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts @@ -30,7 +30,7 @@ import { DbRefreshStateRow, DbSearchRow, DbPageInfo, -} from './database/tables'; +} from '../database/tables'; function parsePagination(input?: EntityPagination): { limit?: number; diff --git a/plugins/catalog-backend/src/next/index.ts b/plugins/catalog-backend/src/next/index.ts index bbc68651ea..055478e6f3 100644 --- a/plugins/catalog-backend/src/next/index.ts +++ b/plugins/catalog-backend/src/next/index.ts @@ -18,10 +18,10 @@ export type { CatalogEnvironment } from './NextCatalogBuilder'; export { NextCatalogBuilder } from './NextCatalogBuilder'; export { createNextRouter } from './NextRouter'; export type { NextRouterOptions } from './NextRouter'; -export * from './processing'; +export * from '../processing'; export { createRandomRefreshInterval } from './refresh'; export type { RefreshIntervalFunction } from './refresh'; -export * from './stitching'; +export * from '../stitching'; export type { EntityProvider, EntityProviderConnection, diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index 4131b5835b..18deca42df 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -15,7 +15,7 @@ */ import { Entity, Location, LocationSpec } from '@backstage/catalog-model'; -import { DeferredEntity } from './processing/types'; +import { DeferredEntity } from '../processing/types'; export interface LocationService { createLocation( diff --git a/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts similarity index 97% rename from plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.test.ts rename to plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts index 098f3ac14d..5ae4d58ec7 100644 --- a/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts @@ -30,10 +30,10 @@ import { CatalogProcessorEmit, CatalogProcessorParser, results, -} from '../../ingestion'; -import { CatalogRulesEnforcer } from '../../ingestion/CatalogRules'; +} from '../ingestion'; +import { CatalogRulesEnforcer } from '../ingestion/CatalogRules'; import { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; -import { defaultEntityDataParser } from '../../ingestion/processors/util/parse'; +import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; import { ConfigReader } from '@backstage/config'; class FooBarProcessor implements CatalogProcessor { diff --git a/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts similarity index 98% rename from plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts rename to plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts index 7a3977bcca..e973dcbf0c 100644 --- a/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts @@ -31,8 +31,8 @@ import { Logger } from 'winston'; import { CatalogProcessor, CatalogProcessorParser, -} from '../../ingestion/processors'; -import * as results from '../../ingestion/processors/results'; +} from '../ingestion/processors'; +import * as results from '../ingestion/processors/results'; import { CatalogProcessingOrchestrator, EntityProcessingRequest, @@ -48,7 +48,7 @@ import { validateEntityEnvelope, isObject, } from './util'; -import { CatalogRulesEnforcer } from '../../ingestion/CatalogRules'; +import { CatalogRulesEnforcer } from '../ingestion/CatalogRules'; import { ProcessorCacheManager } from './ProcessorCacheManager'; type Context = { diff --git a/plugins/catalog-backend/src/next/processing/ProcessorCacheManager.test.ts b/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts similarity index 98% rename from plugins/catalog-backend/src/next/processing/ProcessorCacheManager.test.ts rename to plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts index 44c96da1ca..e7f9fb313b 100644 --- a/plugins/catalog-backend/src/next/processing/ProcessorCacheManager.test.ts +++ b/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { CatalogProcessor } from '../../ingestion/processors'; +import { CatalogProcessor } from '../ingestion/processors'; import { ProcessorCacheManager } from './ProcessorCacheManager'; class MyProcessor implements CatalogProcessor { diff --git a/plugins/catalog-backend/src/next/processing/ProcessorCacheManager.ts b/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts similarity index 96% rename from plugins/catalog-backend/src/next/processing/ProcessorCacheManager.ts rename to plugins/catalog-backend/src/processing/ProcessorCacheManager.ts index 16945b7cc9..378974e33e 100644 --- a/plugins/catalog-backend/src/next/processing/ProcessorCacheManager.ts +++ b/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts @@ -15,8 +15,8 @@ */ import { JsonObject, JsonValue } from '@backstage/config'; -import { CatalogProcessor } from '../../ingestion/processors'; -import { CatalogProcessorCache } from '../../ingestion/processors/types'; +import { CatalogProcessor } from '../ingestion/processors'; +import { CatalogProcessorCache } from '../ingestion/processors/types'; import { isObject } from './util'; class SingleProcessorSubCache implements CatalogProcessorCache { diff --git a/plugins/catalog-backend/src/next/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts similarity index 96% rename from plugins/catalog-backend/src/next/processing/ProcessorOutputCollector.ts rename to plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 507def964a..9ff4c9240c 100644 --- a/plugins/catalog-backend/src/next/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -22,8 +22,8 @@ import { stringifyLocationReference, } from '@backstage/catalog-model'; import { Logger } from 'winston'; -import { CatalogProcessorResult } from '../../ingestion'; -import { locationSpecToLocationEntity } from '../util'; +import { CatalogProcessorResult } from '../ingestion'; +import { locationSpecToLocationEntity } from '../next/util'; import { DeferredEntity } from './types'; import { getEntityLocationRef, diff --git a/plugins/catalog-backend/src/next/processing/index.ts b/plugins/catalog-backend/src/processing/index.ts similarity index 100% rename from plugins/catalog-backend/src/next/processing/index.ts rename to plugins/catalog-backend/src/processing/index.ts diff --git a/plugins/catalog-backend/src/next/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts similarity index 100% rename from plugins/catalog-backend/src/next/processing/types.ts rename to plugins/catalog-backend/src/processing/types.ts diff --git a/plugins/catalog-backend/src/next/processing/util.ts b/plugins/catalog-backend/src/processing/util.ts similarity index 100% rename from plugins/catalog-backend/src/next/processing/util.ts rename to plugins/catalog-backend/src/processing/util.ts diff --git a/plugins/catalog-backend/src/next/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts similarity index 100% rename from plugins/catalog-backend/src/next/stitching/Stitcher.test.ts rename to plugins/catalog-backend/src/stitching/Stitcher.test.ts diff --git a/plugins/catalog-backend/src/next/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts similarity index 100% rename from plugins/catalog-backend/src/next/stitching/Stitcher.ts rename to plugins/catalog-backend/src/stitching/Stitcher.ts diff --git a/plugins/catalog-backend/src/next/stitching/buildEntitySearch.test.ts b/plugins/catalog-backend/src/stitching/buildEntitySearch.test.ts similarity index 100% rename from plugins/catalog-backend/src/next/stitching/buildEntitySearch.test.ts rename to plugins/catalog-backend/src/stitching/buildEntitySearch.test.ts diff --git a/plugins/catalog-backend/src/next/stitching/buildEntitySearch.ts b/plugins/catalog-backend/src/stitching/buildEntitySearch.ts similarity index 100% rename from plugins/catalog-backend/src/next/stitching/buildEntitySearch.ts rename to plugins/catalog-backend/src/stitching/buildEntitySearch.ts diff --git a/plugins/catalog-backend/src/next/stitching/index.ts b/plugins/catalog-backend/src/stitching/index.ts similarity index 100% rename from plugins/catalog-backend/src/next/stitching/index.ts rename to plugins/catalog-backend/src/stitching/index.ts diff --git a/plugins/catalog-backend/src/next/stitching/util.ts b/plugins/catalog-backend/src/stitching/util.ts similarity index 100% rename from plugins/catalog-backend/src/next/stitching/util.ts rename to plugins/catalog-backend/src/stitching/util.ts From 37d131ab2f3a7ee4b60f021db23f70b906125d6c Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 12 Oct 2021 10:33:00 +0200 Subject: [PATCH 2/8] Remove unused context package Signed-off-by: Johan Haals --- .../src/next/Context/BackgroundContext.ts | 26 ------------ .../src/next/Context/ContextWithValue.ts | 39 ------------------ .../src/next/Context/TransactionValue.test.ts | 37 ----------------- .../src/next/Context/TransactionValue.ts | 40 ------------------- .../catalog-backend/src/next/Context/index.ts | 21 ---------- .../catalog-backend/src/next/Context/types.ts | 23 ----------- 6 files changed, 186 deletions(-) delete mode 100644 plugins/catalog-backend/src/next/Context/BackgroundContext.ts delete mode 100644 plugins/catalog-backend/src/next/Context/ContextWithValue.ts delete mode 100644 plugins/catalog-backend/src/next/Context/TransactionValue.test.ts delete mode 100644 plugins/catalog-backend/src/next/Context/TransactionValue.ts delete mode 100644 plugins/catalog-backend/src/next/Context/index.ts delete mode 100644 plugins/catalog-backend/src/next/Context/types.ts diff --git a/plugins/catalog-backend/src/next/Context/BackgroundContext.ts b/plugins/catalog-backend/src/next/Context/BackgroundContext.ts deleted file mode 100644 index c41fa8e3c5..0000000000 --- a/plugins/catalog-backend/src/next/Context/BackgroundContext.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Context, ContextKey } from './types'; - -/** - * A base Context implementation that does not hold any value. - */ -export class BackgroundContext implements Context { - getContextValue(key: ContextKey): T { - return key.defaultValue; - } -} diff --git a/plugins/catalog-backend/src/next/Context/ContextWithValue.ts b/plugins/catalog-backend/src/next/Context/ContextWithValue.ts deleted file mode 100644 index cc88c03679..0000000000 --- a/plugins/catalog-backend/src/next/Context/ContextWithValue.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Context, ContextKey } from './types'; - -/** - * A Context implementation that holds a single value, optionally extending an existing context. - */ -export class ContextWithValue implements Context { - static create(parent: Context, key: ContextKey, value: unknown) { - return new ContextWithValue(parent, key, value); - } - - private constructor( - private readonly parent: Context, - private readonly key: ContextKey, - private readonly value: unknown, - ) {} - - getContextValue(key: ContextKey): T { - if (this.key === key) { - return this.value as T; - } - return this.parent.getContextValue(key); - } -} diff --git a/plugins/catalog-backend/src/next/Context/TransactionValue.test.ts b/plugins/catalog-backend/src/next/Context/TransactionValue.test.ts deleted file mode 100644 index 20165eff2f..0000000000 --- a/plugins/catalog-backend/src/next/Context/TransactionValue.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { TransactionValue } from './TransactionValue'; -import { Knex } from 'knex'; -import { BackgroundContext } from './BackgroundContext'; - -describe('TransactionValue Context', () => { - it('should be able to store tx values and retrieve them from a context', () => { - const tx = {} as Knex.Transaction; - const ctx = new BackgroundContext(); - - const nextCtx = TransactionValue.in(ctx, tx); - - expect(TransactionValue.from(nextCtx)).toBe(tx); - }); - - it('should throw when there is no tx value in the context', () => { - const ctx = new BackgroundContext(); - - expect(() => TransactionValue.from(ctx)).toThrow( - /No transaction available in context/, - ); - }); -}); diff --git a/plugins/catalog-backend/src/next/Context/TransactionValue.ts b/plugins/catalog-backend/src/next/Context/TransactionValue.ts deleted file mode 100644 index af069c6d8e..0000000000 --- a/plugins/catalog-backend/src/next/Context/TransactionValue.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Context, ContextKey } from './types'; -import { Knex } from 'knex'; -import { ContextWithValue } from './ContextWithValue'; - -const transactionContextKey = new ContextKey( - undefined, -); - -/** - * TransactionValue handles the wrapping of a knex transaction in a Context. - */ -export class TransactionValue { - static in(parent: Context, tx: Knex.Transaction) { - return ContextWithValue.create(parent, transactionContextKey, tx); - } - - static from(context: Context): Knex.Transaction { - const transaction = context.getContextValue(transactionContextKey); - if (!transaction) { - throw new Error(`No transaction available in context`); - } - return transaction; - } -} diff --git a/plugins/catalog-backend/src/next/Context/index.ts b/plugins/catalog-backend/src/next/Context/index.ts deleted file mode 100644 index 822edba478..0000000000 --- a/plugins/catalog-backend/src/next/Context/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { BackgroundContext } from './BackgroundContext'; -export { ContextWithValue } from './ContextWithValue'; -export { TransactionValue } from './TransactionValue'; -export { ContextKey } from './types'; -export type { Context } from './types'; diff --git a/plugins/catalog-backend/src/next/Context/types.ts b/plugins/catalog-backend/src/next/Context/types.ts deleted file mode 100644 index 062bc884e9..0000000000 --- a/plugins/catalog-backend/src/next/Context/types.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export class ContextKey { - constructor(readonly defaultValue: T) {} -} - -export interface Context { - getContextValue(key: ContextKey): T; -} From 54e5cd14aced55f6d92ce207fce6dcf9aef653be Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 12 Oct 2021 10:46:45 +0200 Subject: [PATCH 3/8] Move EntityProviders to providers Signed-off-by: Johan Haals --- plugins/catalog-backend/src/next/NextCatalogBuilder.ts | 4 ++-- .../{next => providers}/ConfigLocationEntityProvider.test.ts | 2 +- .../src/{next => providers}/ConfigLocationEntityProvider.ts | 4 ++-- .../src/{next => providers}/DefaultLocationStore.test.ts | 0 .../src/{next => providers}/DefaultLocationStore.ts | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) rename plugins/catalog-backend/src/{next => providers}/ConfigLocationEntityProvider.test.ts (98%) rename plugins/catalog-backend/src/{next => providers}/ConfigLocationEntityProvider.ts (93%) rename plugins/catalog-backend/src/{next => providers}/DefaultLocationStore.test.ts (100%) rename plugins/catalog-backend/src/{next => providers}/DefaultLocationStore.ts (98%) diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 79f6a75143..964cf4538a 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -64,12 +64,12 @@ import { EntityProvider, LocationService, } from '../next/types'; -import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider'; +import { ConfigLocationEntityProvider } from '../providers/ConfigLocationEntityProvider'; import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { applyDatabaseMigrations } from '../database/migrations'; import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; import { DefaultLocationService } from './DefaultLocationService'; -import { DefaultLocationStore } from './DefaultLocationStore'; +import { DefaultLocationStore } from '../providers/DefaultLocationStore'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; import { Stitcher } from '../stitching/Stitcher'; diff --git a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.test.ts b/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts similarity index 98% rename from plugins/catalog-backend/src/next/ConfigLocationEntityProvider.test.ts rename to plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts index 0663ad8d04..a44c803c5f 100644 --- a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.test.ts +++ b/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts @@ -17,7 +17,7 @@ import { ConfigReader } from '@backstage/config'; import path from 'path'; import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider'; -import { EntityProviderConnection } from './types'; +import { EntityProviderConnection } from '../next/types'; describe('ConfigLocationEntityProvider', () => { it('should apply mutation with the correct paths in the config', async () => { diff --git a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts b/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts similarity index 93% rename from plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts rename to plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts index 4f1c93da3e..ce88488d8f 100644 --- a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts +++ b/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts @@ -17,8 +17,8 @@ import { Config } from '@backstage/config'; import path from 'path'; import { getEntityLocationRef } from '../processing/util'; -import { EntityProvider, EntityProviderConnection } from './types'; -import { locationSpecToLocationEntity } from './util'; +import { EntityProvider, EntityProviderConnection } from '../next/types'; +import { locationSpecToLocationEntity } from '../next/util'; export class ConfigLocationEntityProvider implements EntityProvider { constructor(private readonly config: Config) {} diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts similarity index 100% rename from plugins/catalog-backend/src/next/DefaultLocationStore.test.ts rename to plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.ts b/plugins/catalog-backend/src/providers/DefaultLocationStore.ts similarity index 98% rename from plugins/catalog-backend/src/next/DefaultLocationStore.ts rename to plugins/catalog-backend/src/providers/DefaultLocationStore.ts index fdada36737..5498b7105e 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/providers/DefaultLocationStore.ts @@ -24,8 +24,8 @@ import { EntityProvider, EntityProviderConnection, LocationStore, -} from './types'; -import { locationSpecToLocationEntity } from './util'; +} from '../next/types'; +import { locationSpecToLocationEntity } from '../next/util'; export class DefaultLocationStore implements LocationStore, EntityProvider { private _connection: EntityProviderConnection | undefined; From ee8d55e02b507098c4a9382faa892171c7c9a96b Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 12 Oct 2021 11:21:40 +0200 Subject: [PATCH 4/8] Move providers and processing to separate folders Signed-off-by: Johan Haals --- .../catalog-backend/src/database/metrics.ts | 2 +- plugins/catalog-backend/src/index.ts | 2 ++ .../src/next/DefaultRefreshService.test.ts | 2 +- .../src/next/NextCatalogBuilder.ts | 10 +++---- plugins/catalog-backend/src/next/index.ts | 4 --- plugins/catalog-backend/src/next/types.ts | 19 ------------ .../DefaultCatalogProcessingEngine.test.ts | 2 +- .../DefaultCatalogProcessingEngine.ts | 6 ++-- .../{next => processing}/TaskPipeline.test.ts | 0 .../src/{next => processing}/TaskPipeline.ts | 0 .../catalog-backend/src/processing/index.ts | 1 + .../catalog-backend/src/processing/types.ts | 5 ++++ .../ConfigLocationEntityProvider.test.ts | 2 +- .../providers/ConfigLocationEntityProvider.ts | 2 +- .../src/providers/DefaultLocationStore.ts | 8 ++--- .../catalog-backend/src/providers/index.ts | 21 +++++++++++++ .../catalog-backend/src/providers/types.ts | 30 +++++++++++++++++++ .../src/{next => util}/metrics.ts | 0 18 files changed, 74 insertions(+), 42 deletions(-) rename plugins/catalog-backend/src/{next => processing}/DefaultCatalogProcessingEngine.test.ts (99%) rename plugins/catalog-backend/src/{next => processing}/DefaultCatalogProcessingEngine.ts (99%) rename plugins/catalog-backend/src/{next => processing}/TaskPipeline.test.ts (100%) rename plugins/catalog-backend/src/{next => processing}/TaskPipeline.ts (100%) create mode 100644 plugins/catalog-backend/src/providers/index.ts create mode 100644 plugins/catalog-backend/src/providers/types.ts rename plugins/catalog-backend/src/{next => util}/metrics.ts (100%) diff --git a/plugins/catalog-backend/src/database/metrics.ts b/plugins/catalog-backend/src/database/metrics.ts index 0f1ad54820..b6c4e248dc 100644 --- a/plugins/catalog-backend/src/database/metrics.ts +++ b/plugins/catalog-backend/src/database/metrics.ts @@ -15,7 +15,7 @@ */ import { Knex } from 'knex'; -import { createGaugeMetric } from '../next/metrics'; +import { createGaugeMetric } from '../util/metrics'; import { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables'; export function initDatabaseMetrics(knex: Knex) { diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index 4d84330f27..0c72926210 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -26,3 +26,5 @@ export * from './legacy'; export * from './search'; export * from './util'; export * from './next'; +export * from './processing'; +export * from './providers'; diff --git a/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts b/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts index 42838f39da..0ea475a528 100644 --- a/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts +++ b/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts @@ -26,7 +26,7 @@ import { DbRefreshStateRow, } from '../database/tables'; import { ProcessingDatabase } from '../database/types'; -import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; +import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; import { EntityProcessingRequest } from '../processing/types'; import { Stitcher } from '../stitching/Stitcher'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 964cf4538a..869af1581f 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -59,15 +59,12 @@ import { } from '../ingestion/processors/PlaceholderProcessor'; import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; import { LocationAnalyzer } from '../ingestion/types'; -import { - CatalogProcessingEngine, - EntityProvider, - LocationService, -} from '../next/types'; +import { EntityProvider } from '../providers/types'; +import { CatalogProcessingEngine } from '../processing/types'; import { ConfigLocationEntityProvider } from '../providers/ConfigLocationEntityProvider'; import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { applyDatabaseMigrations } from '../database/migrations'; -import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; +import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; import { DefaultLocationService } from './DefaultLocationService'; import { DefaultLocationStore } from '../providers/DefaultLocationStore'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; @@ -82,6 +79,7 @@ import { DefaultRefreshService } from './DefaultRefreshService'; import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules'; import { Config } from '@backstage/config'; import { Logger } from 'winston'; +import { LocationService } from './types'; export type CatalogEnvironment = { logger: Logger; diff --git a/plugins/catalog-backend/src/next/index.ts b/plugins/catalog-backend/src/next/index.ts index 055478e6f3..cc3ad0b979 100644 --- a/plugins/catalog-backend/src/next/index.ts +++ b/plugins/catalog-backend/src/next/index.ts @@ -23,10 +23,6 @@ export { createRandomRefreshInterval } from './refresh'; export type { RefreshIntervalFunction } from './refresh'; export * from '../stitching'; export type { - EntityProvider, - EntityProviderConnection, - EntityProviderMutation, - CatalogProcessingEngine, LocationService, LocationStore, RefreshOptions, diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index 18deca42df..4e3d52e205 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -15,7 +15,6 @@ */ import { Entity, Location, LocationSpec } from '@backstage/catalog-model'; -import { DeferredEntity } from '../processing/types'; export interface LocationService { createLocation( @@ -34,11 +33,6 @@ export interface LocationStore { deleteLocation(id: string): Promise; } -export interface CatalogProcessingEngine { - start(): Promise; - stop(): Promise; -} - /** * Options for requesting a refresh of entities in the catalog. * @@ -60,16 +54,3 @@ export interface RefreshService { */ refresh(options: RefreshOptions): Promise; } - -export type EntityProviderMutation = - | { type: 'full'; entities: DeferredEntity[] } - | { type: 'delta'; added: DeferredEntity[]; removed: DeferredEntity[] }; - -export interface EntityProviderConnection { - applyMutation(mutation: EntityProviderMutation): Promise; -} - -export interface EntityProvider { - getProviderName(): string; - connect(connection: EntityProviderConnection): Promise; -} diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts similarity index 99% rename from plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts rename to plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts index 2bae899514..044c3e7c4b 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts @@ -20,7 +20,7 @@ import { DateTime } from 'luxon'; import waitForExpect from 'wait-for-expect'; import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; -import { CatalogProcessingOrchestrator } from '../processing/types'; +import { CatalogProcessingOrchestrator } from './types'; import { Stitcher } from '../stitching/Stitcher'; describe('DefaultCatalogProcessingEngine', () => { diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts similarity index 99% rename from plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts rename to plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index 3fb870230b..d0faff25d5 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -24,19 +24,19 @@ import { Hash } from 'crypto'; import stableStringify from 'fast-json-stable-stringify'; import { Logger } from 'winston'; import { ProcessingDatabase, RefreshStateItem } from '../database/types'; -import { createCounterMetric, createSummaryMetric } from './metrics'; +import { createCounterMetric, createSummaryMetric } from '../util/metrics'; import { + CatalogProcessingEngine, CatalogProcessingOrchestrator, EntityProcessingResult, } from '../processing/types'; import { Stitcher } from '../stitching/Stitcher'; import { startTaskPipeline } from './TaskPipeline'; import { - CatalogProcessingEngine, EntityProvider, EntityProviderConnection, EntityProviderMutation, -} from './types'; +} from '../providers/types'; const CACHE_TTL = 5; diff --git a/plugins/catalog-backend/src/next/TaskPipeline.test.ts b/plugins/catalog-backend/src/processing/TaskPipeline.test.ts similarity index 100% rename from plugins/catalog-backend/src/next/TaskPipeline.test.ts rename to plugins/catalog-backend/src/processing/TaskPipeline.test.ts diff --git a/plugins/catalog-backend/src/next/TaskPipeline.ts b/plugins/catalog-backend/src/processing/TaskPipeline.ts similarity index 100% rename from plugins/catalog-backend/src/next/TaskPipeline.ts rename to plugins/catalog-backend/src/processing/TaskPipeline.ts diff --git a/plugins/catalog-backend/src/processing/index.ts b/plugins/catalog-backend/src/processing/index.ts index 9e03f18f28..7b60a1670b 100644 --- a/plugins/catalog-backend/src/processing/index.ts +++ b/plugins/catalog-backend/src/processing/index.ts @@ -16,6 +16,7 @@ export type { CatalogProcessingOrchestrator, + CatalogProcessingEngine, EntityProcessingRequest, EntityProcessingResult, DeferredEntity, diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index 8b533714ce..4b16ccef32 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -44,3 +44,8 @@ export type DeferredEntity = { entity: Entity; locationKey?: string; }; + +export interface CatalogProcessingEngine { + start(): Promise; + stop(): Promise; +} diff --git a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts b/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts index a44c803c5f..0663ad8d04 100644 --- a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts +++ b/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts @@ -17,7 +17,7 @@ import { ConfigReader } from '@backstage/config'; import path from 'path'; import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider'; -import { EntityProviderConnection } from '../next/types'; +import { EntityProviderConnection } from './types'; describe('ConfigLocationEntityProvider', () => { it('should apply mutation with the correct paths in the config', async () => { diff --git a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts b/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts index ce88488d8f..e497515035 100644 --- a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts +++ b/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts @@ -17,7 +17,7 @@ import { Config } from '@backstage/config'; import path from 'path'; import { getEntityLocationRef } from '../processing/util'; -import { EntityProvider, EntityProviderConnection } from '../next/types'; +import { EntityProvider, EntityProviderConnection } from './types'; import { locationSpecToLocationEntity } from '../next/util'; export class ConfigLocationEntityProvider implements EntityProvider { diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts b/plugins/catalog-backend/src/providers/DefaultLocationStore.ts index 5498b7105e..fdd33a422d 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/providers/DefaultLocationStore.ts @@ -20,12 +20,10 @@ import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; import { DbLocationsRow } from '../database/tables'; import { getEntityLocationRef } from '../processing/util'; -import { - EntityProvider, - EntityProviderConnection, - LocationStore, -} from '../next/types'; +import { EntityProvider, EntityProviderConnection } from './types'; + import { locationSpecToLocationEntity } from '../next/util'; +import { LocationStore } from '../next'; export class DefaultLocationStore implements LocationStore, EntityProvider { private _connection: EntityProviderConnection | undefined; diff --git a/plugins/catalog-backend/src/providers/index.ts b/plugins/catalog-backend/src/providers/index.ts new file mode 100644 index 0000000000..3b4a1c4b7e --- /dev/null +++ b/plugins/catalog-backend/src/providers/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type { + EntityProvider, + EntityProviderConnection, + EntityProviderMutation, +} from './types'; diff --git a/plugins/catalog-backend/src/providers/types.ts b/plugins/catalog-backend/src/providers/types.ts new file mode 100644 index 0000000000..90c5f13215 --- /dev/null +++ b/plugins/catalog-backend/src/providers/types.ts @@ -0,0 +1,30 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DeferredEntity } from '../processing'; + +export type EntityProviderMutation = + | { type: 'full'; entities: DeferredEntity[] } + | { type: 'delta'; added: DeferredEntity[]; removed: DeferredEntity[] }; + +export interface EntityProviderConnection { + applyMutation(mutation: EntityProviderMutation): Promise; +} + +export interface EntityProvider { + getProviderName(): string; + connect(connection: EntityProviderConnection): Promise; +} diff --git a/plugins/catalog-backend/src/next/metrics.ts b/plugins/catalog-backend/src/util/metrics.ts similarity index 100% rename from plugins/catalog-backend/src/next/metrics.ts rename to plugins/catalog-backend/src/util/metrics.ts From 9d3ff6680f2fe0cf26da2bcb16d3436cba6331a4 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 12 Oct 2021 11:49:21 +0200 Subject: [PATCH 5/8] Move router and *Service to service folder Signed-off-by: Johan Haals --- plugins/catalog-backend/src/index.ts | 1 + .../src/legacy/service/router.test.ts | 2 +- .../src/legacy/service/router.ts | 2 +- .../src/next/NextCatalogBuilder.ts | 8 +-- plugins/catalog-backend/src/next/index.ts | 11 +---- plugins/catalog-backend/src/next/types.ts | 34 +------------ .../DefaultLocationService.test.ts | 2 +- .../DefaultLocationService.ts | 5 +- .../DefaultRefreshService.test.ts | 0 .../DefaultRefreshService.ts | 0 .../src/{next => service}/NextRouter.test.ts | 2 +- .../src/{next => service}/NextRouter.ts | 0 plugins/catalog-backend/src/service/index.ts | 19 +++++++ plugins/catalog-backend/src/service/types.ts | 49 +++++++++++++++++++ 14 files changed, 82 insertions(+), 53 deletions(-) rename plugins/catalog-backend/src/{next => service}/DefaultLocationService.test.ts (99%) rename plugins/catalog-backend/src/{next => service}/DefaultLocationService.ts (95%) rename plugins/catalog-backend/src/{next => service}/DefaultRefreshService.test.ts (100%) rename plugins/catalog-backend/src/{next => service}/DefaultRefreshService.ts (100%) rename plugins/catalog-backend/src/{next => service}/NextRouter.test.ts (99%) rename plugins/catalog-backend/src/{next => service}/NextRouter.ts (100%) create mode 100644 plugins/catalog-backend/src/service/index.ts create mode 100644 plugins/catalog-backend/src/service/types.ts diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index 0c72926210..5fe0ca2483 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -28,3 +28,4 @@ export * from './util'; export * from './next'; export * from './processing'; export * from './providers'; +export * from './service'; diff --git a/plugins/catalog-backend/src/legacy/service/router.test.ts b/plugins/catalog-backend/src/legacy/service/router.test.ts index d15d3799b5..edc7d852ac 100644 --- a/plugins/catalog-backend/src/legacy/service/router.test.ts +++ b/plugins/catalog-backend/src/legacy/service/router.test.ts @@ -25,7 +25,7 @@ import { LocationResponse, LocationsCatalog } from '../catalog/types'; import { HigherOrderOperation } from '../ingestion/types'; import { createRouter } from './router'; import { basicEntityFilter } from '../../service/request'; -import { RefreshService } from '../../next'; +import { RefreshService } from '../../service'; describe('createRouter readonly disabled', () => { let entitiesCatalog: jest.Mocked>; diff --git a/plugins/catalog-backend/src/legacy/service/router.ts b/plugins/catalog-backend/src/legacy/service/router.ts index 4dfa9e49f9..e7d097a9c7 100644 --- a/plugins/catalog-backend/src/legacy/service/router.ts +++ b/plugins/catalog-backend/src/legacy/service/router.ts @@ -34,7 +34,7 @@ import { RefreshService, LocationService, RefreshOptions, -} from '../../next/types'; +} from '../../service/types'; import { basicEntityFilter, parseEntityFilterParams, diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 869af1581f..e1723ff627 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -65,7 +65,7 @@ import { ConfigLocationEntityProvider } from '../providers/ConfigLocationEntityP import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { applyDatabaseMigrations } from '../database/migrations'; import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; -import { DefaultLocationService } from './DefaultLocationService'; +import { DefaultLocationService } from '../service/DefaultLocationService'; import { DefaultLocationStore } from '../providers/DefaultLocationStore'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; @@ -74,12 +74,12 @@ import { createRandomRefreshInterval, RefreshIntervalFunction, } from './refresh'; -import { createNextRouter } from './NextRouter'; -import { DefaultRefreshService } from './DefaultRefreshService'; +import { createNextRouter } from '../service/NextRouter'; +import { DefaultRefreshService } from '../service/DefaultRefreshService'; import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules'; import { Config } from '@backstage/config'; import { Logger } from 'winston'; -import { LocationService } from './types'; +import { LocationService } from '../service/types'; export type CatalogEnvironment = { logger: Logger; diff --git a/plugins/catalog-backend/src/next/index.ts b/plugins/catalog-backend/src/next/index.ts index cc3ad0b979..c3fa4316f5 100644 --- a/plugins/catalog-backend/src/next/index.ts +++ b/plugins/catalog-backend/src/next/index.ts @@ -16,15 +16,6 @@ export type { CatalogEnvironment } from './NextCatalogBuilder'; export { NextCatalogBuilder } from './NextCatalogBuilder'; -export { createNextRouter } from './NextRouter'; -export type { NextRouterOptions } from './NextRouter'; -export * from '../processing'; export { createRandomRefreshInterval } from './refresh'; export type { RefreshIntervalFunction } from './refresh'; -export * from '../stitching'; -export type { - LocationService, - LocationStore, - RefreshOptions, - RefreshService, -} from './types'; +export type { LocationStore } from './types'; diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index 4e3d52e205..b5888a5a3a 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -14,17 +14,7 @@ * limitations under the License. */ -import { Entity, Location, LocationSpec } from '@backstage/catalog-model'; - -export interface LocationService { - createLocation( - spec: LocationSpec, - dryRun: boolean, - ): Promise<{ location: Location; entities: Entity[]; exists?: boolean }>; - listLocations(): Promise; - getLocation(id: string): Promise; - deleteLocation(id: string): Promise; -} +import { Location, LocationSpec } from '@backstage/catalog-model'; export interface LocationStore { createLocation(spec: LocationSpec): Promise; @@ -32,25 +22,3 @@ export interface LocationStore { getLocation(id: string): Promise; deleteLocation(id: string): Promise; } - -/** - * Options for requesting a refresh of entities in the catalog. - * - * @public - */ -export type RefreshOptions = { - /** The reference to a single entity that should be refreshed */ - entityRef: string; -}; - -/** - * A service that manages refreshes of entities in the catalog. - * - * @public - */ -export interface RefreshService { - /** - * Request a refresh of entities in the catalog. - */ - refresh(options: RefreshOptions): Promise; -} diff --git a/plugins/catalog-backend/src/next/DefaultLocationService.test.ts b/plugins/catalog-backend/src/service/DefaultLocationService.test.ts similarity index 99% rename from plugins/catalog-backend/src/next/DefaultLocationService.test.ts rename to plugins/catalog-backend/src/service/DefaultLocationService.test.ts index a089989d20..8d9dee0703 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultLocationService.test.ts @@ -16,7 +16,7 @@ import { DefaultLocationService } from './DefaultLocationService'; import { CatalogProcessingOrchestrator } from '../processing/types'; -import { LocationStore } from './types'; +import { LocationStore } from '../next/types'; describe('DefaultLocationServiceTest', () => { const orchestrator: jest.Mocked = { diff --git a/plugins/catalog-backend/src/next/DefaultLocationService.ts b/plugins/catalog-backend/src/service/DefaultLocationService.ts similarity index 95% rename from plugins/catalog-backend/src/next/DefaultLocationService.ts rename to plugins/catalog-backend/src/service/DefaultLocationService.ts index 325a2a3612..bc066823db 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationService.ts +++ b/plugins/catalog-backend/src/service/DefaultLocationService.ts @@ -24,8 +24,9 @@ import { CatalogProcessingOrchestrator, DeferredEntity, } from '../processing/types'; -import { LocationService, LocationStore } from './types'; -import { locationSpecToMetadataName } from './util'; +import { LocationService } from './types'; +import { LocationStore } from '../next/types'; +import { locationSpecToMetadataName } from '../next/util'; export class DefaultLocationService implements LocationService { constructor( diff --git a/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts similarity index 100% rename from plugins/catalog-backend/src/next/DefaultRefreshService.test.ts rename to plugins/catalog-backend/src/service/DefaultRefreshService.test.ts diff --git a/plugins/catalog-backend/src/next/DefaultRefreshService.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.ts similarity index 100% rename from plugins/catalog-backend/src/next/DefaultRefreshService.ts rename to plugins/catalog-backend/src/service/DefaultRefreshService.ts diff --git a/plugins/catalog-backend/src/next/NextRouter.test.ts b/plugins/catalog-backend/src/service/NextRouter.test.ts similarity index 99% rename from plugins/catalog-backend/src/next/NextRouter.test.ts rename to plugins/catalog-backend/src/service/NextRouter.test.ts index 489e7efdb5..57911c74fb 100644 --- a/plugins/catalog-backend/src/next/NextRouter.test.ts +++ b/plugins/catalog-backend/src/service/NextRouter.test.ts @@ -22,7 +22,7 @@ import express from 'express'; import request from 'supertest'; import { EntitiesCatalog } from '../catalog'; import { LocationService, RefreshService } from './types'; -import { basicEntityFilter } from '../service/request'; +import { basicEntityFilter } from './request'; import { createNextRouter } from './NextRouter'; describe('createNextRouter readonly disabled', () => { diff --git a/plugins/catalog-backend/src/next/NextRouter.ts b/plugins/catalog-backend/src/service/NextRouter.ts similarity index 100% rename from plugins/catalog-backend/src/next/NextRouter.ts rename to plugins/catalog-backend/src/service/NextRouter.ts diff --git a/plugins/catalog-backend/src/service/index.ts b/plugins/catalog-backend/src/service/index.ts new file mode 100644 index 0000000000..f47d796693 --- /dev/null +++ b/plugins/catalog-backend/src/service/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type { LocationService, RefreshService, RefreshOptions } from './types'; +export { createNextRouter } from './NextRouter'; +export type { NextRouterOptions } from './NextRouter'; diff --git a/plugins/catalog-backend/src/service/types.ts b/plugins/catalog-backend/src/service/types.ts new file mode 100644 index 0000000000..eedc688bb3 --- /dev/null +++ b/plugins/catalog-backend/src/service/types.ts @@ -0,0 +1,49 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, LocationSpec, Location } from '@backstage/catalog-model'; + +export interface LocationService { + createLocation( + spec: LocationSpec, + dryRun: boolean, + ): Promise<{ location: Location; entities: Entity[]; exists?: boolean }>; + listLocations(): Promise; + getLocation(id: string): Promise; + deleteLocation(id: string): Promise; +} + +/** + * Options for requesting a refresh of entities in the catalog. + * + * @public + */ +export type RefreshOptions = { + /** The reference to a single entity that should be refreshed */ + entityRef: string; +}; + +/** + * A service that manages refreshes of entities in the catalog. + * + * @public + */ +export interface RefreshService { + /** + * Request a refresh of entities in the catalog. + */ + refresh(options: RefreshOptions): Promise; +} From ccbcab2a42ee582eb896b20f7bf1d586ee497553 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 12 Oct 2021 13:24:29 +0200 Subject: [PATCH 6/8] Move refresh to processing Signed-off-by: Johan Haals --- .../src/database/DefaultProcessingDatabase.test.ts | 2 +- .../catalog-backend/src/database/DefaultProcessingDatabase.ts | 2 +- plugins/catalog-backend/src/next/NextCatalogBuilder.ts | 2 +- plugins/catalog-backend/src/next/index.ts | 2 -- plugins/catalog-backend/src/processing/index.ts | 3 +++ plugins/catalog-backend/src/{next => processing}/refresh.ts | 0 6 files changed, 6 insertions(+), 5 deletions(-) rename plugins/catalog-backend/src/{next => processing}/refresh.ts (100%) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index e77d2cde7b..00b3de2fc7 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -28,7 +28,7 @@ import { DbRefreshStateRow, DbRelationsRow, } from './tables'; -import { createRandomRefreshInterval } from '../next/refresh'; +import { createRandomRefreshInterval } from '../processing/refresh'; import { timestampToDateTime } from './conversion'; import { generateStableHash } from './util'; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index fd16ca5384..9785ac3935 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -33,7 +33,7 @@ import { UpdateEntityCacheOptions, } from './types'; import { DeferredEntity } from '../processing/types'; -import { RefreshIntervalFunction } from '../next/refresh'; +import { RefreshIntervalFunction } from '../processing/refresh'; import { rethrowError, timestampToDateTime } from './conversion'; import { initDatabaseMetrics } from './metrics'; import { diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index e1723ff627..2150c10756 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -73,7 +73,7 @@ import { Stitcher } from '../stitching/Stitcher'; import { createRandomRefreshInterval, RefreshIntervalFunction, -} from './refresh'; +} from '../processing/refresh'; import { createNextRouter } from '../service/NextRouter'; import { DefaultRefreshService } from '../service/DefaultRefreshService'; import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules'; diff --git a/plugins/catalog-backend/src/next/index.ts b/plugins/catalog-backend/src/next/index.ts index c3fa4316f5..c37ebf4ff7 100644 --- a/plugins/catalog-backend/src/next/index.ts +++ b/plugins/catalog-backend/src/next/index.ts @@ -16,6 +16,4 @@ export type { CatalogEnvironment } from './NextCatalogBuilder'; export { NextCatalogBuilder } from './NextCatalogBuilder'; -export { createRandomRefreshInterval } from './refresh'; -export type { RefreshIntervalFunction } from './refresh'; export type { LocationStore } from './types'; diff --git a/plugins/catalog-backend/src/processing/index.ts b/plugins/catalog-backend/src/processing/index.ts index 7b60a1670b..dc8b07eceb 100644 --- a/plugins/catalog-backend/src/processing/index.ts +++ b/plugins/catalog-backend/src/processing/index.ts @@ -22,3 +22,6 @@ export type { DeferredEntity, } from './types'; export { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; + +export { createRandomRefreshInterval } from './refresh'; +export type { RefreshIntervalFunction } from './refresh'; diff --git a/plugins/catalog-backend/src/next/refresh.ts b/plugins/catalog-backend/src/processing/refresh.ts similarity index 100% rename from plugins/catalog-backend/src/next/refresh.ts rename to plugins/catalog-backend/src/processing/refresh.ts From 9a6ad82fbae03a668c1cb86f667b0c28a917e9dc Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 12 Oct 2021 13:41:33 +0200 Subject: [PATCH 7/8] Restructure NextcatalogBuilder, util and NextEntitiesCatalog Signed-off-by: Johan Haals --- plugins/catalog-backend/src/index.ts | 1 - .../src/legacy/service/CatalogBuilder.test.ts | 2 +- .../src/legacy/service/CatalogBuilder.ts | 2 +- plugins/catalog-backend/src/next/index.ts | 19 --------------- plugins/catalog-backend/src/next/types.ts | 24 ------------------- .../processing/ProcessorOutputCollector.ts | 2 +- .../providers/ConfigLocationEntityProvider.ts | 2 +- .../src/providers/DefaultLocationStore.ts | 5 ++-- .../service/DefaultLocationService.test.ts | 2 +- .../src/service/DefaultLocationService.ts | 5 ++-- .../{next => service}/NextCatalogBuilder.ts | 8 +++---- .../NextEntitiesCatalog.test.ts | 0 .../{next => service}/NextEntitiesCatalog.ts | 0 plugins/catalog-backend/src/service/index.ts | 9 ++++++- plugins/catalog-backend/src/service/types.ts | 7 ++++++ .../src/{next/util.ts => util/conversion.ts} | 0 16 files changed, 28 insertions(+), 60 deletions(-) delete mode 100644 plugins/catalog-backend/src/next/index.ts delete mode 100644 plugins/catalog-backend/src/next/types.ts rename plugins/catalog-backend/src/{next => service}/NextCatalogBuilder.ts (98%) rename plugins/catalog-backend/src/{next => service}/NextEntitiesCatalog.test.ts (100%) rename plugins/catalog-backend/src/{next => service}/NextEntitiesCatalog.ts (100%) rename plugins/catalog-backend/src/{next/util.ts => util/conversion.ts} (100%) diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index 5fe0ca2483..7ad888bee4 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -25,7 +25,6 @@ export * from './ingestion'; export * from './legacy'; export * from './search'; export * from './util'; -export * from './next'; export * from './processing'; export * from './providers'; export * from './service'; diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts index 1b366130ed..926aa67635 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts @@ -23,7 +23,7 @@ import { DatabaseManager } from '../database'; import { CatalogProcessorParser } from '../../ingestion'; import * as result from '../../ingestion/processors/results'; import { CatalogBuilder } from './CatalogBuilder'; -import { CatalogEnvironment } from '../../next'; +import { CatalogEnvironment } from '../../service'; const dummyEntity = { apiVersion: 'backstage.io/v1alpha1', diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts index 1a151b77d6..b8e45c7cf3 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts @@ -64,7 +64,7 @@ import { } from '../../ingestion/processors/PlaceholderProcessor'; import { defaultEntityDataParser } from '../../ingestion/processors/util/parse'; import { LocationAnalyzer } from '../../ingestion/types'; -import { CatalogEnvironment, NextCatalogBuilder } from '../../next'; +import { CatalogEnvironment, NextCatalogBuilder } from '../../service'; /** * A builder that helps wire up all of the component parts of the catalog. diff --git a/plugins/catalog-backend/src/next/index.ts b/plugins/catalog-backend/src/next/index.ts deleted file mode 100644 index c37ebf4ff7..0000000000 --- a/plugins/catalog-backend/src/next/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export type { CatalogEnvironment } from './NextCatalogBuilder'; -export { NextCatalogBuilder } from './NextCatalogBuilder'; -export type { LocationStore } from './types'; diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts deleted file mode 100644 index b5888a5a3a..0000000000 --- a/plugins/catalog-backend/src/next/types.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Location, LocationSpec } from '@backstage/catalog-model'; - -export interface LocationStore { - createLocation(spec: LocationSpec): Promise; - listLocations(): Promise; - getLocation(id: string): Promise; - deleteLocation(id: string): Promise; -} diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 9ff4c9240c..dfaa25418b 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -23,7 +23,7 @@ import { } from '@backstage/catalog-model'; import { Logger } from 'winston'; import { CatalogProcessorResult } from '../ingestion'; -import { locationSpecToLocationEntity } from '../next/util'; +import { locationSpecToLocationEntity } from '../util/conversion'; import { DeferredEntity } from './types'; import { getEntityLocationRef, diff --git a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts b/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts index e497515035..389cd8d9ff 100644 --- a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts +++ b/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts @@ -18,7 +18,7 @@ import { Config } from '@backstage/config'; import path from 'path'; import { getEntityLocationRef } from '../processing/util'; import { EntityProvider, EntityProviderConnection } from './types'; -import { locationSpecToLocationEntity } from '../next/util'; +import { locationSpecToLocationEntity } from '../util/conversion'; export class ConfigLocationEntityProvider implements EntityProvider { constructor(private readonly config: Config) {} diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts b/plugins/catalog-backend/src/providers/DefaultLocationStore.ts index fdd33a422d..9214cd0bc1 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/providers/DefaultLocationStore.ts @@ -21,9 +21,8 @@ import { v4 as uuid } from 'uuid'; import { DbLocationsRow } from '../database/tables'; import { getEntityLocationRef } from '../processing/util'; import { EntityProvider, EntityProviderConnection } from './types'; - -import { locationSpecToLocationEntity } from '../next/util'; -import { LocationStore } from '../next'; +import { locationSpecToLocationEntity } from '../util/conversion'; +import { LocationStore } from '../service'; export class DefaultLocationStore implements LocationStore, EntityProvider { private _connection: EntityProviderConnection | undefined; diff --git a/plugins/catalog-backend/src/service/DefaultLocationService.test.ts b/plugins/catalog-backend/src/service/DefaultLocationService.test.ts index 8d9dee0703..a089989d20 100644 --- a/plugins/catalog-backend/src/service/DefaultLocationService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultLocationService.test.ts @@ -16,7 +16,7 @@ import { DefaultLocationService } from './DefaultLocationService'; import { CatalogProcessingOrchestrator } from '../processing/types'; -import { LocationStore } from '../next/types'; +import { LocationStore } from './types'; describe('DefaultLocationServiceTest', () => { const orchestrator: jest.Mocked = { diff --git a/plugins/catalog-backend/src/service/DefaultLocationService.ts b/plugins/catalog-backend/src/service/DefaultLocationService.ts index bc066823db..2166341935 100644 --- a/plugins/catalog-backend/src/service/DefaultLocationService.ts +++ b/plugins/catalog-backend/src/service/DefaultLocationService.ts @@ -24,9 +24,8 @@ import { CatalogProcessingOrchestrator, DeferredEntity, } from '../processing/types'; -import { LocationService } from './types'; -import { LocationStore } from '../next/types'; -import { locationSpecToMetadataName } from '../next/util'; +import { LocationService, LocationStore } from './types'; +import { locationSpecToMetadataName } from '../util/conversion'; export class DefaultLocationService implements LocationService { constructor( diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts similarity index 98% rename from plugins/catalog-backend/src/next/NextCatalogBuilder.ts rename to plugins/catalog-backend/src/service/NextCatalogBuilder.ts index 2150c10756..60a1668f37 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts @@ -65,7 +65,7 @@ import { ConfigLocationEntityProvider } from '../providers/ConfigLocationEntityP import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { applyDatabaseMigrations } from '../database/migrations'; import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; -import { DefaultLocationService } from '../service/DefaultLocationService'; +import { DefaultLocationService } from './DefaultLocationService'; import { DefaultLocationStore } from '../providers/DefaultLocationStore'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; @@ -74,12 +74,12 @@ import { createRandomRefreshInterval, RefreshIntervalFunction, } from '../processing/refresh'; -import { createNextRouter } from '../service/NextRouter'; -import { DefaultRefreshService } from '../service/DefaultRefreshService'; +import { createNextRouter } from './NextRouter'; +import { DefaultRefreshService } from './DefaultRefreshService'; import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules'; import { Config } from '@backstage/config'; import { Logger } from 'winston'; -import { LocationService } from '../service/types'; +import { LocationService } from './types'; export type CatalogEnvironment = { logger: Logger; diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/NextEntitiesCatalog.test.ts similarity index 100% rename from plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts rename to plugins/catalog-backend/src/service/NextEntitiesCatalog.test.ts diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts b/plugins/catalog-backend/src/service/NextEntitiesCatalog.ts similarity index 100% rename from plugins/catalog-backend/src/next/NextEntitiesCatalog.ts rename to plugins/catalog-backend/src/service/NextEntitiesCatalog.ts diff --git a/plugins/catalog-backend/src/service/index.ts b/plugins/catalog-backend/src/service/index.ts index f47d796693..5345f79628 100644 --- a/plugins/catalog-backend/src/service/index.ts +++ b/plugins/catalog-backend/src/service/index.ts @@ -14,6 +14,13 @@ * limitations under the License. */ -export type { LocationService, RefreshService, RefreshOptions } from './types'; +export type { + LocationService, + RefreshService, + RefreshOptions, + LocationStore, +} from './types'; export { createNextRouter } from './NextRouter'; export type { NextRouterOptions } from './NextRouter'; +export type { CatalogEnvironment } from './NextCatalogBuilder'; +export { NextCatalogBuilder } from './NextCatalogBuilder'; diff --git a/plugins/catalog-backend/src/service/types.ts b/plugins/catalog-backend/src/service/types.ts index eedc688bb3..05d3d00f4d 100644 --- a/plugins/catalog-backend/src/service/types.ts +++ b/plugins/catalog-backend/src/service/types.ts @@ -47,3 +47,10 @@ export interface RefreshService { */ refresh(options: RefreshOptions): Promise; } + +export interface LocationStore { + createLocation(spec: LocationSpec): Promise; + listLocations(): Promise; + getLocation(id: string): Promise; + deleteLocation(id: string): Promise; +} diff --git a/plugins/catalog-backend/src/next/util.ts b/plugins/catalog-backend/src/util/conversion.ts similarity index 100% rename from plugins/catalog-backend/src/next/util.ts rename to plugins/catalog-backend/src/util/conversion.ts From 55ff928d5017c832e2203f4fc5197ff8157436b4 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 12 Oct 2021 13:45:04 +0200 Subject: [PATCH 8/8] Add changeset Signed-off-by: Johan Haals --- .changeset/six-pumpkins-lie.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/six-pumpkins-lie.md diff --git a/.changeset/six-pumpkins-lie.md b/.changeset/six-pumpkins-lie.md new file mode 100644 index 0000000000..7f28582d91 --- /dev/null +++ b/.changeset/six-pumpkins-lie.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +This change refactors the internal package structure to remove the `next` catalog folder that was used during the implementation and testing phase of the new catalog engine. The implementation is now the default and is therefore restructured to no longer be packaged under `next/`. This refactor does not change catalog imports from other parts of the project.