Move providers and processing to separate folders

Signed-off-by: Johan Haals <johan.haals@gmail.com>

plugins/catalog-backend/src/database/metrics.ts

@@ -15,7 +15,7 @@
  */
 
 import { Knex } from 'knex';
-import { createGaugeMetric } from '../next/metrics';
+import { createGaugeMetric } from '../util/metrics';
 import { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables';
 
 export function initDatabaseMetrics(knex: Knex) {

plugins/catalog-backend/src/index.ts

@@ -26,3 +26,5 @@ export * from './legacy';
 export * from './search';
 export * from './util';
 export * from './next';
+export * from './processing';
+export * from './providers';

plugins/catalog-backend/src/next/DefaultRefreshService.test.ts

@@ -26,7 +26,7 @@ import {
   DbRefreshStateRow,
 } from '../database/tables';
 import { ProcessingDatabase } from '../database/types';
-import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine';
+import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine';
 import { EntityProcessingRequest } from '../processing/types';
 import { Stitcher } from '../stitching/Stitcher';
 import { Entity, stringifyEntityRef } from '@backstage/catalog-model';

plugins/catalog-backend/src/next/NextCatalogBuilder.ts

@@ -59,15 +59,12 @@ import {
 } from '../ingestion/processors/PlaceholderProcessor';
 import { defaultEntityDataParser } from '../ingestion/processors/util/parse';
 import { LocationAnalyzer } from '../ingestion/types';
-import {
-  CatalogProcessingEngine,
-  EntityProvider,
-  LocationService,
-} from '../next/types';
+import { EntityProvider } from '../providers/types';
+import { CatalogProcessingEngine } from '../processing/types';
 import { ConfigLocationEntityProvider } from '../providers/ConfigLocationEntityProvider';
 import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';
 import { applyDatabaseMigrations } from '../database/migrations';
-import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine';
+import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine';
 import { DefaultLocationService } from './DefaultLocationService';
 import { DefaultLocationStore } from '../providers/DefaultLocationStore';
 import { NextEntitiesCatalog } from './NextEntitiesCatalog';
@@ -82,6 +79,7 @@ import { DefaultRefreshService } from './DefaultRefreshService';
 import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules';
 import { Config } from '@backstage/config';
 import { Logger } from 'winston';
+import { LocationService } from './types';
 
 export type CatalogEnvironment = {
   logger: Logger;

plugins/catalog-backend/src/next/index.ts

@@ -23,10 +23,6 @@ export { createRandomRefreshInterval } from './refresh';
 export type { RefreshIntervalFunction } from './refresh';
 export * from '../stitching';
 export type {
-  EntityProvider,
-  EntityProviderConnection,
-  EntityProviderMutation,
-  CatalogProcessingEngine,
   LocationService,
   LocationStore,
   RefreshOptions,

plugins/catalog-backend/src/next/types.ts

@@ -15,7 +15,6 @@
  */
 
 import { Entity, Location, LocationSpec } from '@backstage/catalog-model';
-import { DeferredEntity } from '../processing/types';
 
 export interface LocationService {
   createLocation(
@@ -34,11 +33,6 @@ export interface LocationStore {
   deleteLocation(id: string): Promise<void>;
 }
 
-export interface CatalogProcessingEngine {
-  start(): Promise<void>;
-  stop(): Promise<void>;
-}
-
 /**
  * Options for requesting a refresh of entities in the catalog.
  *
@@ -60,16 +54,3 @@ export interface RefreshService {
    */
   refresh(options: RefreshOptions): Promise<void>;
 }
-
-export type EntityProviderMutation =
-  | { type: 'full'; entities: DeferredEntity[] }
-  | { type: 'delta'; added: DeferredEntity[]; removed: DeferredEntity[] };
-
-export interface EntityProviderConnection {
-  applyMutation(mutation: EntityProviderMutation): Promise<void>;
-}
-
-export interface EntityProvider {
-  getProviderName(): string;
-  connect(connection: EntityProviderConnection): Promise<void>;
-}

plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts

@@ -20,7 +20,7 @@ import { DateTime } from 'luxon';
 import waitForExpect from 'wait-for-expect';
 import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';
 import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine';
-import { CatalogProcessingOrchestrator } from '../processing/types';
+import { CatalogProcessingOrchestrator } from './types';
 import { Stitcher } from '../stitching/Stitcher';
 
 describe('DefaultCatalogProcessingEngine', () => {

plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts

@@ -24,19 +24,19 @@ import { Hash } from 'crypto';
 import stableStringify from 'fast-json-stable-stringify';
 import { Logger } from 'winston';
 import { ProcessingDatabase, RefreshStateItem } from '../database/types';
-import { createCounterMetric, createSummaryMetric } from './metrics';
+import { createCounterMetric, createSummaryMetric } from '../util/metrics';
 import {
+  CatalogProcessingEngine,
   CatalogProcessingOrchestrator,
   EntityProcessingResult,
 } from '../processing/types';
 import { Stitcher } from '../stitching/Stitcher';
 import { startTaskPipeline } from './TaskPipeline';
 import {
-  CatalogProcessingEngine,
   EntityProvider,
   EntityProviderConnection,
   EntityProviderMutation,
-} from './types';
+} from '../providers/types';
 
 const CACHE_TTL = 5;
 

plugins/catalog-backend/src/processing/index.ts

@@ -16,6 +16,7 @@
 
 export type {
   CatalogProcessingOrchestrator,
+  CatalogProcessingEngine,
   EntityProcessingRequest,
   EntityProcessingResult,
   DeferredEntity,

plugins/catalog-backend/src/processing/types.ts

@@ -44,3 +44,8 @@ export type DeferredEntity = {
   entity: Entity;
   locationKey?: string;
 };
+
+export interface CatalogProcessingEngine {
+  start(): Promise<void>;
+  stop(): Promise<void>;
+}

plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts

@@ -17,7 +17,7 @@
 import { ConfigReader } from '@backstage/config';
 import path from 'path';
 import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider';
-import { EntityProviderConnection } from '../next/types';
+import { EntityProviderConnection } from './types';
 
 describe('ConfigLocationEntityProvider', () => {
   it('should apply mutation with the correct paths in the config', async () => {

plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts

@@ -17,7 +17,7 @@
 import { Config } from '@backstage/config';
 import path from 'path';
 import { getEntityLocationRef } from '../processing/util';
-import { EntityProvider, EntityProviderConnection } from '../next/types';
+import { EntityProvider, EntityProviderConnection } from './types';
 import { locationSpecToLocationEntity } from '../next/util';
 
 export class ConfigLocationEntityProvider implements EntityProvider {

plugins/catalog-backend/src/providers/DefaultLocationStore.ts

@@ -20,12 +20,10 @@ import { Knex } from 'knex';
 import { v4 as uuid } from 'uuid';
 import { DbLocationsRow } from '../database/tables';
 import { getEntityLocationRef } from '../processing/util';
-import {
-  EntityProvider,
-  EntityProviderConnection,
-  LocationStore,
-} from '../next/types';
+import { EntityProvider, EntityProviderConnection } from './types';
+
 import { locationSpecToLocationEntity } from '../next/util';
+import { LocationStore } from '../next';
 
 export class DefaultLocationStore implements LocationStore, EntityProvider {
   private _connection: EntityProviderConnection | undefined;

plugins/catalog-backend/src/providers/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type {
+  EntityProvider,
+  EntityProviderConnection,
+  EntityProviderMutation,
+} from './types';

plugins/catalog-backend/src/providers/types.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { DeferredEntity } from '../processing';
+
+export type EntityProviderMutation =
+  | { type: 'full'; entities: DeferredEntity[] }
+  | { type: 'delta'; added: DeferredEntity[]; removed: DeferredEntity[] };
+
+export interface EntityProviderConnection {
+  applyMutation(mutation: EntityProviderMutation): Promise<void>;
+}
+
+export interface EntityProvider {
+  getProviderName(): string;
+  connect(connection: EntityProviderConnection): Promise<void>;
+}