Merge pull request #7483 from backstage/freben/catalog-legacy-2

Continuing the legacy deprecation journey

.changeset/tricky-poems-decide.md

@@ -2,4 +2,10 @@
 '@backstage/plugin-catalog-backend': patch
 ---
 
-First steps of deprecating the old catalog engine
+A number of classes and types, that were part of the old catalog engine implementation, are now formally marked as deprecated. They will be removed entirely from the code base in a future release.
+
+After upgrading to this version, it is recommended that you take a look inside your `packages/backend/src/plugins/catalog.ts` file (using a code editor), to see if you are using any functionality that it marks as deprecated. If you do, please migrate away from it at your earliest convenience.
+
+Migrating to using the new engine implementation is typically a matter of calling `CatalogBuilder.create({ ... })` instead of `new CatalogBuilder({ ... })`.
+
+If you are seeing deprecation warnings for `createRouter`, you can either use the `router` field from the return value from updated catalog builder, or temporarily call `createNextRouter`. The latter will however also be deprecated at a later time.

docs/features/software-catalog/external-integrations.md

@@ -90,8 +90,7 @@ does so!
 ## Creating a Catalog Data Reader Processor
 
 The recommended way of instantiating the catalog backend classes is to use the
-[`CatalogBuilder`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/service/CatalogBuilder.ts),
-as illustrated in the
+`CatalogBuilder`, as illustrated in the
 [example backend here](https://github.com/backstage/backstage/blob/master/packages/backend/src/plugins/catalog.ts).
 We will create a new
 [`CatalogProcessor`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/ingestion/processors/types.ts)

plugins/catalog-backend/api-report.md

@@ -225,6 +225,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor {
 //
 // @public
 export class CatalogBuilder {
+  // @deprecated
   constructor(env: CatalogEnvironment);
   // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
   addEntityPolicy(...policies: EntityPolicy[]): CatalogBuilder;
@@ -522,7 +523,7 @@ export function createRandomRefreshInterval(options: {
 
 // Warning: (ae-missing-release-tag) "createRouter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export function createRouter(options: RouterOptions): Promise<express.Router>;
 
 // Warning: (ae-missing-release-tag) "Database" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -573,7 +574,7 @@ export type Database = {
 
 // Warning: (ae-missing-release-tag) "DatabaseEntitiesCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export class DatabaseEntitiesCatalog implements EntitiesCatalog {
   constructor(database: Database, logger: Logger_2);
   // (undocumented)
@@ -1442,7 +1443,7 @@ export { results };
 
 // Warning: (ae-missing-release-tag) "RouterOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export interface RouterOptions {
   // (undocumented)
   config: Config;

plugins/catalog-backend/src/catalog/index.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
 export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
 export type {
   EntitiesCatalog,

plugins/catalog-backend/src/index.ts

@@ -25,6 +25,5 @@ export * from './database';
 export * from './ingestion';
 export * from './legacy';
 export * from './search';
-export * from './service';
 export * from './util';
 export * from './next';

plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.test.ts

@@ -16,10 +16,10 @@
 
 import { getVoidLogger } from '@backstage/backend-common';
 import { Entity, LOCATION_ANNOTATION } from '@backstage/catalog-model';
-import { Database, DatabaseManager, Transaction } from '../database';
-import { basicEntityFilter } from '../service/request';
+import { Database, DatabaseManager, Transaction } from '../../database';
+import { basicEntityFilter } from '../../service/request';
 import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
-import { EntityUpsertRequest } from './types';
+import { EntityUpsertRequest } from '../../catalog/types';
 
 describe('DatabaseEntitiesCatalog', () => {
   let db: jest.Mocked<Database>;

plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.ts

@@ -26,17 +26,17 @@ import { ConflictError } from '@backstage/errors';
 import { chunk, groupBy } from 'lodash';
 import limiterFactory from 'p-limit';
 import { Logger } from 'winston';
-import type { Database, DbEntityResponse, Transaction } from '../database';
-import { DbEntitiesRequest } from '../database/types';
-import { basicEntityFilter } from '../service/request';
-import { durationText } from '../util/timing';
+import type { Database, DbEntityResponse, Transaction } from '../../database';
+import { DbEntitiesRequest } from '../../database/types';
+import { basicEntityFilter } from '../../service/request';
+import { durationText } from '../../util/timing';
 import type {
   EntitiesCatalog,
   EntitiesRequest,
   EntitiesResponse,
   EntityUpsertRequest,
   EntityUpsertResponse,
-} from './types';
+} from '../../catalog/types';
 
 type BatchContext = {
   kind: string;
@@ -57,6 +57,7 @@ const BATCH_ATTEMPTS = 3;
 // The number of batches that may be ongoing at the same time.
 const BATCH_CONCURRENCY = 3;
 
+/** @deprecated This was part of the legacy catalog engine */
 export class DatabaseEntitiesCatalog implements EntitiesCatalog {
   constructor(
     private readonly database: Database,

plugins/catalog-backend/src/legacy/catalog/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';

plugins/catalog-backend/src/legacy/index.ts

@@ -14,4 +14,6 @@
  * limitations under the License.
  */
 
+export * from './catalog';
 export * from './ingestion';
+export * from './service';

plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.ts

@@ -33,7 +33,7 @@ import {
  * Placeholder for operations that span several catalogs and/or stretches out
  * in time.
  *
- * @deprecated This class was part of the legacy catalog engine
+ * @deprecated This was part of the legacy catalog engine
  */
 export class HigherOrderOperations implements HigherOrderOperation {
   constructor(

plugins/catalog-backend/src/legacy/ingestion/LocationReaders.ts

@@ -62,7 +62,7 @@ const noopCache = {
 /**
  * Implements the reading of a location through a series of processor tasks.
  *
- * @deprecated This class was part of the legacy catalog engine
+ * @deprecated This was part of the legacy catalog engine
  */
 export class LocationReaders implements LocationReader {
   private readonly options: Options;

plugins/catalog-backend/src/legacy/ingestion/types.ts

@@ -25,7 +25,7 @@ import {
 // LocationReader
 //
 
-/** @deprecated This class was part of the legacy catalog engine */
+/** @deprecated This was part of the legacy catalog engine */
 export type HigherOrderOperation = {
   addLocation(
     spec: LocationSpec,
@@ -34,7 +34,7 @@ export type HigherOrderOperation = {
   refreshAllLocations(): Promise<void>;
 };
 
-/** @deprecated This class was part of the legacy catalog engine */
+/** @deprecated This was part of the legacy catalog engine */
 export type AddLocationResult = {
   location: Location;
   entities: Entity[];
@@ -44,7 +44,7 @@ export type AddLocationResult = {
 // LocationReader
 //
 
-/** @deprecated This class was part of the legacy catalog engine */
+/** @deprecated This was part of the legacy catalog engine */
 export type LocationReader = {
   /**
    * Reads the contents of a location.
@@ -56,20 +56,20 @@ export type LocationReader = {
   read(location: LocationSpec): Promise<ReadLocationResult>;
 };
 
-/** @deprecated This class was part of the legacy catalog engine */
+/** @deprecated This was part of the legacy catalog engine */
 export type ReadLocationResult = {
   entities: ReadLocationEntity[];
   errors: ReadLocationError[];
 };
 
-/** @deprecated This class was part of the legacy catalog engine */
+/** @deprecated This was part of the legacy catalog engine */
 export type ReadLocationEntity = {
   location: LocationSpec;
   entity: Entity;
   relations: EntityRelationSpec[];
 };
 
-/** @deprecated This class was part of the legacy catalog engine */
+/** @deprecated This was part of the legacy catalog engine */
 export type ReadLocationError = {
   location: LocationSpec;
   error: Error;

plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts

@@ -19,10 +19,11 @@ import { Entity } from '@backstage/catalog-model';
 import { ConfigReader } from '@backstage/config';
 import { Knex } from 'knex';
 import yaml from 'yaml';
-import { DatabaseManager } from '../database';
-import { CatalogProcessorParser } from '../ingestion';
-import * as result from '../ingestion/processors/results';
-import { CatalogBuilder, CatalogEnvironment } from './CatalogBuilder';
+import { DatabaseManager } from '../../database';
+import { CatalogProcessorParser } from '../../ingestion';
+import * as result from '../../ingestion/processors/results';
+import { CatalogBuilder } from './CatalogBuilder';
+import { CatalogEnvironment } from '../../next';
 
 const dummyEntity = {
   apiVersion: 'backstage.io/v1alpha1',

plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import { PluginDatabaseManager, UrlReader } from '@backstage/backend-common';
 import {
   DefaultNamespaceEntityPolicy,
   EntityPolicies,
@@ -25,17 +24,15 @@ import {
   SchemaValidEntityPolicy,
   Validators,
 } from '@backstage/catalog-model';
-import { Config } from '@backstage/config';
 import { ScmIntegrations } from '@backstage/integration';
 import lodash from 'lodash';
-import { Logger } from 'winston';
 import {
-  DatabaseEntitiesCatalog,
   DatabaseLocationsCatalog,
   EntitiesCatalog,
   LocationsCatalog,
-} from '../catalog';
-import { DatabaseManager } from '../database';
+} from '../../catalog';
+import { DatabaseEntitiesCatalog } from '../catalog';
+import { DatabaseManager } from '../../database';
 import {
   AnnotateLocationEntityProcessor,
   BitbucketDiscoveryProcessor,
@@ -52,29 +49,22 @@ import {
   PlaceholderResolver,
   StaticLocationProcessor,
   UrlReaderProcessor,
-} from '../ingestion';
+} from '../../ingestion';
 import {
   HigherOrderOperation,
   HigherOrderOperations,
   LocationReaders,
-} from '../legacy/ingestion';
-import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules';
-import { RepoLocationAnalyzer } from '../ingestion/LocationAnalyzer';
+} from '../ingestion';
+import { DefaultCatalogRulesEnforcer } from '../../ingestion/CatalogRules';
+import { RepoLocationAnalyzer } from '../../ingestion/LocationAnalyzer';
 import {
   jsonPlaceholderResolver,
   textPlaceholderResolver,
   yamlPlaceholderResolver,
-} from '../ingestion/processors/PlaceholderProcessor';
-import { defaultEntityDataParser } from '../ingestion/processors/util/parse';
-import { LocationAnalyzer } from '../ingestion/types';
-import { NextCatalogBuilder } from '../next';
-
-export type CatalogEnvironment = {
-  logger: Logger;
-  database: PluginDatabaseManager;
-  config: Config;
-  reader: UrlReader;
-};
+} from '../../ingestion/processors/PlaceholderProcessor';
+import { defaultEntityDataParser } from '../../ingestion/processors/util/parse';
+import { LocationAnalyzer } from '../../ingestion/types';
+import { CatalogEnvironment, NextCatalogBuilder } from '../../next';
 
 /**
  * A builder that helps wire up all of the component parts of the catalog.
@@ -94,6 +84,10 @@ export type CatalogEnvironment = {
  * - Processors can be added or replaced. These implement the functionality of
  *   reading, parsing, validating, and processing the entity data before it is
  *   persisted in the catalog.
+ *
+ * NOTE(freben): Not actually marking the class as deprecated formally, since
+ * it would appear to end users that even using `create` is deprecated. We will
+ * instead hot-swap the entire exported class when we are ready.
  */
 export class CatalogBuilder {
   private readonly env: CatalogEnvironment;
@@ -109,6 +103,7 @@ export class CatalogBuilder {
     return new NextCatalogBuilder(env);
   }
 
+  /** @deprecated Please use CatalogBuilder.create() instead */
   constructor(env: CatalogEnvironment) {
     this.env = env;
     this.entityPolicies = [];

plugins/catalog-backend/src/legacy/service/index.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020 The Backstage Authors
+ * Copyright 2021 The Backstage Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,5 @@
  */
 
 export { CatalogBuilder } from './CatalogBuilder';
-export type { CatalogEnvironment } from './CatalogBuilder';
 export { createRouter } from './router';
 export type { RouterOptions } from './router';

plugins/catalog-backend/src/legacy/service/router.test.ts

@@ -20,12 +20,12 @@ import { NotFoundError } from '@backstage/errors';
 import type { Entity, LocationSpec } from '@backstage/catalog-model';
 import express from 'express';
 import request from 'supertest';
-import { EntitiesCatalog, LocationsCatalog } from '../catalog';
-import { LocationResponse } from '../catalog/types';
-import { HigherOrderOperation } from '../legacy/ingestion/types';
+import { EntitiesCatalog, LocationsCatalog } from '../../catalog';
+import { LocationResponse } from '../../catalog/types';
+import { HigherOrderOperation } from '../ingestion/types';
 import { createRouter } from './router';
-import { basicEntityFilter } from './request';
-import { RefreshService } from '../next';
+import { basicEntityFilter } from '../../service/request';
+import { RefreshService } from '../../next';
 
 describe('createRouter readonly disabled', () => {
   let entitiesCatalog: jest.Mocked<EntitiesCatalog>;

plugins/catalog-backend/src/legacy/service/router.ts

@@ -26,22 +26,27 @@ import express from 'express';
 import Router from 'express-promise-router';
 import { Logger } from 'winston';
 import yn from 'yn';
-import { EntitiesCatalog, LocationsCatalog } from '../catalog';
-import { LocationAnalyzer } from '../ingestion/types';
-import { HigherOrderOperation } from '../legacy/ingestion/types';
-import { RefreshService, LocationService, RefreshOptions } from '../next/types';
+import { EntitiesCatalog, LocationsCatalog } from '../../catalog';
+import { LocationAnalyzer } from '../../ingestion/types';
+import { HigherOrderOperation } from '../ingestion/types';
+import {
+  RefreshService,
+  LocationService,
+  RefreshOptions,
+} from '../../next/types';
 import {
   basicEntityFilter,
   parseEntityFilterParams,
   parseEntityPaginationParams,
   parseEntityTransformParams,
-} from './request';
+} from '../../service/request';
 import {
   disallowReadonlyMode,
   requireRequestBody,
   validateRequestBody,
-} from './util';
+} from '../../service/util';
 
+/** @deprecated This was part of the legacy catalog engine */
 export interface RouterOptions {
   entitiesCatalog?: EntitiesCatalog;
   locationsCatalog?: LocationsCatalog;
@@ -53,6 +58,7 @@ export interface RouterOptions {
   config: Config;
 }
 
+/** @deprecated This was part of the legacy catalog engine */
 export async function createRouter(
   options: RouterOptions,
 ): Promise<express.Router> {

plugins/catalog-backend/src/next/NextCatalogBuilder.ts

@@ -14,7 +14,11 @@
  * limitations under the License.
  */
 
-import { resolvePackagePath } from '@backstage/backend-common';
+import {
+  PluginDatabaseManager,
+  resolvePackagePath,
+  UrlReader,
+} from '@backstage/backend-common';
 import {
   DefaultNamespaceEntityPolicy,
   EntityPolicies,
@@ -75,10 +79,18 @@ import {
   createRandomRefreshInterval,
   RefreshIntervalFunction,
 } from './refresh';
-import { CatalogEnvironment } from '../service/CatalogBuilder';
 import { createNextRouter } from './NextRouter';
 import { DefaultRefreshService } from './DefaultRefreshService';
 import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules';
+import { Config } from '@backstage/config';
+import { Logger } from 'winston';
+
+export type CatalogEnvironment = {
+  logger: Logger;
+  database: PluginDatabaseManager;
+  config: Config;
+  reader: UrlReader;
+};
 
 /**
  * A builder that helps wire up all of the component parts of the catalog.

plugins/catalog-backend/src/next/index.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+export type { CatalogEnvironment } from './NextCatalogBuilder';
 export { NextCatalogBuilder } from './NextCatalogBuilder';
 export { createNextRouter } from './NextRouter';
 export type { NextRouterOptions } from './NextRouter';

plugins/catalog-backend/src/service/standaloneServer.ts

@@ -23,8 +23,8 @@ import {
 import { Server } from 'http';
 import { Logger } from 'winston';
 import { DatabaseManager } from '../database';
-import { CatalogBuilder } from './CatalogBuilder';
-import { createRouter } from './router';
+import { CatalogBuilder } from '../legacy/service/CatalogBuilder';
+import { createRouter } from '../legacy/service';
 
 export interface ServerOptions {
   port: number;
@@ -32,6 +32,7 @@ export interface ServerOptions {
   logger: Logger;
 }
 
+// TODO(freben): Migrate to the next catalog when it's in place
 export async function startStandaloneServer(
   options: ServerOptions,
 ): Promise<Server> {