Merge branch 'backstage:master' into awanlin/azure-devops-frontend-plugin

.changeset/honest-drinks-eat.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-graphiql': patch
+---
+
+Add experimental `experimentalInstallationRecipe` to `package.json`.

.changeset/khaki-planets-prove.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+add a --from <location> option to the plugin install command

.changeset/ninety-islands-compare.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Bump sucrase to version 3.20.2

.changeset/olive-ants-allow.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-auth-backend': patch
+---
+
+Added extra configuration parameters for active directory file system identity

.changeset/popular-students-sniff.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-code-coverage-backend': patch
+---
+
+check for existence of lines property in files

.changeset/spicy-goats-help.md

@@ -0,0 +1,7 @@
+---
+'@backstage/cli': patch
+---
+
+Add an experimental `install <plugin>` command.
+
+Given a `pluginId`, the command looks for NPM packages matching `@backstage/plugin-{pluginId}` or `backstage-plugin-{pluginId}` or `{pluginId}`. It looks for the `experimentalInstallationRecipe` in their `package.json` for the steps of installation. Detailed documentation and API Spec to follow (and to be decided as well).

packages/cli/package.json

@@ -104,7 +104,7 @@
     "run-script-webpack-plugin": "^0.0.11",
     "semver": "^7.3.2",
     "style-loader": "^1.2.1",
-    "sucrase": "^3.20.1",
+    "sucrase": "^3.20.2",
     "tar": "^6.1.2",
     "terser-webpack-plugin": "^5.1.3",
     "ts-loader": "^8.0.17",

packages/cli/src/commands/index.ts

@@ -230,6 +230,15 @@ export function registerCommands(program: CommanderStatic) {
     .command('info')
     .description('Show helpful information for debugging and reporting bugs')
     .action(lazy(() => import('./info').then(m => m.default)));
+
+  program
+    .command('install [plugin-id]', { hidden: true })
+    .option(
+      '--from <packageJsonFilePath>',
+      'Install from a local package.json containing the installation recipe',
+    )
+    .description('Install a Backstage plugin [EXPERIMENTAL]')
+    .action(lazy(() => import('./install/install').then(m => m.default)));
 }
 
 // Wraps an action function so that it always exits and handles errors

packages/cli/src/commands/install/install.ts

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Step, PackageWithInstallRecipe } from './types';
+import { fetchPackageInfo } from '../../lib/versioning';
+import { NotFoundError } from '../../lib/errors';
+import * as stepDefinitionMap from './steps';
+import { Command } from 'commander';
+import fs from 'fs-extra';
+
+const stepDefinitions = Object.values(stepDefinitionMap);
+
+async function fetchPluginPackage(
+  id: string,
+): Promise<PackageWithInstallRecipe> {
+  const searchNames = [`@backstage/plugin-${id}`, `backstage-plugin-${id}`, id];
+
+  for (const name of searchNames) {
+    try {
+      const packageInfo = (await fetchPackageInfo(
+        name,
+      )) as PackageWithInstallRecipe;
+      return packageInfo;
+    } catch (error) {
+      if (error.name !== 'NotFoundError') {
+        throw error;
+      }
+    }
+  }
+
+  throw new NotFoundError(
+    `No matching package found for '${id}', tried ${searchNames.join(', ')}`,
+  );
+}
+
+type Steps = Array<{
+  type: string;
+  step: Step;
+}>;
+
+class PluginInstaller {
+  static async resolveSteps(pkg: PackageWithInstallRecipe) {
+    const steps: Steps = [];
+
+    // collectDependencies
+    // TODO: Deps mean the plugin package itself, and any other backstage plugins/packages it depends on, in its installation recipe.
+    const dependencies = [];
+    dependencies.push({
+      target: 'packages/app',
+      type: 'dependencies' as const,
+      name: pkg.name,
+      query: `^${pkg.version}`,
+    });
+    steps.push({
+      type: 'dependencies',
+      step: stepDefinitionMap.dependencies.create({ dependencies }),
+    });
+
+    for (const step of pkg.experimentalInstallationRecipe?.steps ?? []) {
+      const { type } = step;
+
+      const definition = stepDefinitions.find(d => d.type === type);
+      if (definition) {
+        steps.push({
+          type,
+          step: definition.deserialize(step, pkg),
+        });
+      } else {
+        throw new Error(`Unsupported step type: ${type}`);
+      }
+    }
+
+    return steps;
+  }
+
+  constructor(private readonly steps: Steps) {}
+
+  async run() {
+    for (const { type, step } of this.steps) {
+      // TODO(Rugvip): Add spinners, nicer message about the step.
+      console.log(`Running step ${type}`);
+      await step.run();
+    }
+  }
+}
+
+export default async (pluginId?: string, cmd?: Command) => {
+  // TODO(himanshu): If no plugin id is provided, it should list all plugins available. Maybe in some other command?
+
+  let pkg: PackageWithInstallRecipe;
+  if (pluginId) {
+    pkg = await fetchPluginPackage(pluginId);
+  } else if (cmd?.from) {
+    // TODO(himanshu): Also support reading directly from url
+    pkg = await fs.readJson(cmd.from);
+  } else {
+    throw new Error(
+      'Missing both <plugin-id> or a package.json file path in the --from flag.',
+    );
+  }
+
+  const steps = await PluginInstaller.resolveSteps(pkg);
+  const installer = new PluginInstaller(steps);
+  await installer.run();
+};

packages/cli/src/commands/install/steps/appRoute.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import { paths } from '../../../lib/paths';
+import { Step, createStepDefinition } from '../types';
+
+type Data = {
+  path: string;
+  element: string;
+  packageName: string;
+};
+
+class AppRouteStep implements Step {
+  constructor(private readonly data: Data) {}
+
+  async run() {
+    const { path, element, packageName } = this.data;
+
+    const appTsxPath = paths.resolveTargetRoot('packages/app/src/App.tsx');
+    const contents = await fs.readFile(appTsxPath, 'utf-8');
+    let failed = false;
+
+    // Add a new route just above the end of the FlatRoutes block
+    const contentsWithRoute = contents.replace(
+      /(\s*)<\/FlatRoutes>/,
+      `$1  <Route path="${path}" element={${element}} />$1</FlatRoutes>`,
+    );
+    if (contentsWithRoute === contents) {
+      failed = true;
+    }
+
+    // Grab the component name from the element
+    const componentName = element.match(/[A-Za-z0-9]+/)?.[0];
+    if (!componentName) {
+      throw new Error(`Could not find component name in ${element}`);
+    }
+
+    // Add plugin import
+    // TODO(Rugvip): Attempt to add this among the other plugin imports
+    const contentsWithImport = contentsWithRoute.replace(
+      /^import /m,
+      `import { ${componentName} } from '${packageName}';\nimport `,
+    );
+    if (contentsWithImport === contentsWithRoute) {
+      failed = true;
+    }
+
+    if (failed) {
+      console.log(
+        'Failed to automatically add a route to package/app/src/App.tsx',
+      );
+      console.log(`Action needed, add the following:`);
+      console.log(`1. import { ${componentName} } from '${packageName}';`);
+      console.log(`2. <Route path="${path}" element={${element}} />`);
+    } else {
+      await fs.writeFile(appTsxPath, contentsWithImport);
+    }
+  }
+}
+
+export const appRoute = createStepDefinition<Data>({
+  type: 'app-route',
+
+  deserialize(obj, pkg) {
+    const { path, element } = obj;
+    if (!path || typeof path !== 'string') {
+      throw new Error("Invalid install step, 'path' must be a string");
+    }
+    if (!element || typeof element !== 'string') {
+      throw new Error("Invalid install step, 'element' must be a string");
+    }
+    return new AppRouteStep({ path, element, packageName: pkg.name });
+  },
+
+  create(data: Data) {
+    return new AppRouteStep(data);
+  },
+});

packages/cli/src/commands/install/steps/dependencies.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import chalk from 'chalk';
+import sortBy from 'lodash/sortBy';
+import groupBy from 'lodash/groupBy';
+import { paths } from '../../../lib/paths';
+import { run } from '../../../lib/run';
+import { Step, createStepDefinition } from '../types';
+
+type Data = {
+  dependencies: Array<{
+    target: string;
+    type: 'dependencies';
+    name: string;
+    query: string;
+  }>;
+};
+
+class DependenciesStep implements Step {
+  constructor(private readonly data: Data) {}
+
+  async run() {
+    const { dependencies } = this.data;
+    // yarn --cwd packages/app add
+    const byTarget = groupBy(dependencies, 'target');
+
+    // Go through each target package and install the dependencies.
+    for (const [target, deps] of Object.entries(byTarget)) {
+      const pkgPath = paths.resolveTargetRoot(target, 'package.json');
+      const pkgJson = await fs.readJson(pkgPath);
+
+      // Populate each type of dependency object, dependencies, devDependencies, etc.
+      const depTypes = new Set<string>();
+      for (const dep of deps) {
+        depTypes.add(dep.type);
+        pkgJson[dep.type][dep.name] = dep.query;
+      }
+
+      // Be nice and sort the dependencies alphabetically
+      for (const depType of depTypes) {
+        pkgJson[depType] = Object.fromEntries(
+          sortBy(Object.entries(pkgJson[depType]), ([key]) => key),
+        );
+      }
+      await fs.writeJson(pkgPath, pkgJson, { spaces: 2 });
+    }
+
+    console.log();
+    console.log(
+      `Running ${chalk.blue('yarn install')} to install new versions`,
+    );
+    console.log();
+    await run('yarn', ['install']);
+  }
+}
+
+export const dependencies = createStepDefinition<Data>({
+  type: 'dependencies',
+
+  deserialize() {
+    throw new Error('The dependency step may not be defined in JSON');
+  },
+
+  create(data: Data) {
+    return new DependenciesStep(data);
+  },
+});

packages/cli/src/commands/install/steps/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { appRoute } from './appRoute';
+export { dependencies } from './dependencies';
+export { message } from './message';

packages/cli/src/commands/install/steps/message.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Step, createStepDefinition } from '../types';
+
+type Data = {
+  message: string;
+};
+
+class MessageStep implements Step {
+  constructor(private readonly data: Data) {}
+
+  async run() {
+    console.log(this.data.message);
+  }
+}
+
+export const message = createStepDefinition<Data>({
+  type: 'message',
+
+  deserialize(obj) {
+    const { message: msg } = obj;
+
+    if (!msg || (typeof msg !== 'string' && !Array.isArray(msg))) {
+      throw new Error(
+        "Invalid install step, 'message' must be a string or array",
+      );
+    }
+    return new MessageStep({ message: [msg].flat().join('') });
+  },
+
+  create(data: Data) {
+    return new MessageStep(data);
+  },
+});

packages/cli/src/commands/install/types.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { YarnInfoInspectData } from '../../lib/versioning';
+import { JsonObject } from '@backstage/config';
+
+/**
+ * TODO: possible types
+ *
+ * frontend-deps: Install one or many frontend packages in a Backstage app
+ * backend-deps: Install one or many backend packages in a Backstage app
+ * app-config: Update app-config.yaml (and ask for inputs). E.g. Use local or docker for techdocs.builder
+ * frontend-route: Add a frontend route to the plugin homepage
+ * backend-route: Add a backend route to the plugin
+ * entity-page-tab: Add a tab on Catalog’s entity page
+ * sidebar-item: Add a sidebar item
+ * frontend-api: Add a custom API
+ */
+
+/** A serialized install step as it appears in JSON */
+export type SerializedStep = {
+  type: string;
+} & unknown;
+
+export type InstallationRecipe = {
+  type?: 'frontend' | 'backend';
+  steps: SerializedStep[];
+};
+
+/** package.json data */
+export type PackageWithInstallRecipe = YarnInfoInspectData & {
+  version: string;
+  experimentalInstallationRecipe?: InstallationRecipe;
+};
+
+export interface Step {
+  run(): Promise<void>;
+}
+
+export interface StepDefinition<Options> {
+  /** The string identifying this type of step */
+  type: string;
+
+  /** Deserializes and validate a JSON description of the step data */
+  deserialize(obj: JsonObject, pkg: PackageWithInstallRecipe): Step;
+
+  /** Creates a step using known parameters */
+  create(options: Options): Step;
+}
+
+/** Creates a new step definition. Only used as a helper for type inference */
+export function createStepDefinition<T>(
+  config: StepDefinition<T>,
+): StepDefinition<T> {
+  return config;
+}

packages/cli/src/lib/versioning/index.ts

@@ -16,3 +16,4 @@
 
 export { Lockfile } from './Lockfile';
 export { fetchPackageInfo, mapDependencies } from './packages';
+export type { YarnInfoInspectData } from './packages';

packages/cli/src/lib/versioning/packages.ts

@@ -27,7 +27,7 @@ const DEP_TYPES = [
 ];
 
 // Package data as returned by `yarn info`
-type YarnInfoInspectData = {
+export type YarnInfoInspectData = {
   name: string;
   'dist-tags': { latest: string };
   versions: string[];

plugins/auth-backend/config.d.ts

@@ -50,6 +50,8 @@ export interface Config {
         issuer: string;
         cert: string;
         privateKey?: string;
+        authnContext?: string[];
+        identifierFormat?: string;
         decryptionPvk?: string;
         signatureAlgorithm?: 'sha256' | 'sha512';
         digestAlgorithm?: string;

plugins/auth-backend/src/providers/saml/provider.ts

@@ -127,6 +127,8 @@ export const createSamlProvider = (
       issuer: config.getString('issuer'),
       cert: config.getString('cert'),
       privateCert: config.getOptionalString('privateKey'),
+      authnContext: config.getOptionalStringArray('authnContext'),
+      identifierFormat: config.getOptionalString('identifierFormat'),
       decryptionPvk: config.getOptionalString('decryptionPvk'),
       signatureAlgorithm: config.getOptionalString('signatureAlgorithm') as
         | SignatureAlgorithm

plugins/catalog-backend/api-report.md

@@ -435,7 +435,7 @@ export class CodeOwnersProcessor implements CatalogProcessor {
 
 // Warning: (ae-missing-release-tag) "CommonDatabase" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public
+// @public @deprecated
 export class CommonDatabase implements Database {
   constructor(database: Knex, logger: Logger_2);
   // (undocumented)
@@ -499,7 +499,7 @@ export class CommonDatabase implements Database {
 
 // Warning: (ae-missing-release-tag) "CreateDatabaseOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type CreateDatabaseOptions = {
   logger: Logger_2;
 };
@@ -528,7 +528,7 @@ export function createRouter(options: RouterOptions): Promise<express.Router>;
 
 // Warning: (ae-missing-release-tag) "Database" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public
+// @public @deprecated
 export type Database = {
   transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T>;
   addEntities(
@@ -596,7 +596,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
 
 // Warning: (ae-missing-release-tag) "DatabaseLocationsCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export class DatabaseLocationsCatalog implements LocationsCatalog {
   constructor(database: Database);
   // (undocumented)
@@ -624,7 +624,7 @@ export class DatabaseLocationsCatalog implements LocationsCatalog {
 
 // Warning: (ae-missing-release-tag) "DatabaseLocationUpdateLogEvent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type DatabaseLocationUpdateLogEvent = {
   id: string;
   status: DatabaseLocationUpdateLogStatus;
@@ -646,7 +646,7 @@ export enum DatabaseLocationUpdateLogStatus {
 
 // Warning: (ae-missing-release-tag) "DatabaseManager" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export class DatabaseManager {
   // (undocumented)
   static createDatabase(
@@ -665,7 +665,7 @@ export class DatabaseManager {
 
 // Warning: (ae-missing-release-tag) "DbEntitiesRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type DbEntitiesRequest = {
   filter?: EntityFilter;
   pagination?: EntityPagination;
@@ -673,7 +673,7 @@ export type DbEntitiesRequest = {
 
 // Warning: (ae-missing-release-tag) "DbEntitiesResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type DbEntitiesResponse = {
   entities: DbEntityResponse[];
   pageInfo: DbPageInfo;
@@ -681,7 +681,7 @@ export type DbEntitiesResponse = {
 
 // Warning: (ae-missing-release-tag) "DbEntityRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type DbEntityRequest = {
   locationId?: string;
   entity: Entity;
@@ -690,7 +690,7 @@ export type DbEntityRequest = {
 
 // Warning: (ae-missing-release-tag) "DbEntityResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type DbEntityResponse = {
   locationId?: string;
   entity: Entity;
@@ -698,7 +698,7 @@ export type DbEntityResponse = {
 
 // Warning: (ae-missing-release-tag) "DbLocationsRow" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type DbLocationsRow = {
   id: string;
   type: string;
@@ -707,7 +707,7 @@ export type DbLocationsRow = {
 
 // Warning: (ae-missing-release-tag) "DbLocationsRowWithStatus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type DbLocationsRowWithStatus = DbLocationsRow & {
   status: string | null;
   timestamp: string | null;
@@ -716,7 +716,7 @@ export type DbLocationsRowWithStatus = DbLocationsRow & {
 
 // Warning: (ae-missing-release-tag) "DbPageInfo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type DbPageInfo =
   | {
       hasNextPage: false;
@@ -812,7 +812,7 @@ export function durationText(startTimestamp: [number, number]): string;
 export type EntitiesCatalog = {
   entities(request?: EntitiesRequest): Promise<EntitiesResponse>;
   removeEntityByUid(uid: string): Promise<void>;
-  batchAddOrUpdateEntities(
+  batchAddOrUpdateEntities?(
     requests: EntityUpsertRequest[],
     options?: {
       locationId?: string;
@@ -943,7 +943,7 @@ export type EntityProviderMutation =
 
 // Warning: (ae-missing-release-tag) "EntityUpsertRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type EntityUpsertRequest = {
   entity: Entity;
   relations: EntityRelationSpec[];
@@ -951,7 +951,7 @@ export type EntityUpsertRequest = {
 
 // Warning: (ae-missing-release-tag) "EntityUpsertResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type EntityUpsertResponse = {
   entityId: string;
   entity?: Entity;
@@ -1157,7 +1157,7 @@ export class LocationReaders implements LocationReader {
 
 // Warning: (ae-missing-release-tag) "LocationResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type LocationResponse = {
   data: Location_2;
   currentStatus: LocationUpdateStatus;
@@ -1165,7 +1165,7 @@ export type LocationResponse = {
 
 // Warning: (ae-missing-release-tag) "LocationsCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type LocationsCatalog = {
   addLocation(location: Location_2): Promise<Location_2>;
   removeLocation(id: string): Promise<void>;
@@ -1220,7 +1220,7 @@ export interface LocationStore {
 
 // Warning: (ae-missing-release-tag) "LocationUpdateLogEvent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type LocationUpdateLogEvent = {
   id: string;
   status: 'fail' | 'success';
@@ -1232,7 +1232,7 @@ export type LocationUpdateLogEvent = {
 
 // Warning: (ae-missing-release-tag) "LocationUpdateStatus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public (undocumented)
+// @public @deprecated (undocumented)
 export type LocationUpdateStatus = {
   timestamp: string | null;
   status: string | null;
@@ -1487,7 +1487,7 @@ export class StaticLocationProcessor implements StaticLocationProcessor {
 
 // Warning: (ae-missing-release-tag) "Transaction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
-// @public
+// @public @deprecated
 export type Transaction = {
   rollback(): Promise<unknown>;
 };
@@ -1512,20 +1512,20 @@ export class UrlReaderProcessor implements CatalogProcessor {
 
 // Warnings were encountered during analysis:
 //
-// src/catalog/types.d.ts:52:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
-// src/catalog/types.d.ts:53:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
-// src/catalog/types.d.ts:54:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
-// src/database/types.d.ts:125:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:131:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:132:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:146:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:147:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:148:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:150:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:163:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:164:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/database/types.d.ts:165:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/catalog/types.d.ts:97:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
+// src/catalog/types.d.ts:98:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
+// src/catalog/types.d.ts:99:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
 // src/ingestion/processors/GithubMultiOrgReaderProcessor.d.ts:23:9 - (ae-forgotten-export) The symbol "GithubMultiOrgConfig" needs to be exported by the entry point index.d.ts
 // src/ingestion/types.d.ts:8:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/legacy/database/types.d.ts:98:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/legacy/database/types.d.ts:104:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/legacy/database/types.d.ts:105:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/legacy/database/types.d.ts:119:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/legacy/database/types.d.ts:120:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/legacy/database/types.d.ts:121:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/legacy/database/types.d.ts:123:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/legacy/database/types.d.ts:136:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/legacy/database/types.d.ts:137:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// src/legacy/database/types.d.ts:138:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 // src/legacy/ingestion/types.d.ts:19:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 ```

plugins/catalog-backend/src/catalog/index.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
 export type {
   EntitiesCatalog,
   EntitiesRequest,
@@ -22,9 +21,8 @@ export type {
   EntityAncestryResponse,
   EntityUpsertRequest,
   EntityUpsertResponse,
-  LocationResponse,
-  LocationsCatalog,
-  LocationUpdateLogEvent,
-  LocationUpdateStatus,
   PageInfo,
+  EntitiesSearchFilter,
+  EntityFilter,
+  EntityPagination,
 } from './types';

plugins/catalog-backend/src/catalog/types.ts

@@ -14,12 +14,52 @@
  * limitations under the License.
  */
 
-import { Entity, EntityRelationSpec, Location } from '@backstage/catalog-model';
-import { EntityFilter, EntityPagination } from '../database/types';
+import { Entity, EntityRelationSpec } from '@backstage/catalog-model';
 
-//
-// Entities
-//
+/**
+ * A filter expression for entities.
+ *
+ * Any (at least one) of the outer sets must match, within which all of the
+ * individual filters must match.
+ */
+export type EntityFilter = {
+  anyOf: { allOf: EntitiesSearchFilter[] }[];
+};
+
+/**
+ * A pagination rule for entities.
+ */
+export type EntityPagination = {
+  limit?: number;
+  offset?: number;
+  after?: string;
+};
+
+/**
+ * Matches rows in the entities_search table.
+ */
+export type EntitiesSearchFilter = {
+  /**
+   * The key to match on.
+   *
+   * Matches are always case insensitive.
+   */
+  key: string;
+
+  /**
+   * Match on plain equality of values.
+   *
+   * If undefined, this factor is not taken into account. Otherwise, match on
+   * values that are equal to any of the given array items. Matches are always
+   * case insensitive.
+   */
+  matchValueIn?: string[];
+
+  /**
+   * Match on existence of key.
+   */
+  matchValueExists?: boolean;
+};
 
 export type PageInfo =
   | {
@@ -41,11 +81,13 @@ export type EntitiesResponse = {
   pageInfo: PageInfo;
 };
 
+/** @deprecated This was part of the legacy catalog engine */
 export type EntityUpsertRequest = {
   entity: Entity;
   relations: EntityRelationSpec[];
 };
 
+/** @deprecated This was part of the legacy catalog engine */
 export type EntityUpsertResponse = {
   entityId: string;
   entity?: Entity;
@@ -79,12 +121,14 @@ export type EntitiesCatalog = {
   /**
    * Writes a number of entities efficiently to storage.
    *
+   * @deprecated This method was part of the legacy catalog engine an will be removed.
+   *
    * @param requests - The entities and their relations
    * @param options.locationId - The location that they all belong to (default none)
    * @param options.dryRun - Whether to throw away the results (default false)
    * @param options.outputEntities - Whether to return the resulting entities (default false)
    */
-  batchAddOrUpdateEntities(
+  batchAddOrUpdateEntities?(
     requests: EntityUpsertRequest[],
     options?: {
       locationId?: string;
@@ -100,44 +144,3 @@ export type EntitiesCatalog = {
    */
   entityAncestry(entityRef: string): Promise<EntityAncestryResponse>;
 };
-
-//
-// Locations
-//
-
-export type LocationUpdateStatus = {
-  timestamp: string | null;
-  status: string | null;
-  message: string | null;
-};
-
-export type LocationUpdateLogEvent = {
-  id: string;
-  status: 'fail' | 'success';
-  location_id: string;
-  entity_name: string;
-  created_at?: string;
-  message?: string;
-};
-
-export type LocationResponse = {
-  data: Location;
-  currentStatus: LocationUpdateStatus;
-};
-
-export type LocationsCatalog = {
-  addLocation(location: Location): Promise<Location>;
-  removeLocation(id: string): Promise<void>;
-  locations(): Promise<LocationResponse[]>;
-  location(id: string): Promise<LocationResponse>;
-  locationHistory(id: string): Promise<LocationUpdateLogEvent[]>;
-  logUpdateSuccess(
-    locationId: string,
-    entityName?: string | string[],
-  ): Promise<void>;
-  logUpdateFailure(
-    locationId: string,
-    error?: Error,
-    entityName?: string,
-  ): Promise<void>;
-};

plugins/catalog-backend/src/index.ts

@@ -21,7 +21,6 @@
  */
 
 export * from './catalog';
-export * from './database';
 export * from './ingestion';
 export * from './legacy';
 export * from './search';

plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.test.ts

@@ -16,7 +16,7 @@
 
 import { getVoidLogger } from '@backstage/backend-common';
 import { Entity, LOCATION_ANNOTATION } from '@backstage/catalog-model';
-import { Database, DatabaseManager, Transaction } from '../../database';
+import { Database, DatabaseManager, Transaction } from '../database';
 import { basicEntityFilter } from '../../service/request';
 import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
 import { EntityUpsertRequest } from '../../catalog/types';

plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.ts

@@ -26,8 +26,8 @@ import { ConflictError } from '@backstage/errors';
 import { chunk, groupBy } from 'lodash';
 import limiterFactory from 'p-limit';
 import { Logger } from 'winston';
-import type { Database, DbEntityResponse, Transaction } from '../../database';
-import { DbEntitiesRequest } from '../../database/types';
+import type { Database, DbEntityResponse, Transaction } from '../database';
+import { DbEntitiesRequest } from '../database/types';
 import { basicEntityFilter } from '../../service/request';
 import { durationText } from '../../util/timing';
 import type {

plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts

@@ -22,6 +22,7 @@ import {
 } from '../database/types';
 import { LocationResponse, LocationsCatalog } from './types';
 
+/** @deprecated This was part of the legacy catalog engine */
 export class DatabaseLocationsCatalog implements LocationsCatalog {
   constructor(private readonly database: Database) {}
 

plugins/catalog-backend/src/legacy/catalog/index.ts

@@ -15,3 +15,10 @@
  */
 
 export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
+export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
+export type {
+  LocationResponse,
+  LocationsCatalog,
+  LocationUpdateLogEvent,
+  LocationUpdateStatus,
+} from './types';

plugins/catalog-backend/src/legacy/catalog/types.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Location } from '@backstage/catalog-model';
+
+//
+// Locations
+//
+
+/** @deprecated This was part of the legacy catalog engine */
+export type LocationUpdateStatus = {
+  timestamp: string | null;
+  status: string | null;
+  message: string | null;
+};
+
+/** @deprecated This was part of the legacy catalog engine */
+export type LocationUpdateLogEvent = {
+  id: string;
+  status: 'fail' | 'success';
+  location_id: string;
+  entity_name: string;
+  created_at?: string;
+  message?: string;
+};
+
+/** @deprecated This was part of the legacy catalog engine */
+export type LocationResponse = {
+  data: Location;
+  currentStatus: LocationUpdateStatus;
+};
+
+/** @deprecated This was part of the legacy catalog engine */
+export type LocationsCatalog = {
+  addLocation(location: Location): Promise<Location>;
+  removeLocation(id: string): Promise<void>;
+  locations(): Promise<LocationResponse[]>;
+  location(id: string): Promise<LocationResponse>;
+  locationHistory(id: string): Promise<LocationUpdateLogEvent[]>;
+  logUpdateSuccess(
+    locationId: string,
+    entityName?: string | string[],
+  ): Promise<void>;
+  logUpdateFailure(
+    locationId: string,
+    error?: Error,
+    entityName?: string,
+  ): Promise<void>;
+};

plugins/catalog-backend/src/legacy/database/CommonDatabase.test.ts

@@ -16,7 +16,7 @@
 
 import { Entity, Location, parseEntityRef } from '@backstage/catalog-model';
 import { ConflictError } from '@backstage/errors';
-import { basicEntityFilter } from '../service/request';
+import { basicEntityFilter } from '../../service/request';
 import { DatabaseManager } from './DatabaseManager';
 import type {
   DbEntityRequest,

plugins/catalog-backend/src/legacy/database/CommonDatabase.ts

@@ -44,9 +44,9 @@ import {
   DbLocationsRow,
   DbLocationsRowWithStatus,
   DbPageInfo,
-  EntityPagination,
   Transaction,
 } from './types';
+import { EntityPagination } from '../../catalog/types';
 
 // The number of items that are sent per batch to the database layer, when
 // doing .batchInsert calls to knex. This needs to be low enough to not cause
@@ -55,7 +55,8 @@ import {
 const BATCH_SIZE = 50;
 
 /**
- * The core database implementation.
+ * The core database implementation..
+ * @deprecated This was part of the legacy catalog engin
  */
 export class CommonDatabase implements Database {
   constructor(

plugins/catalog-backend/src/legacy/database/DatabaseManager.ts

@@ -26,6 +26,7 @@ const migrationsDir = resolvePackagePath(
   'migrations',
 );
 
+/** @deprecated This was part of the legacy catalog engine */
 export type CreateDatabaseOptions = {
   logger: Logger;
 };
@@ -34,6 +35,7 @@ const defaultOptions: CreateDatabaseOptions = {
   logger: getVoidLogger(),
 };
 
+/** @deprecated This was part of the legacy catalog engine */
 export class DatabaseManager {
   public static async createDatabase(
     knex: Knex,

plugins/catalog-backend/src/legacy/database/index.ts

@@ -21,9 +21,6 @@ export type {
   Database,
   DbEntityRequest,
   DbEntityResponse,
-  EntitiesSearchFilter,
-  EntityFilter,
-  EntityPagination,
   Transaction,
   DbEntitiesRequest,
   DbEntitiesResponse,

plugins/catalog-backend/src/legacy/database/types.ts

@@ -20,7 +20,9 @@ import type {
   EntityRelationSpec,
   Location,
 } from '@backstage/catalog-model';
+import { EntityFilter, EntityPagination } from '../../catalog/types';
 
+/** @deprecated This was part of the legacy catalog engine */
 export type DbEntitiesRow = {
   id: string;
   location_id: string | null;
@@ -30,22 +32,26 @@ export type DbEntitiesRow = {
   data: string;
 };
 
+/** @deprecated This was part of the legacy catalog engine */
 export type DbEntityRequest = {
   locationId?: string;
   entity: Entity;
   relations: EntityRelationSpec[];
 };
 
+/** @deprecated This was part of the legacy catalog engine */
 export type DbEntitiesRequest = {
   filter?: EntityFilter;
   pagination?: EntityPagination;
 };
 
+/** @deprecated This was part of the legacy catalog engine */
 export type DbEntitiesResponse = {
   entities: DbEntityResponse[];
   pageInfo: DbPageInfo;
 };
 
+/** @deprecated This was part of the legacy catalog engine */
 export type DbPageInfo =
   | {
       hasNextPage: false;
@@ -55,11 +61,13 @@ export type DbPageInfo =
       endCursor: string;
     };
 
+/** @deprecated This was part of the legacy catalog engine */
 export type DbEntityResponse = {
   locationId?: string;
   entity: Entity;
 };
 
+/** @deprecated This was part of the legacy catalog engine */
 export type DbEntitiesRelationsRow = {
   originating_entity_id: string;
   source_full_name: string;
@@ -67,18 +75,21 @@ export type DbEntitiesRelationsRow = {
   target_full_name: string;
 };
 
+/** @deprecated This was part of the legacy catalog engine */
 export type DbEntitiesSearchRow = {
   entity_id: string;
   key: string;
   value: string | null;
 };
 
+/** @deprecated This was part of the legacy catalog engine */
 export type DbLocationsRow = {
   id: string;
   type: string;
   target: string;
 };
 
+/** @deprecated This was part of the legacy catalog engine */
 export type DbLocationsRowWithStatus = DbLocationsRow & {
   status: string | null;
   timestamp: string | null;
@@ -90,6 +101,7 @@ export enum DatabaseLocationUpdateLogStatus {
   SUCCESS = 'success',
 }
 
+/** @deprecated This was part of the legacy catalog engine */
 export type DatabaseLocationUpdateLogEvent = {
   id: string;
   status: DatabaseLocationUpdateLogStatus;
@@ -99,53 +111,10 @@ export type DatabaseLocationUpdateLogEvent = {
   message?: string;
 };
 
-/**
- * Matches rows in the entities_search table.
- */
-export type EntitiesSearchFilter = {
-  /**
-   * The key to match on.
-   *
-   * Matches are always case insensitive.
-   */
-  key: string;
-
-  /**
-   * Match on plain equality of values.
-   *
-   * If undefined, this factor is not taken into account. Otherwise, match on
-   * values that are equal to any of the given array items. Matches are always
-   * case insensitive.
-   */
-  matchValueIn?: string[];
-
-  /**
-   * Match on existence of key.
-   */
-  matchValueExists?: boolean;
-};
-
-/**
- * A filter expression for entities.
- *
- * Any (at least one) of the outer sets must match, within which all of the
- * individual filters must match.
- */
-export type EntityFilter = {
-  anyOf: { allOf: EntitiesSearchFilter[] }[];
-};
-
-/**
- * A pagination rule for entities.
- */
-export type EntityPagination = {
-  limit?: number;
-  offset?: number;
-  after?: string;
-};
-
 /**
  * An abstraction for transactions of the underlying database technology.
+ *
+ * @deprecated This was part of the legacy catalog engine
  */
 export type Transaction = {
   rollback(): Promise<unknown>;
@@ -154,6 +123,7 @@ export type Transaction = {
 /**
  * An abstraction on top of the underlying database, wrapping the basic CRUD
  * needs.
+ * @deprecated This was part of the legacy catalog engine
  */
 export type Database = {
   /**

plugins/catalog-backend/src/legacy/index.ts

@@ -17,3 +17,4 @@
 export * from './catalog';
 export * from './ingestion';
 export * from './service';
+export * from './database';

plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts

@@ -16,14 +16,15 @@
 
 import { getVoidLogger } from '@backstage/backend-common';
 import { Entity, Location, LocationSpec } from '@backstage/catalog-model';
-import { EntitiesCatalog, LocationsCatalog } from '../../catalog';
-import { LocationUpdateStatus } from '../../catalog/types';
-import { DatabaseLocationUpdateLogStatus } from '../../database/types';
+import { EntitiesCatalog } from '../../catalog';
+import { LocationsCatalog } from '../catalog';
+import { LocationUpdateStatus } from '../catalog/types';
+import { DatabaseLocationUpdateLogStatus } from '../database/types';
 import { HigherOrderOperations } from './HigherOrderOperations';
 import { LocationReader } from './types';
 
 describe('HigherOrderOperations', () => {
-  let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
+  let entitiesCatalog: jest.Mocked<Required<EntitiesCatalog>>;
   let locationsCatalog: jest.Mocked<LocationsCatalog>;
   let locationReader: jest.Mocked<LocationReader>;
   let higherOrderOperation: HigherOrderOperations;

plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.ts

@@ -21,7 +21,8 @@ import {
 } from '@backstage/catalog-model';
 import { v4 as uuidv4 } from 'uuid';
 import { Logger } from 'winston';
-import { EntitiesCatalog, LocationsCatalog } from '../../catalog';
+import { EntitiesCatalog } from '../../catalog';
+import { LocationsCatalog } from '../catalog';
 import { durationText } from '../../util';
 import {
   AddLocationResult,
@@ -95,14 +96,12 @@ export class HigherOrderOperations implements HigherOrderOperation {
       return { location, entities: [] };
     }
 
-    const writtenEntities = await this.entitiesCatalog.batchAddOrUpdateEntities(
-      readerOutput.entities,
-      {
-        locationId: dryRun ? undefined : location.id,
-        dryRun,
-        outputEntities: true,
-      },
-    );
+    const writtenEntities = await this.entitiesCatalog
+      .batchAddOrUpdateEntities!(readerOutput.entities, {
+      locationId: dryRun ? undefined : location.id,
+      dryRun,
+      outputEntities: true,
+    });
 
     const entities = writtenEntities.map(e => e.entity!);
 
@@ -186,7 +185,7 @@ export class HigherOrderOperations implements HigherOrderOperation {
     startTimestamp = process.hrtime();
 
     try {
-      await this.entitiesCatalog.batchAddOrUpdateEntities(
+      await this.entitiesCatalog.batchAddOrUpdateEntities!(
         readerOutput.entities,
         { locationId: location.id },
       );

plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts

@@ -19,7 +19,7 @@ import { Entity } from '@backstage/catalog-model';
 import { ConfigReader } from '@backstage/config';
 import { Knex } from 'knex';
 import yaml from 'yaml';
-import { DatabaseManager } from '../../database';
+import { DatabaseManager } from '../database';
 import { CatalogProcessorParser } from '../../ingestion';
 import * as result from '../../ingestion/processors/results';
 import { CatalogBuilder } from './CatalogBuilder';

plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts

@@ -26,13 +26,13 @@ import {
 } from '@backstage/catalog-model';
 import { ScmIntegrations } from '@backstage/integration';
 import lodash from 'lodash';
+import { EntitiesCatalog } from '../../catalog';
 import {
+  DatabaseEntitiesCatalog,
   DatabaseLocationsCatalog,
-  EntitiesCatalog,
   LocationsCatalog,
-} from '../../catalog';
-import { DatabaseEntitiesCatalog } from '../catalog';
-import { DatabaseManager } from '../../database';
+} from '../catalog';
+import { DatabaseManager } from '../database';
 import {
   AnnotateLocationEntityProcessor,
   BitbucketDiscoveryProcessor,

plugins/catalog-backend/src/legacy/service/router.test.ts

@@ -20,15 +20,15 @@ import { NotFoundError } from '@backstage/errors';
 import type { Entity, LocationSpec } from '@backstage/catalog-model';
 import express from 'express';
 import request from 'supertest';
-import { EntitiesCatalog, LocationsCatalog } from '../../catalog';
-import { LocationResponse } from '../../catalog/types';
+import { EntitiesCatalog } from '../../catalog';
+import { LocationResponse, LocationsCatalog } from '../catalog/types';
 import { HigherOrderOperation } from '../ingestion/types';
 import { createRouter } from './router';
 import { basicEntityFilter } from '../../service/request';
 import { RefreshService } from '../../next';
 
 describe('createRouter readonly disabled', () => {
-  let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
+  let entitiesCatalog: jest.Mocked<Required<EntitiesCatalog>>;
   let locationsCatalog: jest.Mocked<LocationsCatalog>;
   let higherOrderOperation: jest.Mocked<HigherOrderOperation>;
   let app: express.Express;

plugins/catalog-backend/src/legacy/service/router.ts

@@ -26,7 +26,8 @@ import express from 'express';
 import Router from 'express-promise-router';
 import { Logger } from 'winston';
 import yn from 'yn';
-import { EntitiesCatalog, LocationsCatalog } from '../../catalog';
+import { EntitiesCatalog } from '../../catalog';
+import { LocationsCatalog } from '../catalog';
 import { LocationAnalyzer } from '../../ingestion/types';
 import { HigherOrderOperation } from '../ingestion/types';
 import {
@@ -124,7 +125,7 @@ export async function createRouter(
         disallowReadonlyMode(readonlyEnabled);
 
         const body = await requireRequestBody(req);
-        const [result] = await entitiesCatalog.batchAddOrUpdateEntities([
+        const [result] = await entitiesCatalog.batchAddOrUpdateEntities!([
           { entity: body as Entity, relations: [] },
         ]);
         const response = await entitiesCatalog.entities({

plugins/catalog-backend/src/next/DefaultLocationStore.test.ts

@@ -15,7 +15,7 @@
  */
 import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
 import { v4 as uuid } from 'uuid';
-import { DatabaseManager } from './database/DatabaseManager';
+import { applyDatabaseMigrations } from './database/migrations';
 import { DefaultLocationStore } from './DefaultLocationStore';
 
 describe('DefaultLocationStore', () => {
@@ -25,7 +25,7 @@ describe('DefaultLocationStore', () => {
 
   async function createLocationStore(databaseId: TestDatabaseId) {
     const knex = await databases.init(databaseId);
-    await DatabaseManager.createDatabase(knex);
+    await applyDatabaseMigrations(knex);
     const connection = { applyMutation: jest.fn() };
     const store = new DefaultLocationStore(knex);
     await store.connect(connection);

plugins/catalog-backend/src/next/DefaultRefreshService.test.ts

@@ -19,7 +19,7 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
 import { createHash } from 'crypto';
 import { Knex } from 'knex';
 import { Logger } from 'winston';
-import { DatabaseManager } from './database/DatabaseManager';
+import { applyDatabaseMigrations } from './database/migrations';
 import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase';
 import {
   DbRefreshStateReferencesRow,
@@ -44,7 +44,7 @@ describe('Refresh integration', () => {
     logger: Logger = defaultLogger,
   ) {
     const knex = await databases.init(databaseId);
-    await DatabaseManager.createDatabase(knex);
+    await applyDatabaseMigrations(knex);
     return {
       knex,
       db: new DefaultProcessingDatabase({

plugins/catalog-backend/src/next/NextCatalogBuilder.ts

@@ -14,11 +14,7 @@
  * limitations under the License.
  */
 
-import {
-  PluginDatabaseManager,
-  resolvePackagePath,
-  UrlReader,
-} from '@backstage/backend-common';
+import { PluginDatabaseManager, UrlReader } from '@backstage/backend-common';
 import {
   DefaultNamespaceEntityPolicy,
   EntityPolicies,
@@ -33,12 +29,13 @@ import { ScmIntegrations } from '@backstage/integration';
 import { createHash } from 'crypto';
 import { Router } from 'express';
 import lodash from 'lodash';
+import { EntitiesCatalog } from '../catalog';
 import {
   DatabaseLocationsCatalog,
-  EntitiesCatalog,
   LocationsCatalog,
-} from '../catalog';
-import { CommonDatabase } from '../database/CommonDatabase';
+  CommonDatabase,
+} from '../legacy';
+
 import {
   AnnotateLocationEntityProcessor,
   BitbucketDiscoveryProcessor,
@@ -69,6 +66,7 @@ import {
 } from '../next/types';
 import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider';
 import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase';
+import { applyDatabaseMigrations } from './database/migrations';
 import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine';
 import { DefaultLocationService } from './DefaultLocationService';
 import { DefaultLocationStore } from './DefaultLocationStore';
@@ -289,6 +287,7 @@ export class NextCatalogBuilder {
    */
   async build(): Promise<{
     entitiesCatalog: EntitiesCatalog;
+    /** @deprecated This will be removed */
     locationsCatalog: LocationsCatalog;
     locationAnalyzer: LocationAnalyzer;
     processingEngine: CatalogProcessingEngine;
@@ -302,12 +301,7 @@ export class NextCatalogBuilder {
     const parser = this.parser || defaultEntityDataParser;
 
     const dbClient = await database.getClient();
-    await dbClient.migrate.latest({
-      directory: resolvePackagePath(
-        '@backstage/plugin-catalog-backend',
-        'migrations',
-      ),
-    });
+    await applyDatabaseMigrations(dbClient);
 
     const db = new CommonDatabase(dbClient, logger);
 

plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts

@@ -18,7 +18,7 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
 import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
 import { Knex } from 'knex';
 import { v4 as uuid } from 'uuid';
-import { DatabaseManager } from './database/DatabaseManager';
+import { applyDatabaseMigrations } from './database/migrations';
 import {
   DbFinalEntitiesRow,
   DbRefreshStateReferencesRow,
@@ -33,7 +33,7 @@ describe('NextEntitiesCatalog', () => {
 
   async function createDatabase(databaseId: TestDatabaseId) {
     const knex = await databases.init(databaseId);
-    await DatabaseManager.createDatabase(knex);
+    await applyDatabaseMigrations(knex);
     return { knex };
   }
 

plugins/catalog-backend/src/next/NextEntitiesCatalog.ts

@@ -22,13 +22,14 @@ import {
   EntitiesRequest,
   EntitiesResponse,
   EntityAncestryResponse,
+  EntityPagination,
 } from '../catalog/types';
-import { DbPageInfo, EntityPagination } from '../database/types';
 import {
   DbFinalEntitiesRow,
   DbRefreshStateReferencesRow,
   DbRefreshStateRow,
   DbSearchRow,
+  DbPageInfo,
 } from './database/tables';
 
 function parsePagination(input?: EntityPagination): {

plugins/catalog-backend/src/next/database/DatabaseManager.ts

@@ -1,99 +0,0 @@
-/*
- * Copyright 2020 The Backstage Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common';
-import knexFactory, { Knex } from 'knex';
-import { v4 as uuid } from 'uuid';
-import { Logger } from 'winston';
-import { CommonDatabase } from '../../database/CommonDatabase';
-import { Database } from '../../database/types';
-
-export type CreateDatabaseOptions = {
-  logger: Logger;
-};
-
-const defaultOptions: CreateDatabaseOptions = {
-  logger: getVoidLogger(),
-};
-
-export class DatabaseManager {
-  public static async createDatabase(
-    knex: Knex,
-    options: Partial<CreateDatabaseOptions> = {},
-  ): Promise<Database> {
-    const migrationsDir = resolvePackagePath(
-      '@backstage/plugin-catalog-backend',
-      'migrations',
-    );
-
-    await knex.migrate.latest({
-      directory: migrationsDir,
-    });
-    const { logger } = { ...defaultOptions, ...options };
-    return new CommonDatabase(knex, logger);
-  }
-
-  public static async createInMemoryDatabase(): Promise<Database> {
-    const knex = await this.createInMemoryDatabaseConnection();
-    return await this.createDatabase(knex);
-  }
-
-  public static async createInMemoryDatabaseConnection(): Promise<Knex> {
-    const knex = knexFactory({
-      client: 'sqlite3',
-      connection: ':memory:',
-      useNullAsDefault: true,
-    });
-
-    knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
-      resource.run('PRAGMA foreign_keys = ON', () => {});
-    });
-
-    return knex;
-  }
-
-  public static async createTestDatabase(): Promise<Database> {
-    const knex = await this.createTestDatabaseConnection();
-    return await this.createDatabase(knex);
-  }
-
-  public static async createTestDatabaseConnection(): Promise<Knex> {
-    const config: Knex.Config<any> = {
-      client: 'sqlite3',
-      connection: ':memory:',
-      useNullAsDefault: true,
-    };
-
-    let knex = knexFactory(config);
-    if (typeof config.connection !== 'string') {
-      const tempDbName = `d${uuid().replace(/-/g, '')}`;
-      await knex.raw(`CREATE DATABASE ${tempDbName};`);
-      knex = knexFactory({
-        ...config,
-        connection: {
-          ...config.connection,
-          database: tempDbName,
-        },
-      });
-    }
-
-    knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
-      resource.run('PRAGMA foreign_keys = ON', () => {});
-    });
-
-    return knex;
-  }
-}

plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts

@@ -21,7 +21,7 @@ import { Knex } from 'knex';
 import * as uuid from 'uuid';
 import { Logger } from 'winston';
 import { DateTime } from 'luxon';
-import { DatabaseManager } from './DatabaseManager';
+import { applyDatabaseMigrations } from './migrations';
 import { DefaultProcessingDatabase } from './DefaultProcessingDatabase';
 import {
   DbRefreshStateReferencesRow,
@@ -43,7 +43,7 @@ describe('Default Processing Database', () => {
     logger: Logger = defaultLogger,
   ) {
     const knex = await databases.init(databaseId);
-    await DatabaseManager.createDatabase(knex);
+    await applyDatabaseMigrations(knex);
     return {
       knex,
       db: new DefaultProcessingDatabase({

plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts

@@ -20,17 +20,8 @@ import { Knex } from 'knex';
 import lodash from 'lodash';
 import { v4 as uuid } from 'uuid';
 import type { Logger } from 'winston';
-import { Transaction } from '../../database';
-import { DeferredEntity } from '../processing/types';
-import { RefreshIntervalFunction } from '../refresh';
-import { rethrowError, timestampToDateTime } from './conversion';
-import { initDatabaseMetrics } from './metrics';
-import {
-  DbRefreshStateReferencesRow,
-  DbRefreshStateRow,
-  DbRelationsRow,
-} from './tables';
 import {
+  Transaction,
   GetProcessableEntitiesResult,
   ProcessingDatabase,
   RefreshStateItem,
@@ -41,6 +32,16 @@ import {
   ListAncestorsResult,
   UpdateEntityCacheOptions,
 } from './types';
+import { DeferredEntity } from '../processing/types';
+import { RefreshIntervalFunction } from '../refresh';
+import { rethrowError, timestampToDateTime } from './conversion';
+import { initDatabaseMetrics } from './metrics';
+import {
+  DbRefreshStateReferencesRow,
+  DbRefreshStateRow,
+  DbRelationsRow,
+} from './tables';
+
 import { generateStableHash } from './util';
 
 // The number of items that are sent per batch to the database layer, when

plugins/catalog-backend/src/next/database/metrics.ts

@@ -15,9 +15,8 @@
  */
 
 import { Knex } from 'knex';
-import { DbLocationsRow } from '../../database/types';
 import { createGaugeMetric } from '../metrics';
-import { DbRefreshStateRow, DbRelationsRow } from './tables';
+import { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables';
 
 export function initDatabaseMetrics(knex: Knex) {
   const seen = new Set<string>();

plugins/catalog-backend/src/next/database/migrations.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { resolvePackagePath } from '@backstage/backend-common';
+import { Knex } from 'knex';
+
+export async function applyDatabaseMigrations(knex: Knex): Promise<void> {
+  const migrationsDir = resolvePackagePath(
+    '@backstage/plugin-catalog-backend',
+    'migrations',
+  );
+
+  await knex.migrate.latest({
+    directory: migrationsDir,
+  });
+}

plugins/catalog-backend/src/next/database/tables.ts

@@ -14,6 +14,15 @@
  * limitations under the License.
  */
 
+export type DbPageInfo =
+  | {
+      hasNextPage: false;
+    }
+  | {
+      hasNextPage: true;
+      endCursor: string;
+    };
+
 export type DbLocationsRow = {
   id: string;
   type: string;

plugins/catalog-backend/src/next/database/types.ts

@@ -17,9 +17,15 @@
 import { Entity, EntityRelationSpec } from '@backstage/catalog-model';
 import { JsonObject } from '@backstage/config';
 import { DateTime } from 'luxon';
-import { Transaction } from '../../database/types';
 import { DeferredEntity } from '../processing/types';
 
+/**
+ * An abstraction for transactions of the underlying database technology.
+ */
+export type Transaction = {
+  rollback(): Promise<unknown>;
+};
+
 export type AddUnprocessedEntitiesResult = {};
 
 export type UpdateProcessedEntityOptions = {

plugins/catalog-backend/src/next/stitching/Stitcher.test.ts

@@ -17,7 +17,7 @@
 import { getVoidLogger } from '@backstage/backend-common';
 import { TestDatabases } from '@backstage/backend-test-utils';
 import { Entity } from '@backstage/catalog-model';
-import { DatabaseManager } from '../database/DatabaseManager';
+import { applyDatabaseMigrations } from '../database/migrations';
 import {
   DbFinalEntitiesRow,
   DbRefreshStateReferencesRow,
@@ -37,7 +37,7 @@ describe('Stitcher', () => {
     'runs the happy path for %p',
     async databaseId => {
       const db = await databases.init(databaseId);
-      await DatabaseManager.createDatabase(db);
+      await applyDatabaseMigrations(db);
 
       const stitcher = new Stitcher(db, logger);
       let entities: DbFinalEntitiesRow[];

plugins/catalog-backend/src/service/request/basicEntityFilter.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { EntitiesSearchFilter, EntityFilter } from '../../database';
+import { EntitiesSearchFilter, EntityFilter } from '../../catalog';
 
 /**
  * Forms a full EntityFilter based on a single key-value(s) object.

plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts

@@ -15,7 +15,7 @@
  */
 
 import { InputError } from '@backstage/errors';
-import { EntitiesSearchFilter, EntityFilter } from '../../database';
+import { EntitiesSearchFilter, EntityFilter } from '../../catalog';
 import { parseStringsParam } from './common';
 
 /**

plugins/catalog-backend/src/service/request/parseEntityPaginationParams.ts

@@ -15,7 +15,7 @@
  */
 
 import { InputError } from '@backstage/errors';
-import { EntityPagination } from '../../database';
+import { EntityPagination } from '../../catalog';
 import { parseIntegerParam, parseStringParam } from './common';
 
 /**

plugins/catalog-backend/src/service/standaloneServer.ts

@@ -22,7 +22,7 @@ import {
 } from '@backstage/backend-common';
 import { Server } from 'http';
 import { Logger } from 'winston';
-import { DatabaseManager } from '../database';
+import { DatabaseManager } from '../legacy/database';
 import { CatalogBuilder } from '../legacy/service/CatalogBuilder';
 import { createRouter } from '../legacy/service';
 

plugins/code-coverage-backend/src/service/converter/jacoco.ts

@@ -83,7 +83,7 @@ export class Jacoco implements Converter {
   private extractLines(sourcefile: JacocoSourceFile): ParsedLine[] {
     const parsed: ParsedLine[] = [];
 
-    sourcefile.line.forEach(l => {
+    sourcefile.line?.forEach(l => {
       parsed.push({
         number: parseInt(l.$.nr, 10),
         missed_instructions: parseInt(l.$.mi, 10),

plugins/code-coverage-backend/src/service/converter/types.ts

@@ -79,7 +79,7 @@ export type JacocoSourceFile = {
   $: {
     name: string;
   };
-  line: JacocoLine[];
+  line: JacocoLine[] | undefined;
 };
 export type JacocoLine = {
   $: {

plugins/graphiql/package.json

@@ -60,5 +60,22 @@
   },
   "files": [
     "dist"
-  ]
+  ],
+  "experimentalInstallationRecipe": {
+    "type": "frontend-plugin",
+    "steps": [
+      {
+        "type": "app-route",
+        "path": "/graphiql",
+        "element": "<GraphiQLPage />"
+      },
+      {
+        "type": "message",
+        "message": [
+          "The GraphiQL plugin has been installed, but you still need to add API endpoints. ",
+          "See https://github.com/backstage/backstage/tree/master/plugins/graphiql#adding-graphql-endpoints"
+        ]
+      }
+    ]
+  }
 }

yarn.lock

@@ -25765,10 +25765,10 @@ subscriptions-transport-ws@^0.9.18, subscriptions-transport-ws@^0.9.19:
     symbol-observable "^1.0.4"
     ws "^5.2.0 || ^6.0.0 || ^7.0.0"
 
-sucrase@^3.18.0, sucrase@^3.20.1:
-  version "3.20.1"
-  resolved "https://registry.npmjs.org/sucrase/-/sucrase-3.20.1.tgz#1c055e97d0fab2f9857f02461364075b3a4ab226"
-  integrity sha512-BIG59HaJOxNct9Va6KvT5yzBA/rcMGetzvZyTx0ZdCcspIbpJTPS64zuAfYlJuOj+3WaI5JOdA+F0bJQQi8ZiQ==
+sucrase@^3.18.0, sucrase@^3.20.2:
+  version "3.20.2"
+  resolved "https://registry.npmjs.org/sucrase/-/sucrase-3.20.2.tgz#28a28dc58a55be0d6916d5c9b2440d203e9ffe62"
+  integrity sha512-EdJ5M6VEvToIZwIWiZ71cxe4CklDRG8PdSjUSst+BZCUGlaEhnrdQo/LOXsuq3MjWRbfepg1XTffClK0Tmo0HQ==
   dependencies:
     commander "^4.0.0"
     glob "7.1.6"