Create azure and gitlab catalog modules, move related processors into them

Signed-off-by: Fredrik Adelöw <freben@gmail.com>

.changeset/clever-garlics-rescue.md

@@ -0,0 +1,20 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+**BREAKING**: Removed `GitLabDiscoveryProcessor`, which now instead should be imported from `@backstage/plugin-catalog-backend-module-gitlab`. NOTE THAT this processor was part of the default set of processors in the catalog backend, and if you are a user of discovery on GitLab, you MUST now add it manually in the catalog initialization code of your backend.
+
+```diff
+// In packages/backend/src/plugins/catalog.ts
++import { GitLabDiscoveryProcessor } from '@backstage/plugin-scaffolder-backend-module-gitlab';
+
+ export default async function createPlugin(
+   env: PluginEnvironment,
+ ): Promise<Router> {
+   const builder = await CatalogBuilder.create(env);
++  builder.addProcessor(
++    GitLabDiscoveryProcessor.fromConfig(env.config, { logger: env.logger })
++  );
+```
+
+**BREAKING**: Removed `AzureDevOpsDiscoveryProcessor`, which now instead should be imported from `@backstage/plugin-catalog-backend-module-azure`. This processor was not part of the set of default processors. If you were using it, you should already have a reference to it in your backend code and only need to update the import.

.changeset/rich-ravens-clap.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-azure': minor
+---
+
+Added package, moving out azure specific functionality from the catalog-backend

.changeset/silent-cats-kneel.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab': minor
+---
+
+Added package, moving out gitlab specific functionality from the catalog-backend

docs/integrations/gitlab/discovery.md

@@ -34,3 +34,19 @@ The target is composed of three parts:
   a similar variation for catalog files stored in the root directory of each
   repository. If you want to use the repository's default branch use the `*`
   wildcard, e.g.: `/blob/*/catalog-info.yaml`
+
+Finally, you will have to add the processor in the catalog initialization code
+of your backend.
+
+```diff
+// In packages/backend/src/plugins/catalog.ts
++import { GitLabDiscoveryProcessor } from '@backstage/plugin-scaffolder-backend-module-gitlab';
+
+ export default async function createPlugin(
+   env: PluginEnvironment,
+ ): Promise<Router> {
+   const builder = await CatalogBuilder.create(env);
++  builder.addProcessor(
++    GitLabDiscoveryProcessor.fromConfig(env.config, { logger: env.logger })
++  );
+```

plugins/catalog-backend-module-azure/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint.backend')],
+};

plugins/catalog-backend-module-azure/README.md

@@ -0,0 +1,8 @@
+# Catalog Backend Module for Azure
+
+This is an extension module to the plugin-catalog-backend plugin, providing extensions targeted at Azure cloud offerings.
+
+## Getting started
+
+See [Backstage documentation](https://backstage.io/docs/integrations/azure/org) for details on how to install
+and configure the plugin.

plugins/catalog-backend-module-azure/api-report.md

@@ -0,0 +1,35 @@
+## API Report File for "@backstage/plugin-catalog-backend-module-azure"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
+import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
+import { Config } from '@backstage/config';
+import { LocationSpec } from '@backstage/plugin-catalog-backend';
+import { Logger as Logger_2 } from 'winston';
+import { ScmIntegrationRegistry } from '@backstage/integration';
+
+// @public
+export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
+  constructor(options: {
+    integrations: ScmIntegrationRegistry;
+    logger: Logger_2;
+  });
+  // (undocumented)
+  static fromConfig(
+    config: Config,
+    options: {
+      logger: Logger_2;
+    },
+  ): AzureDevOpsDiscoveryProcessor;
+  // (undocumented)
+  getProcessorName(): string;
+  // (undocumented)
+  readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean>;
+}
+```

plugins/catalog-backend-module-azure/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@backstage/plugin-catalog-backend-module-azure",
+  "description": "A Backstage catalog backend module that helps integrate towards Azure",
+  "version": "0.0.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/catalog-backend-module-azure"
+  },
+  "keywords": [
+    "backstage"
+  ],
+  "scripts": {
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "clean": "backstage-cli package clean",
+    "start": "backstage-cli package start"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "^0.12.0",
+    "@backstage/catalog-model": "^0.12.0",
+    "@backstage/config": "^0.1.15",
+    "@backstage/errors": "^0.2.2",
+    "@backstage/integration": "^0.8.0",
+    "@backstage/plugin-catalog-backend": "^0.23.0",
+    "@backstage/types": "^0.1.3",
+    "lodash": "^4.17.21",
+    "msw": "^0.35.0",
+    "node-fetch": "^2.6.7",
+    "winston": "^3.2.1"
+  },
+  "devDependencies": {
+    "@backstage/backend-test-utils": "^0.1.20",
+    "@backstage/cli": "^0.15.0",
+    "@types/lodash": "^4.14.151"
+  },
+  "files": [
+    "dist"
+  ]
+}

plugins/catalog-backend-module-azure/src/AzureDevOpsDiscoveryProcessor.test.ts

@@ -16,12 +16,12 @@
 
 import { getVoidLogger } from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
-import { codeSearch } from './lib';
+import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import {
   AzureDevOpsDiscoveryProcessor,
   parseUrl,
 } from './AzureDevOpsDiscoveryProcessor';
-import { LocationSpec } from '../../api';
+import { codeSearch } from './lib';
 
 jest.mock('./lib');
 const mockCodeSearch = codeSearch as jest.MockedFunction<typeof codeSearch>;

plugins/catalog-backend-module-azure/src/AzureDevOpsDiscoveryProcessor.ts

@@ -19,13 +19,13 @@ import {
   ScmIntegrationRegistry,
   ScmIntegrations,
 } from '@backstage/integration';
-import { Logger } from 'winston';
 import {
   CatalogProcessor,
   CatalogProcessorEmit,
   LocationSpec,
   processingResult,
-} from '../../api';
+} from '@backstage/plugin-catalog-backend';
+import { Logger } from 'winston';
 import { codeSearch } from './lib';
 
 /**

plugins/catalog-backend-module-azure/src/index.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A Backstage catalog backend module that helps integrate towards Azure
+ *
+ * @packageDocumentation
+ */
+
+export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor';

plugins/catalog-backend-module-azure/src/lib/azure.test.ts

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-import { setupServer } from 'msw/node';
 import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
 import { rest } from 'msw';
+import { setupServer } from 'msw/node';
 import { codeSearch, CodeSearchResponse } from './azure';
 
 describe('azure', () => {

plugins/catalog-backend-module-azure/src/setupTests.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 The Backstage Authors
+ * Copyright 2020 The Backstage Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,4 +14,4 @@
  * limitations under the License.
  */
 
-export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor';
+export {};

plugins/catalog-backend-module-gitlab/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint.backend')],
+};

plugins/catalog-backend-module-gitlab/README.md

@@ -0,0 +1,8 @@
+# Catalog Backend Module for GitLab
+
+This is an extension module to the plugin-catalog-backend plugin, providing extensions targeted at GitLab offerings.
+
+## Getting started
+
+See [Backstage documentation](https://backstage.io/docs/integrations/gitlab/discovery) for details on how to install
+and configure the plugin.

plugins/catalog-backend-module-gitlab/api-report.md

@@ -0,0 +1,30 @@
+## API Report File for "@backstage/plugin-catalog-backend-module-gitlab"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
+import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
+import { Config } from '@backstage/config';
+import { LocationSpec } from '@backstage/plugin-catalog-backend';
+import { Logger as Logger_2 } from 'winston';
+
+// @public
+export class GitLabDiscoveryProcessor implements CatalogProcessor {
+  // (undocumented)
+  static fromConfig(
+    config: Config,
+    options: {
+      logger: Logger_2;
+    },
+  ): GitLabDiscoveryProcessor;
+  // (undocumented)
+  getProcessorName(): string;
+  // (undocumented)
+  readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean>;
+}
+```

plugins/catalog-backend-module-gitlab/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@backstage/plugin-catalog-backend-module-gitlab",
+  "description": "A Backstage catalog backend module that helps integrate towards GitLab",
+  "version": "0.0.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/catalog-backend-module-gitlab"
+  },
+  "keywords": [
+    "backstage"
+  ],
+  "scripts": {
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "clean": "backstage-cli package clean",
+    "start": "backstage-cli package start"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "^0.12.0",
+    "@backstage/catalog-model": "^0.12.0",
+    "@backstage/config": "^0.1.15",
+    "@backstage/errors": "^0.2.2",
+    "@backstage/integration": "^0.8.0",
+    "@backstage/plugin-catalog-backend": "^0.23.0",
+    "@backstage/types": "^0.1.3",
+    "lodash": "^4.17.21",
+    "msw": "^0.35.0",
+    "node-fetch": "^2.6.7",
+    "winston": "^3.2.1"
+  },
+  "devDependencies": {
+    "@backstage/backend-test-utils": "^0.1.20",
+    "@backstage/cli": "^0.15.0",
+    "@types/lodash": "^4.14.151"
+  },
+  "files": [
+    "dist"
+  ]
+}

plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.test.ts

@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-import { ConfigReader } from '@backstage/config';
 import { getVoidLogger } from '@backstage/backend-common';
-import { GitLabDiscoveryProcessor, parseUrl } from './GitLabDiscoveryProcessor';
-import { setupServer } from 'msw/node';
+import { ConfigReader } from '@backstage/config';
+import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { GitLabDiscoveryProcessor, parseUrl } from './GitLabDiscoveryProcessor';
 import { GitLabProject } from './lib';
-import { LocationSpec } from '../../api';
 
 const server = setupServer();
 

plugins/catalog-backend-module-gitlab/src/GitLabDiscoveryProcessor.ts

@@ -14,24 +14,24 @@
  * limitations under the License.
  */
 
-import { Config } from '@backstage/config';
-import {
-  ScmIntegrationRegistry,
-  ScmIntegrations,
-} from '@backstage/integration';
-import { Logger } from 'winston';
-import {
-  CatalogProcessor,
-  CatalogProcessorEmit,
-  LocationSpec,
-  processingResult,
-} from '../../api';
-import { GitLabClient, GitLabProject, paginated } from './lib';
 import {
   CacheClient,
   CacheManager,
   PluginCacheManager,
 } from '@backstage/backend-common';
+import { Config } from '@backstage/config';
+import {
+  ScmIntegrationRegistry,
+  ScmIntegrations,
+} from '@backstage/integration';
+import {
+  CatalogProcessor,
+  CatalogProcessorEmit,
+  LocationSpec,
+  processingResult,
+} from '@backstage/plugin-catalog-backend';
+import { Logger } from 'winston';
+import { GitLabClient, GitLabProject, paginated } from './lib';
 
 /**
  * Extracts repositories out of an GitLab instance.

plugins/catalog-backend-module-gitlab/src/index.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A Backstage catalog backend module that helps integrate towards GitLab
+ *
+ * @packageDocumentation
+ */
+
+export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor';

plugins/catalog-backend-module-gitlab/src/setupTests.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 The Backstage Authors
+ * Copyright 2020 The Backstage Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,4 +14,4 @@
  * limitations under the License.
  */
 
-export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor';
+export {};

plugins/catalog-backend/api-report.md

@@ -97,29 +97,6 @@ export class AnnotateScmSlugEntityProcessor implements CatalogProcessor {
   preProcessEntity(entity: Entity, location: LocationSpec): Promise<Entity>;
 }
 
-// @public
-export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
-  constructor(options: {
-    integrations: ScmIntegrationRegistry;
-    logger: Logger_2;
-  });
-  // (undocumented)
-  static fromConfig(
-    config: Config,
-    options: {
-      logger: Logger_2;
-    },
-  ): AzureDevOpsDiscoveryProcessor;
-  // (undocumented)
-  getProcessorName(): string;
-  // (undocumented)
-  readLocation(
-    location: LocationSpec,
-    _optional: boolean,
-    emit: CatalogProcessorEmit,
-  ): Promise<boolean>;
-}
-
 // @public (undocumented)
 export class BitbucketDiscoveryProcessor implements CatalogProcessor {
   constructor(options: {
@@ -769,25 +746,6 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
   ): Promise<boolean>;
 }
 
-// @public
-export class GitLabDiscoveryProcessor implements CatalogProcessor {
-  // (undocumented)
-  static fromConfig(
-    config: Config,
-    options: {
-      logger: Logger_2;
-    },
-  ): GitLabDiscoveryProcessor;
-  // (undocumented)
-  getProcessorName(): string;
-  // (undocumented)
-  readLocation(
-    location: LocationSpec,
-    _optional: boolean,
-    emit: CatalogProcessorEmit,
-  ): Promise<boolean>;
-}
-
 // @public @deprecated (undocumented)
 function inputError(
   atLocation: LocationSpec,

plugins/catalog-backend/src/modules/index.ts

@@ -14,9 +14,7 @@
  * limitations under the License.
  */
 
-export * from './azure';
 export * from './bitbucket';
 export * from './codeowners';
 export * from './core';
 export * from './github';
-export * from './gitlab';

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -49,10 +49,8 @@ import {
   BuiltinKindsEntityProcessor,
   CodeOwnersProcessor,
   FileReaderProcessor,
-  AzureDevOpsDiscoveryProcessor,
   GithubDiscoveryProcessor,
   GithubOrgReaderProcessor,
-  GitLabDiscoveryProcessor,
   PlaceholderProcessor,
   PlaceholderResolver,
   UrlReaderProcessor,
@@ -360,7 +358,6 @@ export class CatalogBuilder {
     return [
       new FileReaderProcessor(),
       BitbucketDiscoveryProcessor.fromConfig(config, { logger }),
-      AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }),
       GithubDiscoveryProcessor.fromConfig(config, {
         logger,
         githubCredentialsProvider,
@@ -369,7 +366,6 @@ export class CatalogBuilder {
         logger,
         githubCredentialsProvider,
       }),
-      GitLabDiscoveryProcessor.fromConfig(config, { logger }),
       new UrlReaderProcessor({ reader, logger }),
       CodeOwnersProcessor.fromConfig(config, { logger, reader }),
       new AnnotateLocationEntityProcessor({ integrations }),

scripts/api-extractor.ts

@@ -223,6 +223,8 @@ const NO_WARNING_PACKAGES = [
   'plugins/auth-node',
   'plugins/catalog-backend',
   'plugins/catalog-backend-module-aws',
+  'plugins/catalog-backend-module-azure',
+  'plugins/catalog-backend-module-gitlab',
   'plugins/catalog-backend-module-ldap',
   'plugins/catalog-backend-module-msgraph',
   'plugins/catalog-common',