From d4934e19b1dce71c94c48fe2d19d3a9d3e443ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 9 Mar 2022 20:53:13 +0100 Subject: [PATCH] move gitlab to a separate package too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/clever-garlics-rescue.md | 32 +++++ .changeset/pink-chicken-smile.md | 5 + .github/styles/vocab.txt | 2 +- docs/integrations/github/discovery.md | 47 +++++++ docs/integrations/github/org.md | 3 +- .../.eslintrc.js | 1 + .../catalog-backend-module-github/README.md | 8 ++ .../api-report.md | 126 ++++++++++++++++++ .../package.json | 57 ++++++++ .../src}/GitHubOrgEntityProvider.test.ts | 2 +- .../src}/GitHubOrgEntityProvider.ts | 8 +- .../src}/GithubDiscoveryProcessor.test.ts | 2 +- .../src}/GithubDiscoveryProcessor.ts | 8 +- .../src}/GithubMultiOrgReaderProcessor.ts | 20 +-- .../src}/GithubOrgReaderProcessor.test.ts | 4 +- .../src}/GithubOrgReaderProcessor.ts | 23 ++-- .../src}/index.ts | 8 +- .../src}/lib/config.test.ts | 0 .../src}/lib/config.ts | 0 .../src}/lib/github.test.ts | 0 .../src}/lib/github.ts | 0 .../src}/lib/index.ts | 1 + .../src/lib}/org.test.ts | 0 .../src/lib}/org.ts | 0 .../src}/lib/util.test.ts | 0 .../src}/lib/util.ts | 0 .../src/setupTests.ts | 17 +++ plugins/catalog-backend/api-report.md | 112 ---------------- plugins/catalog-backend/package.json | 1 - plugins/catalog-backend/src/modules/index.ts | 1 - .../src/service/CatalogBuilder.ts | 18 +-- 31 files changed, 340 insertions(+), 166 deletions(-) create mode 100644 .changeset/pink-chicken-smile.md create mode 100644 plugins/catalog-backend-module-github/.eslintrc.js create mode 100644 plugins/catalog-backend-module-github/README.md create mode 100644 plugins/catalog-backend-module-github/api-report.md create mode 100644 plugins/catalog-backend-module-github/package.json rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/GitHubOrgEntityProvider.test.ts (98%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/GitHubOrgEntityProvider.ts (97%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/GithubDiscoveryProcessor.test.ts (99%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/GithubDiscoveryProcessor.ts (99%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/GithubMultiOrgReaderProcessor.ts (98%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/GithubOrgReaderProcessor.test.ts (98%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/GithubOrgReaderProcessor.ts (97%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/index.ts (85%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/lib/config.test.ts (100%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/lib/config.ts (100%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/lib/github.test.ts (100%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/lib/github.ts (100%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/lib/index.ts (93%) rename plugins/{catalog-backend/src/modules/util => catalog-backend-module-github/src/lib}/org.test.ts (100%) rename plugins/{catalog-backend/src/modules/util => catalog-backend-module-github/src/lib}/org.ts (100%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/lib/util.test.ts (100%) rename plugins/{catalog-backend/src/modules/github => catalog-backend-module-github/src}/lib/util.ts (100%) create mode 100644 plugins/catalog-backend-module-github/src/setupTests.ts diff --git a/.changeset/clever-garlics-rescue.md b/.changeset/clever-garlics-rescue.md index feaf1c2ef4..dcfa258575 100644 --- a/.changeset/clever-garlics-rescue.md +++ b/.changeset/clever-garlics-rescue.md @@ -2,6 +2,38 @@ '@backstage/plugin-catalog-backend': minor --- +**BREAKING**: Removed `GithubDiscoveryProcessor`, `GithubMultiOrgReaderProcessor`, `GitHubOrgEntityProvider`, `GithubOrgReaderProcessor`, and `GithubMultiOrgConfig` which now instead should be imported from `@backstage/plugin-catalog-backend-module-github`. NOTE THAT the `GithubDiscoveryProcessor` and `GithubOrgReaderProcessor` were part of the default set of processors in the catalog backend, and if you are a user of discovery or location based org ingestion on GitLab, you MUST now add them manually in the catalog initialization code of your backend. + +```diff +// In packages/backend/src/plugins/catalog.ts ++import { ++ GithubDiscoveryProcessor, ++ GithubOrgReaderProcessor, ++} from '@backstage/plugin-catalog-backend-module-github'; ++import { ++ ScmIntegrations, ++ DefaultGithubCredentialsProvider ++} from '@backstage/integration'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); ++ const integrations = ScmIntegrations.fromConfig(config); ++ const githubCredentialsProvider = ++ DefaultGithubCredentialsProvider.fromIntegrations(integrations); ++ builder.addProcessor( ++ GithubDiscoveryProcessor.fromConfig(config, { ++ logger, ++ githubCredentialsProvider, ++ }), ++ GithubOrgReaderProcessor.fromConfig(config, { ++ logger, ++ githubCredentialsProvider, ++ }), ++ ); +``` + **BREAKING**: Removed `GitLabDiscoveryProcessor`, which now instead should be imported from `@backstage/plugin-catalog-backend-module-gitlab`. NOTE THAT this processor was part of the default set of processors in the catalog backend, and if you are a user of discovery on GitLab, you MUST now add it manually in the catalog initialization code of your backend. ```diff diff --git a/.changeset/pink-chicken-smile.md b/.changeset/pink-chicken-smile.md new file mode 100644 index 0000000000..71fd8aad4e --- /dev/null +++ b/.changeset/pink-chicken-smile.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': minor +--- + +Added package, moving out GitHub specific functionality from the catalog-backend diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index aa9d1982c5..ba162e603c 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -102,7 +102,7 @@ Firekube Firestore Fiverr gitbeaker -GitHub +github gitlab GitLab Gource diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 39cb026703..866943e002 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -12,6 +12,53 @@ organization and register entities matching the configured path. This can be useful as an alternative to static locations or manually adding things to the catalog. +## Installation + +You will have to add the processors in the catalog initialization code of your +backend. They are not installed by default, therefore you have to add a +dependency to `@backstage/plugin-catalog-backend-module-github` to your backend +package. + +```bash +# From your Backstage root directory +cd packages/backend +yarn add @backstage/plugin-catalog-backend-module-github +``` + +And then add the processors to your catalog builder: + +```diff +// In packages/backend/src/plugins/catalog.ts ++import { ++ GithubDiscoveryProcessor, ++ GithubOrgReaderProcessor, ++} from '@backstage/plugin-catalog-backend-module-github'; ++import { ++ ScmIntegrations, ++ DefaultGithubCredentialsProvider ++} from '@backstage/integration'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); ++ const integrations = ScmIntegrations.fromConfig(config); ++ const githubCredentialsProvider = ++ DefaultGithubCredentialsProvider.fromIntegrations(integrations); ++ builder.addProcessor( ++ GithubDiscoveryProcessor.fromConfig(config, { ++ logger, ++ githubCredentialsProvider, ++ }), ++ GithubOrgReaderProcessor.fromConfig(config, { ++ logger, ++ githubCredentialsProvider, ++ }), ++ ); +``` + +## Configuration + To use the discovery processor, you'll need a GitHub integration [set up](locations.md) with a `GITHUB_TOKEN`. Then you can add a location target to the catalog configuration: diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 5843944e8a..ad0be6d071 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -19,8 +19,7 @@ entities that mirror your org setup. ## Installation -The processor that performs the import, `GithubOrgReaderProcessor`, comes -installed with the default setup of Backstage. +See the [discovery](discovery.md) article for installation instructions. ## Configuration diff --git a/plugins/catalog-backend-module-github/.eslintrc.js b/plugins/catalog-backend-module-github/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-backend-module-github/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-github/README.md b/plugins/catalog-backend-module-github/README.md new file mode 100644 index 0000000000..0d923d992c --- /dev/null +++ b/plugins/catalog-backend-module-github/README.md @@ -0,0 +1,8 @@ +# Catalog Backend Module for GitHub + +This is an extension module to the plugin-catalog-backend plugin, providing extensions targeted at GitHub offerings. + +## Getting started + +See [Backstage documentation](https://backstage.io/docs/integrations/github/discovery) for details on how to install +and configure the plugin. diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md new file mode 100644 index 0000000000..a1b3d34058 --- /dev/null +++ b/plugins/catalog-backend-module-github/api-report.md @@ -0,0 +1,126 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-github" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; +import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; +import { Config } from '@backstage/config'; +import { EntityProvider } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { GithubCredentialsProvider } from '@backstage/integration'; +import { GitHubIntegrationConfig } from '@backstage/integration'; +import { LocationSpec } from '@backstage/plugin-catalog-backend'; +import { Logger } from 'winston'; +import { ScmIntegrationRegistry } from '@backstage/integration'; + +// @public +export class GithubDiscoveryProcessor implements CatalogProcessor { + constructor(options: { + integrations: ScmIntegrationRegistry; + logger: Logger; + githubCredentialsProvider?: GithubCredentialsProvider; + }); + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger; + githubCredentialsProvider?: GithubCredentialsProvider; + }, + ): GithubDiscoveryProcessor; + // (undocumented) + getProcessorName(): string; + // (undocumented) + readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise; +} + +// @public +export type GithubMultiOrgConfig = Array<{ + name: string; + groupNamespace: string; + userNamespace: string | undefined; +}>; + +// @public +export class GithubMultiOrgReaderProcessor implements CatalogProcessor { + constructor(options: { + integrations: ScmIntegrationRegistry; + logger: Logger; + orgs: GithubMultiOrgConfig; + githubCredentialsProvider?: GithubCredentialsProvider; + }); + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger; + githubCredentialsProvider?: GithubCredentialsProvider; + }, + ): GithubMultiOrgReaderProcessor; + // (undocumented) + getProcessorName(): string; + // (undocumented) + readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise; +} + +// @public (undocumented) +export class GitHubOrgEntityProvider implements EntityProvider { + constructor(options: { + id: string; + orgUrl: string; + gitHubConfig: GitHubIntegrationConfig; + logger: Logger; + githubCredentialsProvider?: GithubCredentialsProvider; + }); + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + config: Config, + options: { + id: string; + orgUrl: string; + logger: Logger; + githubCredentialsProvider?: GithubCredentialsProvider; + }, + ): GitHubOrgEntityProvider; + // (undocumented) + getProviderName(): string; + // (undocumented) + read(): Promise; +} + +// @public +export class GithubOrgReaderProcessor implements CatalogProcessor { + constructor(options: { + integrations: ScmIntegrationRegistry; + logger: Logger; + githubCredentialsProvider?: GithubCredentialsProvider; + }); + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger; + githubCredentialsProvider?: GithubCredentialsProvider; + }, + ): GithubOrgReaderProcessor; + // (undocumented) + getProcessorName(): string; + // (undocumented) + readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise; +} +``` diff --git a/plugins/catalog-backend-module-github/package.json b/plugins/catalog-backend-module-github/package.json new file mode 100644 index 0000000000..fe4607ac16 --- /dev/null +++ b/plugins/catalog-backend-module-github/package.json @@ -0,0 +1,57 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-github", + "description": "A Backstage catalog backend module that helps integrate towards GitHub", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "backend-plugin-module" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-github" + }, + "keywords": [ + "backstage" + ], + "scripts": { + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean", + "start": "backstage-cli package start" + }, + "dependencies": { + "@backstage/backend-common": "^0.13.0-next.0", + "@backstage/catalog-model": "^0.13.0-next.0", + "@backstage/config": "^0.1.15", + "@backstage/errors": "^0.2.2", + "@backstage/integration": "^0.8.0", + "@backstage/plugin-catalog-backend": "^0.24.0-next.0", + "@backstage/types": "^0.1.3", + "@octokit/graphql": "^4.5.8", + "lodash": "^4.17.21", + "msw": "^0.35.0", + "node-fetch": "^2.6.7", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/backend-test-utils": "^0.1.21-next.0", + "@backstage/cli": "^0.15.2-next.0", + "@types/lodash": "^4.14.151" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.test.ts similarity index 98% rename from plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts rename to plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.test.ts index dd3bf490fb..3086386af8 100644 --- a/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.test.ts @@ -20,8 +20,8 @@ import { GithubCredentialsProvider, GitHubIntegrationConfig, } from '@backstage/integration'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; -import { EntityProviderConnection } from '../../api'; import { GitHubOrgEntityProvider, withLocations, diff --git a/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts similarity index 97% rename from plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts rename to plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts index 76389968ae..5bf501aff0 100644 --- a/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts @@ -27,16 +27,20 @@ import { ScmIntegrations, SingleInstanceGithubCredentialsProvider, } from '@backstage/integration'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; import { merge } from 'lodash'; import { Logger } from 'winston'; -import { EntityProvider, EntityProviderConnection } from '../../api'; import { + assignGroupsToUsers, + buildOrgHierarchy, getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, } from './lib'; -import { assignGroupsToUsers, buildOrgHierarchy } from '../util/org'; // TODO: Consider supporting an (optional) webhook that reacts on org changes /** @public */ diff --git a/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts rename to plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.test.ts index 5440539b27..b18e3baf75 100644 --- a/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.test.ts @@ -20,7 +20,7 @@ import { DefaultGithubCredentialsProvider, ScmIntegrations, } from '@backstage/integration'; -import { LocationSpec } from '../../api'; +import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; import { getOrganizationRepositories } from './lib'; diff --git a/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts b/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.ts similarity index 99% rename from plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts rename to plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.ts index 4aff470f52..78a1bca169 100644 --- a/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.ts @@ -21,15 +21,15 @@ import { ScmIntegrationRegistry, ScmIntegrations, } from '@backstage/integration'; -import { graphql } from '@octokit/graphql'; -import { Logger } from 'winston'; -import { getOrganizationRepositories } from './lib'; import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, processingResult, -} from '../../api'; +} from '@backstage/plugin-catalog-backend'; +import { graphql } from '@octokit/graphql'; +import { Logger } from 'winston'; +import { getOrganizationRepositories } from './lib'; /** * Extracts repositories out of a GitHub org. diff --git a/plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/GithubMultiOrgReaderProcessor.ts similarity index 98% rename from plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts rename to plugins/catalog-backend-module-github/src/GithubMultiOrgReaderProcessor.ts index dbd858fc0d..321e9cc8f8 100644 --- a/plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/GithubMultiOrgReaderProcessor.ts @@ -23,21 +23,21 @@ import { ScmIntegrationRegistry, ScmIntegrations, } from '@backstage/integration'; -import { graphql } from '@octokit/graphql'; -import { Logger } from 'winston'; -import { - getOrganizationTeams, - getOrganizationUsers, - GithubMultiOrgConfig, - readGithubMultiOrgConfig, -} from './lib'; import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, processingResult, -} from '../../api'; -import { buildOrgHierarchy } from '../util/org'; +} from '@backstage/plugin-catalog-backend'; +import { graphql } from '@octokit/graphql'; +import { Logger } from 'winston'; +import { + buildOrgHierarchy, + getOrganizationTeams, + getOrganizationUsers, + GithubMultiOrgConfig, + readGithubMultiOrgConfig, +} from './lib'; /** * Extracts teams and users out of a multiple GitHub orgs namespaced per org. diff --git a/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts rename to plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.test.ts index 9ce278ac44..2bb0c538eb 100644 --- a/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.test.ts @@ -17,12 +17,12 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { - ScmIntegrations, GithubCredentialsProvider, + ScmIntegrations, } from '@backstage/integration'; +import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; -import { LocationSpec } from '../../api'; jest.mock('@octokit/graphql'); diff --git a/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts similarity index 97% rename from plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts rename to plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts index e85e7b874d..84c12e82ba 100644 --- a/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts @@ -16,26 +16,27 @@ import { Config } from '@backstage/config'; import { + DefaultGithubCredentialsProvider, + GithubCredentialsProvider, GithubCredentialType, ScmIntegrationRegistry, ScmIntegrations, - GithubCredentialsProvider, - DefaultGithubCredentialsProvider, } from '@backstage/integration'; -import { graphql } from '@octokit/graphql'; -import { Logger } from 'winston'; -import { - getOrganizationTeams, - getOrganizationUsers, - parseGitHubOrgUrl, -} from './lib'; import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, processingResult, -} from '../../api'; -import { assignGroupsToUsers, buildOrgHierarchy } from '../util/org'; +} from '@backstage/plugin-catalog-backend'; +import { graphql } from '@octokit/graphql'; +import { Logger } from 'winston'; +import { + assignGroupsToUsers, + buildOrgHierarchy, + getOrganizationTeams, + getOrganizationUsers, + parseGitHubOrgUrl, +} from './lib'; type GraphQL = typeof graphql; diff --git a/plugins/catalog-backend/src/modules/github/index.ts b/plugins/catalog-backend-module-github/src/index.ts similarity index 85% rename from plugins/catalog-backend/src/modules/github/index.ts rename to plugins/catalog-backend-module-github/src/index.ts index 7958818096..1394fcd21b 100644 --- a/plugins/catalog-backend/src/modules/github/index.ts +++ b/plugins/catalog-backend-module-github/src/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2022 The Backstage Authors + * Copyright 2020 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,12 @@ * limitations under the License. */ +/** + * A Backstage catalog backend module that helps integrate towards GitHub + * + * @packageDocumentation + */ + export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; diff --git a/plugins/catalog-backend/src/modules/github/lib/config.test.ts b/plugins/catalog-backend-module-github/src/lib/config.test.ts similarity index 100% rename from plugins/catalog-backend/src/modules/github/lib/config.test.ts rename to plugins/catalog-backend-module-github/src/lib/config.test.ts diff --git a/plugins/catalog-backend/src/modules/github/lib/config.ts b/plugins/catalog-backend-module-github/src/lib/config.ts similarity index 100% rename from plugins/catalog-backend/src/modules/github/lib/config.ts rename to plugins/catalog-backend-module-github/src/lib/config.ts diff --git a/plugins/catalog-backend/src/modules/github/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts similarity index 100% rename from plugins/catalog-backend/src/modules/github/lib/github.test.ts rename to plugins/catalog-backend-module-github/src/lib/github.test.ts diff --git a/plugins/catalog-backend/src/modules/github/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts similarity index 100% rename from plugins/catalog-backend/src/modules/github/lib/github.ts rename to plugins/catalog-backend-module-github/src/lib/github.ts diff --git a/plugins/catalog-backend/src/modules/github/lib/index.ts b/plugins/catalog-backend-module-github/src/lib/index.ts similarity index 93% rename from plugins/catalog-backend/src/modules/github/lib/index.ts rename to plugins/catalog-backend-module-github/src/lib/index.ts index 3e31d9e00c..26ec3e8516 100644 --- a/plugins/catalog-backend/src/modules/github/lib/index.ts +++ b/plugins/catalog-backend-module-github/src/lib/index.ts @@ -21,4 +21,5 @@ export { getOrganizationTeams, getOrganizationUsers, } from './github'; +export { assignGroupsToUsers, buildOrgHierarchy } from './org'; export { parseGitHubOrgUrl } from './util'; diff --git a/plugins/catalog-backend/src/modules/util/org.test.ts b/plugins/catalog-backend-module-github/src/lib/org.test.ts similarity index 100% rename from plugins/catalog-backend/src/modules/util/org.test.ts rename to plugins/catalog-backend-module-github/src/lib/org.test.ts diff --git a/plugins/catalog-backend/src/modules/util/org.ts b/plugins/catalog-backend-module-github/src/lib/org.ts similarity index 100% rename from plugins/catalog-backend/src/modules/util/org.ts rename to plugins/catalog-backend-module-github/src/lib/org.ts diff --git a/plugins/catalog-backend/src/modules/github/lib/util.test.ts b/plugins/catalog-backend-module-github/src/lib/util.test.ts similarity index 100% rename from plugins/catalog-backend/src/modules/github/lib/util.test.ts rename to plugins/catalog-backend-module-github/src/lib/util.test.ts diff --git a/plugins/catalog-backend/src/modules/github/lib/util.ts b/plugins/catalog-backend-module-github/src/lib/util.ts similarity index 100% rename from plugins/catalog-backend/src/modules/github/lib/util.ts rename to plugins/catalog-backend-module-github/src/lib/util.ts diff --git a/plugins/catalog-backend-module-github/src/setupTests.ts b/plugins/catalog-backend-module-github/src/setupTests.ts new file mode 100644 index 0000000000..d3232290a7 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 5669596eb4..714fdffb87 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -16,8 +16,6 @@ import { Entity } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; import express from 'express'; import { GetEntitiesRequest } from '@backstage/catalog-client'; -import { GithubCredentialsProvider } from '@backstage/integration'; -import { GitHubIntegrationConfig } from '@backstage/integration'; import { JsonObject } from '@backstage/types'; import { JsonValue } from '@backstage/types'; import { Location as Location_2 } from '@backstage/catalog-client'; @@ -595,116 +593,6 @@ function generalError( message: string, ): CatalogProcessorResult; -// @public -export class GithubDiscoveryProcessor implements CatalogProcessor { - constructor(options: { - integrations: ScmIntegrationRegistry; - logger: Logger; - githubCredentialsProvider?: GithubCredentialsProvider; - }); - // (undocumented) - static fromConfig( - config: Config, - options: { - logger: Logger; - githubCredentialsProvider?: GithubCredentialsProvider; - }, - ): GithubDiscoveryProcessor; - // (undocumented) - getProcessorName(): string; - // (undocumented) - readLocation( - location: LocationSpec, - _optional: boolean, - emit: CatalogProcessorEmit, - ): Promise; -} - -// @public -export type GithubMultiOrgConfig = Array<{ - name: string; - groupNamespace: string; - userNamespace: string | undefined; -}>; - -// @public -export class GithubMultiOrgReaderProcessor implements CatalogProcessor { - constructor(options: { - integrations: ScmIntegrationRegistry; - logger: Logger; - orgs: GithubMultiOrgConfig; - githubCredentialsProvider?: GithubCredentialsProvider; - }); - // (undocumented) - static fromConfig( - config: Config, - options: { - logger: Logger; - githubCredentialsProvider?: GithubCredentialsProvider; - }, - ): GithubMultiOrgReaderProcessor; - // (undocumented) - getProcessorName(): string; - // (undocumented) - readLocation( - location: LocationSpec, - _optional: boolean, - emit: CatalogProcessorEmit, - ): Promise; -} - -// @public (undocumented) -export class GitHubOrgEntityProvider implements EntityProvider { - constructor(options: { - id: string; - orgUrl: string; - gitHubConfig: GitHubIntegrationConfig; - logger: Logger; - githubCredentialsProvider?: GithubCredentialsProvider; - }); - // (undocumented) - connect(connection: EntityProviderConnection): Promise; - // (undocumented) - static fromConfig( - config: Config, - options: { - id: string; - orgUrl: string; - logger: Logger; - githubCredentialsProvider?: GithubCredentialsProvider; - }, - ): GitHubOrgEntityProvider; - // (undocumented) - getProviderName(): string; - // (undocumented) - read(): Promise; -} - -// @public -export class GithubOrgReaderProcessor implements CatalogProcessor { - constructor(options: { - integrations: ScmIntegrationRegistry; - logger: Logger; - githubCredentialsProvider?: GithubCredentialsProvider; - }); - // (undocumented) - static fromConfig( - config: Config, - options: { - logger: Logger; - githubCredentialsProvider?: GithubCredentialsProvider; - }, - ): GithubOrgReaderProcessor; - // (undocumented) - getProcessorName(): string; - // (undocumented) - readLocation( - location: LocationSpec, - _optional: boolean, - emit: CatalogProcessorEmit, - ): Promise; -} - // @public @deprecated (undocumented) function inputError( atLocation: LocationSpec, diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 700141fb94..e9aee074f3 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -46,7 +46,6 @@ "@backstage/plugin-scaffolder-common": "^0.3.0-next.0", "@backstage/plugin-search-common": "^0.3.1-next.0", "@backstage/types": "^0.1.3", - "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", diff --git a/plugins/catalog-backend/src/modules/index.ts b/plugins/catalog-backend/src/modules/index.ts index 6509dcccee..10ebf65fed 100644 --- a/plugins/catalog-backend/src/modules/index.ts +++ b/plugins/catalog-backend/src/modules/index.ts @@ -16,4 +16,3 @@ export * from './codeowners'; export * from './core'; -export * from './github'; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 578f390f09..317eef1cea 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -28,11 +28,7 @@ import { stringifyEntityRef, Validators, } from '@backstage/catalog-model'; -import { - GithubCredentialsProvider, - ScmIntegrations, - DefaultGithubCredentialsProvider, -} from '@backstage/integration'; +import { ScmIntegrations } from '@backstage/integration'; import { createHash } from 'crypto'; import { Router } from 'express'; import lodash, { keyBy } from 'lodash'; @@ -48,8 +44,6 @@ import { BuiltinKindsEntityProcessor, CodeOwnersProcessor, FileReaderProcessor, - GithubDiscoveryProcessor, - GithubOrgReaderProcessor, PlaceholderProcessor, PlaceholderResolver, UrlReaderProcessor, @@ -351,19 +345,9 @@ export class CatalogBuilder { getDefaultProcessors(): CatalogProcessor[] { const { config, logger, reader } = this.env; const integrations = ScmIntegrations.fromConfig(config); - const githubCredentialsProvider: GithubCredentialsProvider = - DefaultGithubCredentialsProvider.fromIntegrations(integrations); return [ new FileReaderProcessor(), - GithubDiscoveryProcessor.fromConfig(config, { - logger, - githubCredentialsProvider, - }), - GithubOrgReaderProcessor.fromConfig(config, { - logger, - githubCredentialsProvider, - }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new AnnotateLocationEntityProcessor({ integrations }),