move gitlab to a separate package too

Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2022-03-09 20:53:13 +01:00
parent 704c8b2c05
commit d4934e19b1
31 changed files with 340 additions and 166 deletions
+32
View File
@@ -2,6 +2,38 @@
'@backstage/plugin-catalog-backend': minor
---
**BREAKING**: Removed `GithubDiscoveryProcessor`, `GithubMultiOrgReaderProcessor`, `GitHubOrgEntityProvider`, `GithubOrgReaderProcessor`, and `GithubMultiOrgConfig` which now instead should be imported from `@backstage/plugin-catalog-backend-module-github`. NOTE THAT the `GithubDiscoveryProcessor` and `GithubOrgReaderProcessor` were part of the default set of processors in the catalog backend, and if you are a user of discovery or location based org ingestion on GitLab, you MUST now add them manually in the catalog initialization code of your backend.
```diff
// In packages/backend/src/plugins/catalog.ts
+import {
+ GithubDiscoveryProcessor,
+ GithubOrgReaderProcessor,
+} from '@backstage/plugin-catalog-backend-module-github';
+import {
+ ScmIntegrations,
+ DefaultGithubCredentialsProvider
+} from '@backstage/integration';
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
+ const integrations = ScmIntegrations.fromConfig(config);
+ const githubCredentialsProvider =
+ DefaultGithubCredentialsProvider.fromIntegrations(integrations);
+ builder.addProcessor(
+ GithubDiscoveryProcessor.fromConfig(config, {
+ logger,
+ githubCredentialsProvider,
+ }),
+ GithubOrgReaderProcessor.fromConfig(config, {
+ logger,
+ githubCredentialsProvider,
+ }),
+ );
```
**BREAKING**: Removed `GitLabDiscoveryProcessor`, which now instead should be imported from `@backstage/plugin-catalog-backend-module-gitlab`. NOTE THAT this processor was part of the default set of processors in the catalog backend, and if you are a user of discovery on GitLab, you MUST now add it manually in the catalog initialization code of your backend.
```diff
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-github': minor
---
Added package, moving out GitHub specific functionality from the catalog-backend
+1 -1
View File
@@ -102,7 +102,7 @@ Firekube
Firestore
Fiverr
gitbeaker
GitHub
github
gitlab
GitLab
Gource
+47
View File
@@ -12,6 +12,53 @@ organization and register entities matching the configured path. This can be
useful as an alternative to static locations or manually adding things to the
catalog.
## Installation
You will have to add the processors in the catalog initialization code of your
backend. They are not installed by default, therefore you have to add a
dependency to `@backstage/plugin-catalog-backend-module-github` to your backend
package.
```bash
# From your Backstage root directory
cd packages/backend
yarn add @backstage/plugin-catalog-backend-module-github
```
And then add the processors to your catalog builder:
```diff
// In packages/backend/src/plugins/catalog.ts
+import {
+ GithubDiscoveryProcessor,
+ GithubOrgReaderProcessor,
+} from '@backstage/plugin-catalog-backend-module-github';
+import {
+ ScmIntegrations,
+ DefaultGithubCredentialsProvider
+} from '@backstage/integration';
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
+ const integrations = ScmIntegrations.fromConfig(config);
+ const githubCredentialsProvider =
+ DefaultGithubCredentialsProvider.fromIntegrations(integrations);
+ builder.addProcessor(
+ GithubDiscoveryProcessor.fromConfig(config, {
+ logger,
+ githubCredentialsProvider,
+ }),
+ GithubOrgReaderProcessor.fromConfig(config, {
+ logger,
+ githubCredentialsProvider,
+ }),
+ );
```
## Configuration
To use the discovery processor, you'll need a GitHub integration
[set up](locations.md) with a `GITHUB_TOKEN`. Then you can add a location target
to the catalog configuration:
+1 -2
View File
@@ -19,8 +19,7 @@ entities that mirror your org setup.
## Installation
The processor that performs the import, `GithubOrgReaderProcessor`, comes
installed with the default setup of Backstage.
See the [discovery](discovery.md) article for installation instructions.
## Configuration
@@ -0,0 +1 @@
module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
@@ -0,0 +1,8 @@
# Catalog Backend Module for GitHub
This is an extension module to the plugin-catalog-backend plugin, providing extensions targeted at GitHub offerings.
## Getting started
See [Backstage documentation](https://backstage.io/docs/integrations/github/discovery) for details on how to install
and configure the plugin.
@@ -0,0 +1,126 @@
## API Report File for "@backstage/plugin-catalog-backend-module-github"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
import { Config } from '@backstage/config';
import { EntityProvider } from '@backstage/plugin-catalog-backend';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { GithubCredentialsProvider } from '@backstage/integration';
import { GitHubIntegrationConfig } from '@backstage/integration';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import { Logger } from 'winston';
import { ScmIntegrationRegistry } from '@backstage/integration';
// @public
export class GithubDiscoveryProcessor implements CatalogProcessor {
constructor(options: {
integrations: ScmIntegrationRegistry;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GithubDiscoveryProcessor;
// (undocumented)
getProcessorName(): string;
// (undocumented)
readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
): Promise<boolean>;
}
// @public
export type GithubMultiOrgConfig = Array<{
name: string;
groupNamespace: string;
userNamespace: string | undefined;
}>;
// @public
export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
constructor(options: {
integrations: ScmIntegrationRegistry;
logger: Logger;
orgs: GithubMultiOrgConfig;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GithubMultiOrgReaderProcessor;
// (undocumented)
getProcessorName(): string;
// (undocumented)
readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
): Promise<boolean>;
}
// @public (undocumented)
export class GitHubOrgEntityProvider implements EntityProvider {
constructor(options: {
id: string;
orgUrl: string;
gitHubConfig: GitHubIntegrationConfig;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
connect(connection: EntityProviderConnection): Promise<void>;
// (undocumented)
static fromConfig(
config: Config,
options: {
id: string;
orgUrl: string;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GitHubOrgEntityProvider;
// (undocumented)
getProviderName(): string;
// (undocumented)
read(): Promise<void>;
}
// @public
export class GithubOrgReaderProcessor implements CatalogProcessor {
constructor(options: {
integrations: ScmIntegrationRegistry;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GithubOrgReaderProcessor;
// (undocumented)
getProcessorName(): string;
// (undocumented)
readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
): Promise<boolean>;
}
```
@@ -0,0 +1,57 @@
{
"name": "@backstage/plugin-catalog-backend-module-github",
"description": "A Backstage catalog backend module that helps integrate towards GitHub",
"version": "0.0.0",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
"private": false,
"publishConfig": {
"access": "public",
"main": "dist/index.cjs.js",
"types": "dist/index.d.ts"
},
"backstage": {
"role": "backend-plugin-module"
},
"homepage": "https://backstage.io",
"repository": {
"type": "git",
"url": "https://github.com/backstage/backstage",
"directory": "plugins/catalog-backend-module-github"
},
"keywords": [
"backstage"
],
"scripts": {
"build": "backstage-cli package build",
"lint": "backstage-cli package lint",
"test": "backstage-cli package test",
"prepack": "backstage-cli package prepack",
"postpack": "backstage-cli package postpack",
"clean": "backstage-cli package clean",
"start": "backstage-cli package start"
},
"dependencies": {
"@backstage/backend-common": "^0.13.0-next.0",
"@backstage/catalog-model": "^0.13.0-next.0",
"@backstage/config": "^0.1.15",
"@backstage/errors": "^0.2.2",
"@backstage/integration": "^0.8.0",
"@backstage/plugin-catalog-backend": "^0.24.0-next.0",
"@backstage/types": "^0.1.3",
"@octokit/graphql": "^4.5.8",
"lodash": "^4.17.21",
"msw": "^0.35.0",
"node-fetch": "^2.6.7",
"winston": "^3.2.1"
},
"devDependencies": {
"@backstage/backend-test-utils": "^0.1.21-next.0",
"@backstage/cli": "^0.15.2-next.0",
"@types/lodash": "^4.14.151"
},
"files": [
"dist"
]
}
@@ -20,8 +20,8 @@ import {
GithubCredentialsProvider,
GitHubIntegrationConfig,
} from '@backstage/integration';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { graphql } from '@octokit/graphql';
import { EntityProviderConnection } from '../../api';
import {
GitHubOrgEntityProvider,
withLocations,
@@ -27,16 +27,20 @@ import {
ScmIntegrations,
SingleInstanceGithubCredentialsProvider,
} from '@backstage/integration';
import {
EntityProvider,
EntityProviderConnection,
} from '@backstage/plugin-catalog-backend';
import { graphql } from '@octokit/graphql';
import { merge } from 'lodash';
import { Logger } from 'winston';
import { EntityProvider, EntityProviderConnection } from '../../api';
import {
assignGroupsToUsers,
buildOrgHierarchy,
getOrganizationTeams,
getOrganizationUsers,
parseGitHubOrgUrl,
} from './lib';
import { assignGroupsToUsers, buildOrgHierarchy } from '../util/org';
// TODO: Consider supporting an (optional) webhook that reacts on org changes
/** @public */
@@ -20,7 +20,7 @@ import {
DefaultGithubCredentialsProvider,
ScmIntegrations,
} from '@backstage/integration';
import { LocationSpec } from '../../api';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor';
import { getOrganizationRepositories } from './lib';
@@ -21,15 +21,15 @@ import {
ScmIntegrationRegistry,
ScmIntegrations,
} from '@backstage/integration';
import { graphql } from '@octokit/graphql';
import { Logger } from 'winston';
import { getOrganizationRepositories } from './lib';
import {
CatalogProcessor,
CatalogProcessorEmit,
LocationSpec,
processingResult,
} from '../../api';
} from '@backstage/plugin-catalog-backend';
import { graphql } from '@octokit/graphql';
import { Logger } from 'winston';
import { getOrganizationRepositories } from './lib';
/**
* Extracts repositories out of a GitHub org.
@@ -23,21 +23,21 @@ import {
ScmIntegrationRegistry,
ScmIntegrations,
} from '@backstage/integration';
import { graphql } from '@octokit/graphql';
import { Logger } from 'winston';
import {
getOrganizationTeams,
getOrganizationUsers,
GithubMultiOrgConfig,
readGithubMultiOrgConfig,
} from './lib';
import {
CatalogProcessor,
CatalogProcessorEmit,
LocationSpec,
processingResult,
} from '../../api';
import { buildOrgHierarchy } from '../util/org';
} from '@backstage/plugin-catalog-backend';
import { graphql } from '@octokit/graphql';
import { Logger } from 'winston';
import {
buildOrgHierarchy,
getOrganizationTeams,
getOrganizationUsers,
GithubMultiOrgConfig,
readGithubMultiOrgConfig,
} from './lib';
/**
* Extracts teams and users out of a multiple GitHub orgs namespaced per org.
@@ -17,12 +17,12 @@
import { getVoidLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import {
ScmIntegrations,
GithubCredentialsProvider,
ScmIntegrations,
} from '@backstage/integration';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import { graphql } from '@octokit/graphql';
import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
import { LocationSpec } from '../../api';
jest.mock('@octokit/graphql');
@@ -16,26 +16,27 @@
import { Config } from '@backstage/config';
import {
DefaultGithubCredentialsProvider,
GithubCredentialsProvider,
GithubCredentialType,
ScmIntegrationRegistry,
ScmIntegrations,
GithubCredentialsProvider,
DefaultGithubCredentialsProvider,
} from '@backstage/integration';
import { graphql } from '@octokit/graphql';
import { Logger } from 'winston';
import {
getOrganizationTeams,
getOrganizationUsers,
parseGitHubOrgUrl,
} from './lib';
import {
CatalogProcessor,
CatalogProcessorEmit,
LocationSpec,
processingResult,
} from '../../api';
import { assignGroupsToUsers, buildOrgHierarchy } from '../util/org';
} from '@backstage/plugin-catalog-backend';
import { graphql } from '@octokit/graphql';
import { Logger } from 'winston';
import {
assignGroupsToUsers,
buildOrgHierarchy,
getOrganizationTeams,
getOrganizationUsers,
parseGitHubOrgUrl,
} from './lib';
type GraphQL = typeof graphql;
@@ -1,5 +1,5 @@
/*
* Copyright 2022 The Backstage Authors
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,6 +14,12 @@
* limitations under the License.
*/
/**
* A Backstage catalog backend module that helps integrate towards GitHub
*
* @packageDocumentation
*/
export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor';
export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor';
export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider';
@@ -21,4 +21,5 @@ export {
getOrganizationTeams,
getOrganizationUsers,
} from './github';
export { assignGroupsToUsers, buildOrgHierarchy } from './org';
export { parseGitHubOrgUrl } from './util';
@@ -0,0 +1,17 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {};
-112
View File
@@ -16,8 +16,6 @@ import { Entity } from '@backstage/catalog-model';
import { EntityPolicy } from '@backstage/catalog-model';
import express from 'express';
import { GetEntitiesRequest } from '@backstage/catalog-client';
import { GithubCredentialsProvider } from '@backstage/integration';
import { GitHubIntegrationConfig } from '@backstage/integration';
import { JsonObject } from '@backstage/types';
import { JsonValue } from '@backstage/types';
import { Location as Location_2 } from '@backstage/catalog-client';
@@ -595,116 +593,6 @@ function generalError(
message: string,
): CatalogProcessorResult;
// @public
export class GithubDiscoveryProcessor implements CatalogProcessor {
constructor(options: {
integrations: ScmIntegrationRegistry;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GithubDiscoveryProcessor;
// (undocumented)
getProcessorName(): string;
// (undocumented)
readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
): Promise<boolean>;
}
// @public
export type GithubMultiOrgConfig = Array<{
name: string;
groupNamespace: string;
userNamespace: string | undefined;
}>;
// @public
export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
constructor(options: {
integrations: ScmIntegrationRegistry;
logger: Logger;
orgs: GithubMultiOrgConfig;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GithubMultiOrgReaderProcessor;
// (undocumented)
getProcessorName(): string;
// (undocumented)
readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
): Promise<boolean>;
}
// @public (undocumented)
export class GitHubOrgEntityProvider implements EntityProvider {
constructor(options: {
id: string;
orgUrl: string;
gitHubConfig: GitHubIntegrationConfig;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
connect(connection: EntityProviderConnection): Promise<void>;
// (undocumented)
static fromConfig(
config: Config,
options: {
id: string;
orgUrl: string;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GitHubOrgEntityProvider;
// (undocumented)
getProviderName(): string;
// (undocumented)
read(): Promise<void>;
}
// @public
export class GithubOrgReaderProcessor implements CatalogProcessor {
constructor(options: {
integrations: ScmIntegrationRegistry;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GithubOrgReaderProcessor;
// (undocumented)
getProcessorName(): string;
// (undocumented)
readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
): Promise<boolean>;
}
// @public @deprecated (undocumented)
function inputError(
atLocation: LocationSpec,
-1
View File
@@ -46,7 +46,6 @@
"@backstage/plugin-scaffolder-common": "^0.3.0-next.0",
"@backstage/plugin-search-common": "^0.3.1-next.0",
"@backstage/types": "^0.1.3",
"@octokit/graphql": "^4.5.8",
"@types/express": "^4.17.6",
"codeowners-utils": "^1.0.2",
"core-js": "^3.6.5",
@@ -16,4 +16,3 @@
export * from './codeowners';
export * from './core';
export * from './github';
@@ -28,11 +28,7 @@ import {
stringifyEntityRef,
Validators,
} from '@backstage/catalog-model';
import {
GithubCredentialsProvider,
ScmIntegrations,
DefaultGithubCredentialsProvider,
} from '@backstage/integration';
import { ScmIntegrations } from '@backstage/integration';
import { createHash } from 'crypto';
import { Router } from 'express';
import lodash, { keyBy } from 'lodash';
@@ -48,8 +44,6 @@ import {
BuiltinKindsEntityProcessor,
CodeOwnersProcessor,
FileReaderProcessor,
GithubDiscoveryProcessor,
GithubOrgReaderProcessor,
PlaceholderProcessor,
PlaceholderResolver,
UrlReaderProcessor,
@@ -351,19 +345,9 @@ export class CatalogBuilder {
getDefaultProcessors(): CatalogProcessor[] {
const { config, logger, reader } = this.env;
const integrations = ScmIntegrations.fromConfig(config);
const githubCredentialsProvider: GithubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
return [
new FileReaderProcessor(),
GithubDiscoveryProcessor.fromConfig(config, {
logger,
githubCredentialsProvider,
}),
GithubOrgReaderProcessor.fromConfig(config, {
logger,
githubCredentialsProvider,
}),
new UrlReaderProcessor({ reader, logger }),
CodeOwnersProcessor.fromConfig(config, { logger, reader }),
new AnnotateLocationEntityProcessor({ integrations }),