Merge pull request #10669 from anicke/gerrit-entity-provider

New plugin: "plugin-catalog-backend-module-gerrit"
This commit is contained in:
Johan Haals
2022-04-21 14:20:23 +02:00
committed by GitHub
23 changed files with 845 additions and 1 deletions
@@ -0,0 +1 @@
module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
@@ -0,0 +1,8 @@
# Catalog Backend Module for Gerrit
This is an extension module to the plugin-catalog-backend plugin, providing extensions targeted at Gerrit integrations.
## Getting started
See [Backstage documentation](https://backstage.io/docs/integrations/gerrit/discovery.md)
for details on how to install and configure the plugin.
@@ -0,0 +1,31 @@
## API Report File for "@backstage/plugin-catalog-backend-module-gerrit"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { Config } from '@backstage/config';
import { EntityProvider } from '@backstage/plugin-catalog-backend';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { Logger } from 'winston';
import { TaskRunner } from '@backstage/backend-tasks';
// @public (undocumented)
export class GerritEntityProvider implements EntityProvider {
// (undocumented)
connect(connection: EntityProviderConnection): Promise<void>;
// (undocumented)
static fromConfig(
configRoot: Config,
options: {
logger: Logger;
schedule: TaskRunner;
},
): GerritEntityProvider[];
// (undocumented)
getProviderName(): string;
// (undocumented)
refresh(logger: Logger): Promise<void>;
}
// (No @packageDocumentation comment for this package)
```
+51
View File
@@ -0,0 +1,51 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
interface GerritConfig {
/**
* (Required) The host of the Gerrit integration to use.
* @visibility backend
*/
host: string;
/**
* (Required) The query to use for the "List Projects" API call. Used to limit the
* scope of the projects that the provider tries to ingest.
* @visibility backend
*/
query: string;
/**
* (Optional) Branch.
* The branch where the provider will try to find entities. Defaults to "master".
* @visibility backend
*/
branch?: string;
}
export interface Config {
catalog?: {
/**
* List of provider-specific options and attributes
*/
providers?: {
/**
* GerritEntityProvider configuration
*
* Maps provider id with configuration.
*/
gerrit?: Record<string, GerritConfig>;
};
};
}
@@ -0,0 +1,54 @@
{
"name": "@backstage/plugin-catalog-backend-module-gerrit",
"version": "0.0.0",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
"publishConfig": {
"access": "public",
"main": "dist/index.esm.js",
"types": "dist/index.d.ts"
},
"backstage": {
"role": "backend-plugin-module"
},
"homepage": "https://backstage.io",
"repository": {
"type": "git",
"url": "https://github.com/backstage/backstage",
"directory": "plugins/catalog-backend-module-gerrit"
},
"scripts": {
"start": "backstage-cli package start",
"build": "backstage-cli package build",
"lint": "backstage-cli package lint",
"test": "backstage-cli package test",
"clean": "backstage-cli package clean",
"prepack": "backstage-cli package prepack",
"postpack": "backstage-cli package postpack"
},
"dependencies": {
"@backstage/backend-common": "^0.13.2",
"@backstage/backend-tasks": "^0.3.0",
"@backstage/catalog-model": "^1.0.1",
"@backstage/config": "^1.0.0",
"@backstage/errors": "^1.0.0",
"@backstage/integration": "^1.1.0",
"@backstage/plugin-catalog-backend": "^1.1.0",
"fs-extra": "10.0.1",
"msw": "^0.35.0",
"node-fetch": "^2.6.7",
"uuid": "^8.0.0",
"winston": "^3.2.1"
},
"devDependencies": {
"@backstage/backend-test-utils": "^0.1.23",
"@backstage/cli": "^0.17.0",
"@types/fs-extra": "^9.0.1"
},
"files": [
"dist",
"config.d.ts"
],
"configSchema": "config.d.ts"
}
@@ -0,0 +1,17 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { GerritEntityProvider } from './providers/GerritEntityProvider';
@@ -0,0 +1,206 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { rest } from 'msw';
import fs from 'fs-extra';
import path from 'path';
import { setupServer } from 'msw/node';
import { GerritEntityProvider } from './GerritEntityProvider';
const server = setupServer();
const getJsonFixture = (fileName: string) =>
JSON.parse(
fs.readFileSync(
path.resolve(__dirname, `__fixtures__/${fileName}`),
'utf8',
),
);
class PersistingTaskRunner implements TaskRunner {
private tasks: TaskInvocationDefinition[] = [];
getTasks() {
return this.tasks;
}
run(task: TaskInvocationDefinition): Promise<void> {
this.tasks.push(task);
return Promise.resolve(undefined);
}
}
const logger = getVoidLogger();
describe('GerritEntityProvider', () => {
beforeAll(() => server.listen());
afterEach(() => {
jest.resetAllMocks();
server.resetHandlers();
});
afterAll(() => server.close());
const config = new ConfigReader({
catalog: {
providers: {
gerrit: {
'active-training': {
host: 'g.com',
query: 'state=ACTIVE&prefix=training',
branch: 'main',
},
},
},
},
integrations: {
gerrit: [
{
host: 'g.com',
baseUrl: 'https://g.com/gerrit',
gitilesBaseUrl: 'https:/g.com/gitiles',
},
],
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
};
it('discovers projects from the api.', async () => {
const repoBuffer = fs.readFileSync(
path.resolve(__dirname, '__fixtures__/listProjectsBody.txt'),
);
const expected = getJsonFixture('expectedProviderEntities.json');
server.use(
rest.get('https://g.com/gerrit/projects/', (_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.body(repoBuffer),
),
),
);
const provider = GerritEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
expect(provider.getProviderName()).toEqual(
'gerrit-provider:active-training',
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('gerrit-provider:active-training:refresh');
await (taskDef.fn as () => Promise<void>)();
expect(entityProviderConnection.applyMutation).toBeCalledWith(expected);
});
it('handles api errors.', async () => {
const provider = GerritEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
server.use(
rest.get('https://g.com/gerrit/projects/', (_, res, ctx) =>
res(ctx.status(500, 'Error!.')),
),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
await (taskDef.fn as () => Promise<void>)();
expect(entityProviderConnection.applyMutation).not.toHaveBeenCalled();
});
it('can create multiple providers from config.', async () => {
const configTwoProviders = new ConfigReader({
catalog: {
providers: {
gerrit: {
'active-g1': {
host: 'gerrit1.com',
query: 'state=ACTIVE',
},
'active-g2': {
host: 'gerrit2.com',
query: 'state=ACTIVE',
},
},
},
},
integrations: {
gerrit: [
{
host: 'gerrit1.com',
},
{
host: 'gerrit2.com',
},
],
},
});
const providers = GerritEntityProvider.fromConfig(configTwoProviders, {
logger,
schedule,
});
expect(providers).toHaveLength(2);
expect(providers[0]).toBeInstanceOf(GerritEntityProvider);
expect(providers[0].getProviderName()).toEqual('gerrit-provider:active-g1');
expect(providers[1]).toBeInstanceOf(GerritEntityProvider);
expect(providers[1].getProviderName()).toEqual('gerrit-provider:active-g2');
});
it('throws if integration is missing.', () => {
const configMissingIntegration = new ConfigReader({
catalog: {
providers: {
gerrit: {
'active-g2': {
host: 'gerrit2.com',
query: 'state=ACTIVE',
},
},
},
},
integrations: {
gerrit: [
{
host: 'gerrit1.com',
},
],
},
});
expect(() =>
GerritEntityProvider.fromConfig(configMissingIntegration, {
logger,
schedule,
}),
).toThrow(/No gerrit integration/);
});
});
@@ -0,0 +1,165 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TaskRunner } from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
import { InputError } from '@backstage/errors';
import {
EntityProvider,
EntityProviderConnection,
LocationSpec,
locationSpecToLocationEntity,
} from '@backstage/plugin-catalog-backend';
import fetch, { Response } from 'node-fetch';
import {
GerritIntegration,
getGerritProjectsApiUrl,
getGerritRequestOptions,
parseGerritJsonResponse,
ScmIntegrations,
} from '@backstage/integration';
import * as uuid from 'uuid';
import { Logger } from 'winston';
import { readGerritConfigs } from './config';
import { GerritProjectQueryResult, GerritProviderConfig } from './types';
/** @public */
export class GerritEntityProvider implements EntityProvider {
private readonly config: GerritProviderConfig;
private readonly integration: GerritIntegration;
private readonly logger: Logger;
private readonly scheduleFn: () => Promise<void>;
private connection?: EntityProviderConnection;
static fromConfig(
configRoot: Config,
options: {
logger: Logger;
schedule: TaskRunner;
},
): GerritEntityProvider[] {
const providerConfigs = readGerritConfigs(configRoot);
const integrations = ScmIntegrations.fromConfig(configRoot).gerrit;
const providers: GerritEntityProvider[] = [];
providerConfigs.forEach(providerConfig => {
const integration = integrations.byHost(providerConfig.host);
if (!integration) {
throw new InputError(
`No gerrit integration found that matches host ${providerConfig.host}`,
);
}
providers.push(
new GerritEntityProvider(
providerConfig,
integration,
options.logger,
options.schedule,
),
);
});
return providers;
}
private constructor(
config: GerritProviderConfig,
integration: GerritIntegration,
logger: Logger,
schedule: TaskRunner,
) {
this.config = config;
this.integration = integration;
this.logger = logger.child({
target: this.getProviderName(),
});
this.scheduleFn = this.createScheduleFn(schedule);
}
getProviderName(): string {
return `gerrit-provider:${this.config.id}`;
}
async connect(connection: EntityProviderConnection): Promise<void> {
this.connection = connection;
await this.scheduleFn();
}
private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
return async () => {
const taskId = `${this.getProviderName()}:refresh`;
return schedule.run({
id: taskId,
fn: async () => {
const logger = this.logger.child({
class: GerritEntityProvider.prototype.constructor.name,
taskId,
taskInstanceId: uuid.v4(),
});
try {
await this.refresh(logger);
} catch (error) {
logger.error(error);
}
},
});
};
}
async refresh(logger: Logger): Promise<void> {
if (!this.connection) {
throw new Error('Gerrit discovery connection not initialized');
}
let response: Response;
const baseProjectApiUrl = getGerritProjectsApiUrl(this.integration.config);
const projectQueryUrl = `${baseProjectApiUrl}?${this.config.query}`;
try {
response = await fetch(projectQueryUrl, {
method: 'GET',
...getGerritRequestOptions(this.integration.config),
});
} catch (e) {
throw new Error(
`Failed to list Gerrit projects for query ${this.config.query}, ${e}`,
);
}
const gerritProjectsResponse = (await parseGerritJsonResponse(
response as any,
)) as GerritProjectQueryResult;
const projects = Object.keys(gerritProjectsResponse);
const locations = projects.map(project => this.createLocationSpec(project));
await this.connection.applyMutation({
type: 'full',
entities: locations.map(location => ({
locationKey: this.getProviderName(),
entity: locationSpecToLocationEntity({ location }),
})),
});
logger.info(`Found ${locations.length} locations.`);
}
private createLocationSpec(project: string): LocationSpec {
return {
type: 'url',
target: `${this.integration.config.gitilesBaseUrl}/${project}/+/refs/heads/${this.config.branch}/catalog-info.yaml`,
presence: 'optional',
};
}
}
@@ -0,0 +1,43 @@
{
"entities": [
{
"entity": {
"apiVersion": "backstage.io/v1alpha1",
"kind": "Location",
"metadata": {
"annotations": {
"backstage.io/managed-by-location": "url:https:/g.com/gitiles/training/gerrit/+/refs/heads/main/catalog-info.yaml",
"backstage.io/managed-by-origin-location": "url:https:/g.com/gitiles/training/gerrit/+/refs/heads/main/catalog-info.yaml"
},
"name": "generated-acbbddb9193be55b61de6a7c1d39b3dcaaf27525"
},
"spec": {
"presence": "optional",
"target": "https:/g.com/gitiles/training/gerrit/+/refs/heads/main/catalog-info.yaml",
"type": "url"
}
},
"locationKey": "gerrit-provider:active-training"
},
{
"entity": {
"apiVersion": "backstage.io/v1alpha1",
"kind": "Location",
"metadata": {
"annotations": {
"backstage.io/managed-by-location": "url:https:/g.com/gitiles/training/sample/+/refs/heads/main/catalog-info.yaml",
"backstage.io/managed-by-origin-location": "url:https:/g.com/gitiles/training/sample/+/refs/heads/main/catalog-info.yaml"
},
"name": "generated-4b8debb1c2ffa8d58c0bdf3d75abcf2252494e65"
},
"spec": {
"presence": "optional",
"target": "https:/g.com/gitiles/training/sample/+/refs/heads/main/catalog-info.yaml",
"type": "url"
}
},
"locationKey": "gerrit-provider:active-training"
}
],
"type": "full"
}
@@ -0,0 +1,2 @@
)]}'
{"training/gerrit":{"id":"training%2Fgerrit","state":"ACTIVE","web_links":[{"name":"browse","url":"","target":"_blank"}]},"training/sample":{"id":"training%2Fsample","state":"ACTIVE","web_links":[{"name":"browse","url":"","target":"_blank"}]}}
@@ -0,0 +1,71 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { readGerritConfigs } from './config';
describe('readGerritConfigs', () => {
it('reads all provider configs', () => {
const provider1 = {
host: 'gerrit1.com',
query: 'state=ACTIVE',
branch: 'main',
};
const provider2 = {
host: 'gerrit2.com',
query: 'state=ACTIVE',
branch: 'main',
};
const config = {
catalog: {
providers: {
gerrit: {
'active-g1': provider1,
'active-g2': provider2,
},
},
},
};
const actual = readGerritConfigs(new ConfigReader(config));
expect(actual).toHaveLength(2);
expect(actual[0]).toEqual({ ...provider1, id: 'active-g1' });
expect(actual[1]).toEqual({ ...provider2, id: 'active-g2' });
});
it('provides default values', () => {
const provider = {
host: 'gerrit1.com',
query: 'state=ACTIVE',
};
const config = {
catalog: {
providers: {
gerrit: {
'active-g1': provider,
},
},
},
};
const actual = readGerritConfigs(new ConfigReader(config));
expect(actual).toHaveLength(1);
expect(actual[0]).toEqual({
branch: 'master',
id: 'active-g1',
...provider,
});
});
});
@@ -0,0 +1,47 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config } from '@backstage/config';
import { GerritProviderConfig } from './types';
function readGerritConfig(id: string, config: Config): GerritProviderConfig {
const branch = config.getOptionalString('branch') ?? 'master';
const host = config.getString('host');
const query = config.getString('query');
return {
branch,
host,
id,
query,
};
}
export function readGerritConfigs(config: Config): GerritProviderConfig[] {
const configs: GerritProviderConfig[] = [];
const providerConfigs = config.getOptionalConfig('catalog.providers.gerrit');
if (!providerConfigs) {
return configs;
}
for (const id of providerConfigs.keys()) {
configs.push(readGerritConfig(id, providerConfigs.getConfig(id)));
}
return configs;
}
@@ -0,0 +1,31 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type GerritProjectInfo = {
id: string;
name: string;
parent?: string;
state?: string;
};
export type GerritProjectQueryResult = Record<string, GerritProjectInfo>;
export type GerritProviderConfig = {
host: string;
query: string;
id: string;
branch?: string;
};
@@ -0,0 +1,16 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {};