From 1136ac88b604946f522a6e08333f5e271580384b Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 15:03:51 +0200 Subject: [PATCH] First part of GitLabDiscoveryProcessor (no target parsing yet) Signed-off-by: Roy Jacobs --- .../GitLabDiscoveryProcessor.test.ts | 252 ++++++++++++++++++ .../processors/GitLabDiscoveryProcessor.ts | 129 +++++++++ .../src/ingestion/processors/gitlab/client.ts | 94 +++++++ .../src/ingestion/processors/gitlab/index.ts | 18 ++ .../src/ingestion/processors/gitlab/types.ts | 23 ++ .../src/ingestion/processors/index.ts | 1 + .../src/next/NextCatalogBuilder.ts | 2 + .../src/service/CatalogBuilder.ts | 2 + 8 files changed, 521 insertions(+) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..5eb5c51869 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts @@ -0,0 +1,252 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { + GitLabDiscoveryProcessor, +} from './GitLabDiscoveryProcessor'; +import { setupServer } from 'msw/node'; +import { rest } from 'msw'; +import { GitLabProject } from "./gitlab"; + +const server = setupServer(); + +function setupFakeGitLab( + callback: (request: { + page: number; + }) => { data: GitLabProject[]; nextPage?: number }, +) { + server.use( + rest.get('https://gitlab.fake/api/v4/projects', (req, res, ctx) => { + if (req.headers.get('private-token') !== 'test-token') { + return res(ctx.status(401), ctx.json({})); + } + const page = req.url.searchParams.get('page'); + const response = callback({ + page: parseInt(page!, 10), + }); + + // Filter the fake results based on the `last_activity_after` parameter + const last_activity_after = req.url.searchParams.get( + 'last_activity_after', + ); + const filteredData = response.data.filter( + v => + !last_activity_after || + Date.parse(v.last_activity_at) >= Date.parse(last_activity_after), + ); + + return res( + ctx.set('x-next-page', response.nextPage?.toString() ?? ''), + ctx.json(filteredData), + ); + }), + ); +} + +function getConfig(): any { + return { + backend: { + cache: { store: 'memory' }, + }, + integrations: { + gitlab: [ + { + host: 'gitlab.fake', + apiBaseUrl: 'https://gitlab.fake/api/v4', + token: 'test-token', + }, + ], + }, + }; +} + +function getProcessor(config?: any): GitLabDiscoveryProcessor { + return GitLabDiscoveryProcessor.fromConfig( + new ConfigReader(config || getConfig()), + { + logger: getVoidLogger(), + }, + ); +} + +describe('GitlabDiscoveryProcessor', () => { + beforeAll(() => { + server.listen(); + jest.useFakeTimers('modern'); + jest.setSystemTime(new Date('2001-01-01T12:34:56Z')); + }); + afterEach(() => server.resetHandlers()); + afterAll(() => { + server.close(); + jest.useRealTimers(); + }); + + const location: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/north-star', + }; + + describe('handles repositories', () => { + it('pages through all repositories', async () => { + const processor = getProcessor(); + setupFakeGitLab(request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + nextPage: 2, + }; + case 2: + return { + data: [ + { + id: 2, + archived: false, + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/2', + }, + { + id: 3, + archived: true, // ARCHIVED + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/3', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(location, false, e => { + result.push(e); + }); + expect(result).toEqual([ + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/1/-/blob/main/catalog-info.yaml', + }, + optional: true, + }, + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/2/-/blob/master/catalog-info.yaml', + }, + optional: true, + }, + ]); + }); + + it('uses the previous scan timestamp to filter', async () => { + const processor = getProcessor(); + setupFakeGitLab(request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2000-01-01T00:00:00Z', + web_url: 'https://gitlab.fake/1', + }, + { + id: 2, + archived: false, + default_branch: 'main', + last_activity_at: '2002-01-01T00:00:00Z', + web_url: 'https://gitlab.fake/2', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + + // First scan should find all repos, since no last activity was cached + await processor.readLocation(location, false, e => { + result.push(e); + }); + expect(result).toHaveLength(2); + + // Second scan should have used the mocked Date to set the last scanned time to 2001 + // This should result in only the second repo being scanned, since that has a timestamp of 2002 + const result2: any[] = []; + await processor.readLocation(location, false, e => { + result2.push(e); + }); + expect(result2).toHaveLength(1); + }); + }); + + describe('handles failure', () => { + it('invalid token', async () => { + // Setup an empty fake gitlab, since we don't care about actual results + setupFakeGitLab(_ => { + return { + data: [], + }; + }); + + const config = getConfig(); + config.integrations.gitlab[0].token = 'invalid'; + await expect( + getProcessor(config).readLocation(location, false, _ => {}), + ).rejects.toThrow(/Unauthorized/); + }); + + it('missing integration', async () => { + const config = getConfig(); + delete config.integrations; + await expect( + getProcessor(config).readLocation(location, false, _ => {}), + ).rejects.toThrow(/no GitLab integration/); + }); + + it('location type', async () => { + const incorrectLocation: LocationSpec = { + type: 'something-that-is-not-gitlab-discovery', + target: 'https://gitlab.fake/north-star', + }; + + await expect( + getProcessor().readLocation(incorrectLocation, false, _ => {}), + ).resolves.toBeFalsy(); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts new file mode 100644 index 0000000000..b3cce6e711 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + ScmIntegrations, +} from '@backstage/integration'; +import { Logger } from 'winston'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { GitLabClient, GitLabProject, paginated } from "./gitlab"; +import { CacheClient, CacheManager, PluginCacheManager } from "@backstage/backend-common"; + +/** + * Extracts repositories out of a GitLab org. + */ +export class GitLabDiscoveryProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly logger: Logger; + private readonly cache: CacheClient; + + static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + const pluginCache = CacheManager.fromConfig(config).forPlugin( + 'gitlab-discovery', + ); + + return new GitLabDiscoveryProcessor({ + ...options, + integrations, + pluginCache, + }); + } + + private constructor(options: { + integrations: ScmIntegrations; + pluginCache: PluginCacheManager; + logger: Logger; + }) { + this.integrations = options.integrations; + this.cache = options.pluginCache.getClient(); + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'gitlab-discovery') { + return false; + } + + const integration = this.integrations.gitlab.byUrl(location.target); + if (!integration) { + throw new Error( + `There is no GitLab integration that matches ${location.target}. Please add a configuration entry for it under integrations.gitlab`, + ); + } + + const client = new GitLabClient({ + config: integration.config, + logger: this.logger, + }); + const startTimestamp = Date.now(); + this.logger.info(`Reading GitLab projects from ${location.target}`); + + const projects = paginated(options => client.listProjects(options), { + last_activity_after: await this.updateLastActivity(), + page: 1, + }); + + const result: Result = { + scanned: 0, + matches: [], + }; + for await (const project of projects) { + result.scanned++; + if (!project.archived) { + result.matches.push(project); + } + } + + for (const project of result.matches) { + emit( + results.location( + { + type: 'url', + // The format expected by the GitLabUrlReader: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + target: `${project.web_url}/-/blob/${project.default_branch}/catalog-info.yaml`, + }, + true, + ), + ); + } + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${result.scanned} GitLab repositories in ${duration} seconds`, + ); + + return true; + } + + async updateLastActivity(): Promise { + const lastActivity = await this.cache.get('last-activity'); + await this.cache.set('last-activity', new Date().toISOString()); + return lastActivity as string | undefined; + } +} + +type Result = { + scanned: number; + matches: GitLabProject[]; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts new file mode 100644 index 0000000000..fb7763dd1c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'cross-fetch'; +import { + getGitLabRequestOptions, + GitLabIntegrationConfig, +} from '@backstage/integration'; +import { Logger } from 'winston'; + +export class GitLabClient { + private readonly config: GitLabIntegrationConfig; + private readonly logger: Logger; + + constructor(options: { config: GitLabIntegrationConfig; logger: Logger }) { + this.config = options.config; + this.logger = options.logger; + } + + async listProjects(options?: ListOptions): Promise> { + return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); + } + + private async pagedRequest( + endpoint: string, + options?: ListOptions, + ): Promise> { + const request = new URL(endpoint); + for (const key in options) { + if (options[key]) { + request.searchParams.append(key, options[key]!.toString()); + } + } + + this.logger.debug(`Fetching: ${request.toString()}`); + const response = await fetch( + request.toString(), + getGitLabRequestOptions(this.config), + ); + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + return response.json().then(items => { + const nextPage = response.headers.get('x-next-page'); + + return { + items, + nextPage: nextPage ? Number(nextPage) : null, + } as PagedResponse; + }); + } +} + +export type ListOptions = { + [key: string]: string | number | undefined; + per_page?: number | undefined; + page?: number | undefined; +}; + +export type PagedResponse = { + items: T[]; + nextPage?: number; +}; + +export async function* paginated( + request: (options: ListOptions) => Promise>, + options: ListOptions, +) { + let res; + do { + res = await request(options); + options.page = res.nextPage; + for (const item of res.items) { + yield item; + } + } while (res.nextPage); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts new file mode 100644 index 0000000000..a6c6419d21 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitLabClient, paginated } from './client'; +export type { GitLabProject } from "./types"; diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts new file mode 100644 index 0000000000..a66411ddc8 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type GitLabProject = { + id: number; + default_branch: string; + archived: boolean; + last_activity_at: string; + web_url: string; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 6a44ea0a8f..fe4d374829 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -26,6 +26,7 @@ export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; +export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; export { LocationEntityProcessor } from './LocationEntityProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; export type { PlaceholderResolver } from './PlaceholderProcessor'; diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index fb12077508..88fcc27055 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -50,6 +50,7 @@ import { FileReaderProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, + GitLabDiscoveryProcessor, PlaceholderProcessor, PlaceholderResolver, UrlReaderProcessor, @@ -372,6 +373,7 @@ export class NextCatalogBuilder { BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), + GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new AnnotateLocationEntityProcessor({ integrations }), diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 76b4af7450..70c1890fbf 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -46,6 +46,7 @@ import { FileReaderProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, + GitLabDiscoveryProcessor, HigherOrderOperation, HigherOrderOperations, LocationEntityProcessor, @@ -316,6 +317,7 @@ export class CatalogBuilder { BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), + GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new LocationEntityProcessor({ integrations }),