From 6816352500a767c694e2bba965de7e9a66cacdf0 Mon Sep 17 00:00:00 2001 From: Marcus Eide Date: Fri, 5 May 2023 13:51:20 +0200 Subject: [PATCH] Add discover feature to onboard command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Co-authored-by: Philipp Hugenroth Signed-off-by: Marcus Eide --- .changeset/two-mayflies-tell.md | 6 + packages/cli/package.json | 5 + packages/cli/src/commands/onboard/command.ts | 18 +- .../commands/onboard/discovery/Discovery.ts | 71 ++++++++ .../analyzers/BasicRepositoryAnalyzer.ts | 56 ++++++ .../analyzers/DefaultAnalysisOutputs.ts | 29 ++++ .../analyzers/PackageJsonAnalyzer.ts | 124 +++++++++++++ .../onboard/discovery/analyzers/types.ts | 37 ++++ .../src/commands/onboard/discovery/index.ts | 141 +++++++++++++++ .../github/GithubDiscoveryProvider.ts | 163 ++++++++++++++++++ .../discovery/providers/github/GithubFile.ts | 35 ++++ .../providers/github/GithubRepository.ts | 84 +++++++++ .../gitlab/GitlabDiscoveryProvider.ts | 106 ++++++++++++ .../discovery/providers/gitlab/GitlabFile.ts | 38 ++++ .../providers/gitlab/GitlabProject.ts | 89 ++++++++++ .../onboard/discovery/providers/types.ts | 53 ++++++ packages/cli/src/commands/onboard/files.ts | 6 + packages/integration/api-report.md | 26 +++ .../DefaultGitlabCredentialsProvider.ts | 57 ++++++ ...SingleInstanceGitlabCredentialsProvider.ts | 39 +++++ packages/integration/src/gitlab/index.ts | 2 + packages/integration/src/gitlab/types.ts | 30 ++++ yarn.lock | 17 +- 23 files changed, 1229 insertions(+), 3 deletions(-) create mode 100644 .changeset/two-mayflies-tell.md create mode 100644 packages/cli/src/commands/onboard/discovery/Discovery.ts create mode 100644 packages/cli/src/commands/onboard/discovery/analyzers/BasicRepositoryAnalyzer.ts create mode 100644 packages/cli/src/commands/onboard/discovery/analyzers/DefaultAnalysisOutputs.ts create mode 100644 packages/cli/src/commands/onboard/discovery/analyzers/PackageJsonAnalyzer.ts create mode 100644 packages/cli/src/commands/onboard/discovery/analyzers/types.ts create mode 100644 packages/cli/src/commands/onboard/discovery/index.ts create mode 100644 packages/cli/src/commands/onboard/discovery/providers/github/GithubDiscoveryProvider.ts create mode 100644 packages/cli/src/commands/onboard/discovery/providers/github/GithubFile.ts create mode 100644 packages/cli/src/commands/onboard/discovery/providers/github/GithubRepository.ts create mode 100644 packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabDiscoveryProvider.ts create mode 100644 packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabFile.ts create mode 100644 packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabProject.ts create mode 100644 packages/cli/src/commands/onboard/discovery/providers/types.ts create mode 100644 packages/integration/src/gitlab/DefaultGitlabCredentialsProvider.ts create mode 100644 packages/integration/src/gitlab/SingleInstanceGitlabCredentialsProvider.ts create mode 100644 packages/integration/src/gitlab/types.ts diff --git a/.changeset/two-mayflies-tell.md b/.changeset/two-mayflies-tell.md new file mode 100644 index 0000000000..bf1d42b20b --- /dev/null +++ b/.changeset/two-mayflies-tell.md @@ -0,0 +1,6 @@ +--- +'@backstage/integration': patch +'@backstage/cli': patch +--- + +Add discovery feature to the onboard cli command. diff --git a/packages/cli/package.json b/packages/cli/package.json index 28c5ce7030..363cd8b072 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -30,16 +30,20 @@ "backstage-cli": "bin/backstage-cli" }, "dependencies": { + "@backstage/catalog-model": "workspace:^", "@backstage/cli-common": "workspace:^", "@backstage/cli-node": "workspace:^", "@backstage/config": "workspace:^", "@backstage/config-loader": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/eslint-plugin": "workspace:^", + "@backstage/integration": "workspace:^", "@backstage/release-manifests": "workspace:^", "@backstage/types": "workspace:^", "@esbuild-kit/cjs-loader": "^2.4.1", "@manypkg/get-packages": "^1.1.3", + "@octokit/graphql": "^5.0.0", + "@octokit/graphql-schema": "^13.7.0", "@octokit/oauth-app": "^4.2.0", "@octokit/request": "^6.0.0", "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7", @@ -88,6 +92,7 @@ "express": "^4.17.1", "fork-ts-checker-webpack-plugin": "^7.0.0-alpha.8", "fs-extra": "10.1.0", + "git-url-parse": "^13.0.0", "glob": "^7.1.7", "global-agent": "^3.0.0", "handlebars": "^4.7.3", diff --git a/packages/cli/src/commands/onboard/command.ts b/packages/cli/src/commands/onboard/command.ts index 5cf25953ab..0e57fe892a 100644 --- a/packages/cli/src/commands/onboard/command.ts +++ b/packages/cli/src/commands/onboard/command.ts @@ -19,11 +19,13 @@ import inquirer from 'inquirer'; import { Task } from '../../lib/tasks'; import { auth } from './auth'; import { integrations } from './integrations'; +import { discover } from './discovery'; export async function command(): Promise { const answers = await inquirer.prompt<{ shouldSetupAuth: boolean; shouldSetupScaffolder: boolean; + shouldDiscoverEntities: boolean; }>([ { type: 'confirm', @@ -37,9 +39,17 @@ export async function command(): Promise { message: 'Do you want to use Software Templates in this project?', default: true, }, + { + type: 'confirm', + name: 'shouldDiscoverEntities', + message: + 'Do you want to discover entities and add them to the Software Catalog?', + default: true, + }, ]); - const { shouldSetupAuth, shouldSetupScaffolder } = answers; + const { shouldSetupAuth, shouldSetupScaffolder, shouldDiscoverEntities } = + answers; let providerInfo; if (shouldSetupAuth) { @@ -50,7 +60,11 @@ export async function command(): Promise { await integrations(providerInfo); } - if (!shouldSetupAuth && !shouldSetupScaffolder) { + if (shouldDiscoverEntities) { + await discover(providerInfo); + } + + if (!shouldSetupAuth && !shouldSetupScaffolder && !shouldDiscoverEntities) { Task.log( chalk.yellow( 'If you change your mind, feel free to re-run this command.', diff --git a/packages/cli/src/commands/onboard/discovery/Discovery.ts b/packages/cli/src/commands/onboard/discovery/Discovery.ts new file mode 100644 index 0000000000..b0db4255ad --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/Discovery.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import chalk from 'chalk'; +import { Entity } from '@backstage/catalog-model'; +import { Analyzer } from './analyzers/types'; +import { Provider } from './providers/types'; +import { DefaultAnalysisOutputs } from './analyzers/DefaultAnalysisOutputs'; +import { Task } from '../../../lib/tasks'; + +export class Discovery { + readonly #providers: Provider[] = []; + readonly #analyzers: Analyzer[] = []; + + addProvider(provider: Provider) { + this.#providers.push(provider); + } + + addAnalyzer(analyzer: Analyzer) { + this.#analyzers.push(analyzer); + } + + async run(url: string): Promise<{ entities: Entity[] }> { + Task.log(`Running discovery for ${chalk.cyan(url)}`); + const result: Entity[] = []; + + for (const provider of this.#providers) { + const repositories = await provider.discover(url); + if (repositories && repositories.length) { + Task.log( + `Discovered ${chalk.cyan( + repositories.length, + )} repositories for ${chalk.cyan(provider.name())}`, + ); + + for (const repository of repositories) { + await Task.forItem('Analyzing', repository.name, async () => { + const output = new DefaultAnalysisOutputs(); + for (const analyzer of this.#analyzers) { + await analyzer.analyzeRepository({ repository, output }); + } + + output + .list() + .filter(entry => entry.type === 'entity') + .forEach(({ entity }) => result.push(entity)); + }); + } + + Task.log(`Produced ${chalk.cyan(result.length || 'no')} entities`); + } + } + + return { + entities: result, + }; + } +} diff --git a/packages/cli/src/commands/onboard/discovery/analyzers/BasicRepositoryAnalyzer.ts b/packages/cli/src/commands/onboard/discovery/analyzers/BasicRepositoryAnalyzer.ts new file mode 100644 index 0000000000..82c8c2cdae --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/analyzers/BasicRepositoryAnalyzer.ts @@ -0,0 +1,56 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ComponentEntity } from '@backstage/catalog-model'; +import { AnalysisOutputs, Analyzer } from './types'; +import { Repository } from '../providers/types'; + +/** + * Naive analyzer that produces a single entity that represents the repository + * as a whole. + */ +export class BasicRepositoryAnalyzer implements Analyzer { + name(): string { + return BasicRepositoryAnalyzer.name; + } + + async analyzeRepository(options: { + repository: Repository; + output: AnalysisOutputs; + }): Promise { + const entity: ComponentEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: options.repository.name, + ...(options.repository.description + ? { description: options.repository.description } + : {}), + }, + spec: { + type: 'service', + lifecycle: 'production', + owner: 'user:guest', + }, + }; + + options.output.produce({ + type: 'entity', + path: '/', + entity: entity, + }); + } +} diff --git a/packages/cli/src/commands/onboard/discovery/analyzers/DefaultAnalysisOutputs.ts b/packages/cli/src/commands/onboard/discovery/analyzers/DefaultAnalysisOutputs.ts new file mode 100644 index 0000000000..a370b8f166 --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/analyzers/DefaultAnalysisOutputs.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AnalysisOutput, AnalysisOutputs } from './types'; + +export class DefaultAnalysisOutputs implements AnalysisOutputs { + readonly #outputs = new Map(); + + produce(output: AnalysisOutput) { + this.#outputs.set(output.entity.metadata.name, output); + } + + list() { + return Array.from(this.#outputs).map(([_, output]) => output); + } +} diff --git a/packages/cli/src/commands/onboard/discovery/analyzers/PackageJsonAnalyzer.ts b/packages/cli/src/commands/onboard/discovery/analyzers/PackageJsonAnalyzer.ts new file mode 100644 index 0000000000..1f4cb116ad --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/analyzers/PackageJsonAnalyzer.ts @@ -0,0 +1,124 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + ANNOTATION_SOURCE_LOCATION, + ComponentEntity, +} from '@backstage/catalog-model'; +import z from 'zod'; +import { AnalysisOutputs, Analyzer } from './types'; +import { Repository, RepositoryFile } from '../providers/types'; + +export class PackageJsonAnalyzer implements Analyzer { + name(): string { + return PackageJsonAnalyzer.name; + } + + async analyzeRepository(options: { + repository: Repository; + output: AnalysisOutputs; + }): Promise { + const packageJson = await options.repository.file('package.json'); + if (!packageJson) { + return; + } + + const content = await readPackageJson(packageJson); + if (!content) { + return; + } + + const name = sanitizeName(content?.name) ?? options.repository.name; + + const entity: ComponentEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name, + ...(options.repository.description + ? { description: options.repository.description } + : {}), + tags: ['javascript'], + annotations: { + [ANNOTATION_SOURCE_LOCATION]: `url:${options.repository.url}`, + }, + }, + spec: { + type: 'website', + lifecycle: 'production', + owner: 'user:guest', + }, + }; + + const decorate = options.output + .list() + .find(entry => entry.entity.metadata.name === name); + + if (decorate) { + decorate.entity.spec = { + ...decorate.entity.spec, + type: 'website', + }; + + decorate.entity.metadata.tags = [ + ...(decorate.entity.metadata.tags ?? []), + 'javascript', + ]; + + decorate.entity.metadata.annotations = { + ...decorate.entity.metadata.annotations, + [ANNOTATION_SOURCE_LOCATION]: `url:${options.repository.url}`, + }; + + return; + } + + options.output.produce({ + type: 'entity', + path: '/', + entity, + }); + } +} + +const packageSchema = z.object({ + name: z.string().optional(), +}); + +/** + * Makes sure that a name retrieved from a package.json file + * is reasonable and conforms to the catalog naming format. + */ +function sanitizeName(name?: string) { + return name && name !== 'root' + ? name.replace(/[^a-z0-9A-Z]/g, '_').substring(0, 62) + : undefined; +} + +async function readPackageJson( + file: RepositoryFile, +): Promise | undefined> { + try { + const text = await file.text(); + const result = packageSchema.safeParse(JSON.parse(text)); + if (!result.success) { + return undefined; + } + return { name: result.data.name }; + } catch (e) { + return undefined; + } +} diff --git a/packages/cli/src/commands/onboard/discovery/analyzers/types.ts b/packages/cli/src/commands/onboard/discovery/analyzers/types.ts new file mode 100644 index 0000000000..57b6f2152b --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/analyzers/types.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { Repository } from '../providers/types'; + +export type AnalysisOutput = { + type: 'entity'; + path: string; + entity: Entity; +}; + +export interface AnalysisOutputs { + produce(output: AnalysisOutput): void; + list(): AnalysisOutput[]; +} + +export interface Analyzer { + name(): string; + analyzeRepository(options: { + repository: Repository; + output: AnalysisOutputs; + }): Promise; +} diff --git a/packages/cli/src/commands/onboard/discovery/index.ts b/packages/cli/src/commands/onboard/discovery/index.ts new file mode 100644 index 0000000000..b2b8bddb50 --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/index.ts @@ -0,0 +1,141 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import yaml from 'yaml'; +import inquirer from 'inquirer'; +import chalk from 'chalk'; +import { loadCliConfig } from '../../../lib/config'; +import { updateConfigFile } from '../config'; +import { APP_CONFIG_FILE, DISCOVERED_ENTITIES_FILE } from '../files'; +import { Discovery } from './Discovery'; +import { BasicRepositoryAnalyzer } from './analyzers/BasicRepositoryAnalyzer'; +import { PackageJsonAnalyzer } from './analyzers/PackageJsonAnalyzer'; +import { GithubDiscoveryProvider } from './providers/github/GithubDiscoveryProvider'; +import { GitlabDiscoveryProvider } from './providers/gitlab/GitlabDiscoveryProvider'; +import { GitHubAnswers, GitLabAnswers } from '../auth'; +import { Task } from '../../../lib/tasks'; + +export async function discover(providerInfo?: { + provider: string; + answers: GitHubAnswers | GitLabAnswers; +}) { + Task.log(` + Would you like to scan for - and create - Software Catalog entities? + + You will need to select which SCM (Source Code Management) provider you are using, + and then which repository or organization you want to scan. + + This will generate a new file in the root of your project containing discovered entities, + which will be included in the Software Catalog when you start up Backstage next time. + + Note that this command requires an access token, which can be either added through the integration config or + provided as an environment variable. + `); + + const answers = await inquirer.prompt<{ + shouldContinue: boolean; + provider: string; + url: string; + }>([ + { + type: 'confirm', + name: 'shouldContinue', + message: 'Do you want to continue?', + }, + { + type: 'list', + name: 'provider', + message: 'Please select which SCM provider you want to use:', + choices: ['GitHub', 'GitLab'], + default: providerInfo?.provider, + when: ({ shouldContinue }) => shouldContinue, + }, + { + type: 'input', + name: 'url', + message: `Which repository do you want to scan?`, + when: ({ shouldContinue }) => shouldContinue, + filter: (input, { provider }) => { + if (provider === 'GitLab') { + return `https://gitlab.com/${input}`; + } + if (provider === 'GitHub') { + return `https://github.com/${input}`; + } + return false; + }, + }, + ]); + + if (!answers.shouldContinue) { + Task.log( + chalk.yellow( + 'If you change your mind, feel free to re-run this command.', + ), + ); + return; + } + + const { fullConfig: config } = await loadCliConfig({ args: [] }); + + const discovery = new Discovery(); + + if (answers.provider === 'GitHub') { + discovery.addProvider(GithubDiscoveryProvider.fromConfig(config)); + } + if (answers.provider === 'GitLab') { + discovery.addProvider(GitlabDiscoveryProvider.fromConfig(config)); + } + + discovery.addAnalyzer(new BasicRepositoryAnalyzer()); + discovery.addAnalyzer(new PackageJsonAnalyzer()); + + const { entities } = await discovery.run(answers.url); + + if (!entities.length) { + Task.log( + chalk.yellow(` + We could not find enough information to be able to generate any Software Catalog entities for you. + Perhaps you can try again with a different repository?`), + ); + return; + } + + await Task.forItem('Creating', DISCOVERED_ENTITIES_FILE, async () => { + const payload: string[] = []; + for (const entity of entities) { + payload.push('---\n', yaml.stringify(entity)); + } + await fs.writeFile(DISCOVERED_ENTITIES_FILE, payload.join('')); + }); + + await Task.forItem( + 'Updating', + APP_CONFIG_FILE, + async () => + await updateConfigFile(APP_CONFIG_FILE, { + catalog: { + locations: [ + { + type: 'file', + target: DISCOVERED_ENTITIES_FILE, + }, + ], + }, + }), + ); +} diff --git a/packages/cli/src/commands/onboard/discovery/providers/github/GithubDiscoveryProvider.ts b/packages/cli/src/commands/onboard/discovery/providers/github/GithubDiscoveryProvider.ts new file mode 100644 index 0000000000..d5a3dc4a77 --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/providers/github/GithubDiscoveryProvider.ts @@ -0,0 +1,163 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { + DefaultGithubCredentialsProvider, + GithubCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; +import { graphql } from '@octokit/graphql'; +import { + Repository as GraphqlRepository, + Query as GraphqlQuery, +} from '@octokit/graphql-schema'; +import parseGitUrl from 'git-url-parse'; +import { Provider, Repository } from '../types'; +import { GithubRepository } from './GithubRepository'; + +export class GithubDiscoveryProvider implements Provider { + readonly #envToken: string | undefined; + readonly #scmIntegrations: ScmIntegrations; + readonly #credentialsProvider: GithubCredentialsProvider; + + static fromConfig(config: Config): GithubDiscoveryProvider { + const envToken = process.env.GITHUB_TOKEN || undefined; + const scmIntegrations = ScmIntegrations.fromConfig(config); + const credentialsProvider = + DefaultGithubCredentialsProvider.fromIntegrations(scmIntegrations); + return new GithubDiscoveryProvider( + envToken, + scmIntegrations, + credentialsProvider, + ); + } + + private constructor( + envToken: string | undefined, + integrations: ScmIntegrations, + credentialsProvider: GithubCredentialsProvider, + ) { + this.#envToken = envToken; + this.#scmIntegrations = integrations; + this.#credentialsProvider = credentialsProvider; + } + + name(): string { + return 'GitHub'; + } + + async discover(url: string): Promise { + if (!url.startsWith('https://github.com/')) { + return false; + } + + const scmIntegration = this.#scmIntegrations.github.byUrl(url); + if (!scmIntegration) { + throw new Error(`No GitHub integration found for ${url}`); + } + + const parsed = parseGitUrl(url); + const { name, organization } = parsed; + const org = organization || name; // depends on if it's a repo url or an org url... + + const client = graphql.defaults({ + baseUrl: scmIntegration.config.apiBaseUrl, + headers: await this.#getRequestHeaders(url), + }); + + const { repositories } = await this.#getOrganizationRepositories( + client, + org, + ); + + return repositories + .filter(repo => repo.url.startsWith(url)) + .map(repo => new GithubRepository(client, repo, org)); + } + + async #getRequestHeaders(url: string): Promise> { + const credentials = await this.#credentialsProvider.getCredentials({ + url, + }); + + if (credentials.headers) { + return credentials.headers; + } else if (credentials.token) { + return { authorization: `token ${credentials.token}` }; + } + + if (this.#envToken) { + return { authorization: `token ${this.#envToken}` }; + } + + throw new Error( + 'No token available for GitHub, please configure your integrations or set a GITHUB_TOKEN env variable', + ); + } + + async #getOrganizationRepositories(client: typeof graphql, org: string) { + const query = `query repositories($org: String!, $cursor: String) { + repositoryOwner(login: $org) { + login + repositories(first: 100, after: $cursor) { + nodes { + name + url + description + isArchived + isFork + } + pageInfo { + hasNextPage + endCursor + } + } + } + }`; + + const result: GraphqlRepository[] = []; + + let cursor: string | undefined | null = undefined; + let hasNextPage = true; + + while (hasNextPage) { + const response: GraphqlQuery = await client(query, { + org, + cursor, + }); + + const { repositories: connection } = response.repositoryOwner ?? {}; + + if (!connection) { + throw new Error(`Found no repositories for ${org}`); + } + + for (const repository of connection.nodes ?? []) { + if (repository && !repository.isArchived && !repository.isFork) { + result.push(repository); + } + } + + cursor = connection.pageInfo.endCursor; + hasNextPage = connection.pageInfo.hasNextPage; + } + + return { + repositories: result, + }; + } +} diff --git a/packages/cli/src/commands/onboard/discovery/providers/github/GithubFile.ts b/packages/cli/src/commands/onboard/discovery/providers/github/GithubFile.ts new file mode 100644 index 0000000000..25279b0949 --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/providers/github/GithubFile.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { RepositoryFile } from '../types'; + +export class GithubFile implements RepositoryFile { + readonly #path: string; + readonly #content: string; + + constructor(path: string, content: string) { + this.#path = path; + this.#content = content; + } + + get path(): string { + return this.#path; + } + + async text(): Promise { + return this.#content; + } +} diff --git a/packages/cli/src/commands/onboard/discovery/providers/github/GithubRepository.ts b/packages/cli/src/commands/onboard/discovery/providers/github/GithubRepository.ts new file mode 100644 index 0000000000..f3c68851a0 --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/providers/github/GithubRepository.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { graphql } from '@octokit/graphql'; +import { + Repository as GraphqlRepository, + Blob as GraphqlBlob, +} from '@octokit/graphql-schema'; +import { Repository, RepositoryFile } from '../types'; +import { GithubFile } from './GithubFile'; + +export class GithubRepository implements Repository { + readonly #client: typeof graphql; + readonly #repo: GraphqlRepository; + readonly #org: string; + + constructor(client: typeof graphql, repo: GraphqlRepository, org: string) { + this.#client = client; + this.#repo = repo; + this.#org = org; + } + + get url(): string { + return this.#repo.url; + } + + get name(): string { + return this.#repo.name; + } + + get owner(): string { + return this.#org; + } + + get description(): string | undefined { + return this.#repo.description ?? undefined; + } + + async file(filename: string): Promise { + const content = await this.#getFileContent(filename); + if (!content || content.isBinary || !content.text) { + return undefined; + } + + return new GithubFile(filename, content.text ?? ''); + } + + async #getFileContent(filename: string) { + const query = `query RepoFiles($owner: String!, $name: String!, $expr: String!) { + repository(owner: $owner, name: $name) { + object(expression: $expr) { + ...on Blob { + text + isBinary + } + } + } + }`; + + const response = await this.#client<{ repository: GraphqlRepository }>( + query, + { + name: this.#repo.name, + owner: this.#org, + expr: `HEAD:${filename}`, + }, + ); + + return response.repository.object as GraphqlBlob; + } +} diff --git a/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabDiscoveryProvider.ts b/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabDiscoveryProvider.ts new file mode 100644 index 0000000000..df9ffa4206 --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabDiscoveryProvider.ts @@ -0,0 +1,106 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { + DefaultGitlabCredentialsProvider, + GitlabCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; +import fetch from 'node-fetch'; +import { Provider } from '../types'; +import { GitlabProject, ProjectResponse } from './GitlabProject'; + +export class GitlabDiscoveryProvider implements Provider { + readonly #envToken: string | undefined; + readonly #scmIntegrations: ScmIntegrations; + readonly #credentialsProvider: GitlabCredentialsProvider; + + static fromConfig(config: Config): GitlabDiscoveryProvider { + const envToken = process.env.GITLAB_TOKEN || undefined; + const scmIntegrations = ScmIntegrations.fromConfig(config); + const credentialsProvider = + DefaultGitlabCredentialsProvider.fromIntegrations(scmIntegrations); + + return new GitlabDiscoveryProvider( + envToken, + scmIntegrations, + credentialsProvider, + ); + } + + private constructor( + envToken: string | undefined, + integrations: ScmIntegrations, + credentialsProvider: GitlabCredentialsProvider, + ) { + this.#envToken = envToken; + this.#scmIntegrations = integrations; + this.#credentialsProvider = credentialsProvider; + } + + name(): string { + return 'GitLab'; + } + + async discover(url: string): Promise { + const { origin, pathname } = new URL(url); + const [, user] = pathname.split('/'); + + const scmIntegration = this.#scmIntegrations.gitlab.byUrl(origin); + if (!scmIntegration) { + throw new Error(`No GitLab integration found for ${origin}`); + } + + const headers = await this.#getRequestHeaders(origin); + + const response = await fetch( + `${scmIntegration.config.apiBaseUrl}/users/${user}/projects`, + { headers }, + ); + + if (!response.ok) { + throw new Error(`${response.status} ${response.statusText}`); + } + + const projects: ProjectResponse[] = await response.json(); + + return projects.map( + project => + new GitlabProject(project, scmIntegration.config.apiBaseUrl, headers), + ); + } + + async #getRequestHeaders(url: string): Promise> { + const credentials = await this.#credentialsProvider.getCredentials({ + url, + }); + + if (credentials.headers) { + return credentials.headers; + } else if (credentials.token) { + return { authorization: `Bearer ${credentials.token}` }; + } + + if (this.#envToken) { + return { authorization: `Bearer ${this.#envToken}` }; + } + + throw new Error( + 'No token available for GitLab, please set a GITLAB_TOKEN env variable', + ); + } +} diff --git a/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabFile.ts b/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabFile.ts new file mode 100644 index 0000000000..06f3189ba6 --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabFile.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { RepositoryFile } from '../types'; + +/** + * A single file in a GitLab repository. + */ +export class GitlabFile implements RepositoryFile { + readonly #path: string; + readonly #content: string; + + constructor(path: string, content: string) { + this.#path = path; + this.#content = content; + } + + get path(): string { + return this.#path; + } + + async text(): Promise { + return this.#content; + } +} diff --git a/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabProject.ts b/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabProject.ts new file mode 100644 index 0000000000..258e8176bb --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabProject.ts @@ -0,0 +1,89 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'node-fetch'; +import { GitlabFile } from './GitlabFile'; +import { Repository, RepositoryFile } from '../types'; + +export type ProjectResponse = { + id: string; + name: string; + description: string; + owner: { + username: string; + }; + web_url: string; +}; + +type BranchResponse = { + default: boolean; + name: string; +}; + +type FileContentResponse = { + content: string; +}; + +export class GitlabProject implements Repository { + constructor( + private readonly project: ProjectResponse, + private readonly apiBaseUrl: string, + private readonly headers: { [name: string]: string }, + ) {} + + get url(): string { + return this.project.web_url; + } + + get name(): string { + return this.project.name; + } + + get owner(): string { + return this.project.owner.username; + } + + get description(): string { + return this.project.description; + } + + async file(filename: string): Promise { + const mainBranch = await this.#getMainBranch(); + const content = await this.#getFileContent(filename, mainBranch); + + return new GitlabFile(filename, content); + } + + async #getFileContent(path: string, mainBranch: string): Promise { + const response = await fetch( + `${this.apiBaseUrl}/projects/${this.project.id}/repository/files/${path}?ref=${mainBranch}`, + { headers: this.headers }, + ); + const { content }: FileContentResponse = await response.json(); + + return Buffer.from(content, 'base64').toString('ascii'); + } + + async #getMainBranch(): Promise { + const response = await fetch( + `${this.apiBaseUrl}/projects/${this.project.id}/repository/branches`, + { headers: this.headers }, + ); + const branches: BranchResponse[] = await response.json(); + + return branches.find(branch => branch.default)?.name ?? 'main'; + } +} diff --git a/packages/cli/src/commands/onboard/discovery/providers/types.ts b/packages/cli/src/commands/onboard/discovery/providers/types.ts new file mode 100644 index 0000000000..2d8ce70ecf --- /dev/null +++ b/packages/cli/src/commands/onboard/discovery/providers/types.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Abstraction for a single repository. + */ +export interface Repository { + url: string; + + name: string; + + owner: string; + + description?: string; + + file(filename: string): Promise; +} + +/** + * Abstraction for a single repository file. + */ +export interface RepositoryFile { + /** + * The filepath of the data. + */ + path: string; + + /** + * The textual contents of the file. + */ + text(): Promise; +} + +/** + * One integration that supports discovery of repositories. + */ +export interface Provider { + name(): string; + discover(url: string): Promise; +} diff --git a/packages/cli/src/commands/onboard/files.ts b/packages/cli/src/commands/onboard/files.ts index 17f8a0f85e..febec35d4a 100644 --- a/packages/cli/src/commands/onboard/files.ts +++ b/packages/cli/src/commands/onboard/files.ts @@ -19,7 +19,13 @@ import * as path from 'path'; /* eslint-disable-next-line no-restricted-syntax */ const { targetRoot, ownDir } = findPaths(__dirname); + export const APP_CONFIG_FILE = path.join(targetRoot, 'app-config.local.yaml'); +export const DISCOVERED_ENTITIES_FILE = path.join( + targetRoot, + 'examples', + 'discovered-entities.yaml', +); export const PATCH_FOLDER = path.join( ownDir, 'src', diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index d45c8ca06b..a1a14689ca 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -165,6 +165,18 @@ export class DefaultGithubCredentialsProvider getCredentials(opts: { url: string }): Promise; } +// @public +export class DefaultGitlabCredentialsProvider + implements GitlabCredentialsProvider +{ + // (undocumented) + static fromIntegrations( + integrations: ScmIntegrationRegistry, + ): DefaultGitlabCredentialsProvider; + // (undocumented) + getCredentials(opts: { url: string }): Promise; +} + // @public export function defaultScmResolveUrl(options: { url: string; @@ -478,6 +490,20 @@ export type GithubIntegrationConfig = { apps?: GithubAppConfig[]; }; +// @public (undocumented) +export type GitlabCredentials = { + headers?: { + [name: string]: string; + }; + token?: string; +}; + +// @public (undocumented) +export interface GitlabCredentialsProvider { + // (undocumented) + getCredentials(opts?: { url: string }): Promise; +} + // @public export class GitLabIntegration implements ScmIntegration { constructor(integrationConfig: GitLabIntegrationConfig); diff --git a/packages/integration/src/gitlab/DefaultGitlabCredentialsProvider.ts b/packages/integration/src/gitlab/DefaultGitlabCredentialsProvider.ts new file mode 100644 index 0000000000..71e9a1e2b7 --- /dev/null +++ b/packages/integration/src/gitlab/DefaultGitlabCredentialsProvider.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ScmIntegrationRegistry } from '../registry'; +import { SingleInstanceGitlabCredentialsProvider } from './SingleInstanceGitlabCredentialsProvider'; +import { GitlabCredentials, GitlabCredentialsProvider } from './types'; + +/** + * Handles the creation and caching of credentials for GitLab integrations. + * + * @public + */ +export class DefaultGitlabCredentialsProvider + implements GitlabCredentialsProvider +{ + static fromIntegrations(integrations: ScmIntegrationRegistry) { + const credentialsProviders: Map = + new Map(); + + integrations.gitlab.list().forEach(integration => { + const credentialsProvider = + SingleInstanceGitlabCredentialsProvider.create(integration.config); + credentialsProviders.set(integration.config.host, credentialsProvider); + }); + return new DefaultGitlabCredentialsProvider(credentialsProviders); + } + + private constructor( + private readonly providers: Map, + ) {} + + async getCredentials(opts: { url: string }): Promise { + const parsed = new URL(opts.url); + const provider = this.providers.get(parsed.host); + + if (!provider) { + throw new Error( + `There is no GitLab integration that matches ${opts.url}. Please add a configuration for an integration.`, + ); + } + + return provider.getCredentials(opts); + } +} diff --git a/packages/integration/src/gitlab/SingleInstanceGitlabCredentialsProvider.ts b/packages/integration/src/gitlab/SingleInstanceGitlabCredentialsProvider.ts new file mode 100644 index 0000000000..39f22ab9d5 --- /dev/null +++ b/packages/integration/src/gitlab/SingleInstanceGitlabCredentialsProvider.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GitLabIntegrationConfig } from './config'; +import { GitlabCredentials, GitlabCredentialsProvider } from './types'; + +export class SingleInstanceGitlabCredentialsProvider + implements GitlabCredentialsProvider +{ + static create: ( + config: GitLabIntegrationConfig, + ) => GitlabCredentialsProvider = config => { + return new SingleInstanceGitlabCredentialsProvider(config.token); + }; + + constructor(private readonly token?: string) {} + + async getCredentials(): Promise { + return { + headers: this.token + ? { Authorization: `Bearer ${this.token}` } + : undefined, + token: this.token, + }; + } +} diff --git a/packages/integration/src/gitlab/index.ts b/packages/integration/src/gitlab/index.ts index 1a5063c51c..82ba460dab 100644 --- a/packages/integration/src/gitlab/index.ts +++ b/packages/integration/src/gitlab/index.ts @@ -22,3 +22,5 @@ export { export type { GitLabIntegrationConfig } from './config'; export { getGitLabFileFetchUrl, getGitLabRequestOptions } from './core'; export { GitLabIntegration, replaceGitLabUrlType } from './GitLabIntegration'; +export { DefaultGitlabCredentialsProvider } from './DefaultGitlabCredentialsProvider'; +export type { GitlabCredentials, GitlabCredentialsProvider } from './types'; diff --git a/packages/integration/src/gitlab/types.ts b/packages/integration/src/gitlab/types.ts new file mode 100644 index 0000000000..ef8ccf8d6c --- /dev/null +++ b/packages/integration/src/gitlab/types.ts @@ -0,0 +1,30 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @public + */ +export type GitlabCredentials = { + headers?: { [name: string]: string }; + token?: string; +}; + +/** + * @public + */ +export interface GitlabCredentialsProvider { + getCredentials(opts?: { url: string }): Promise; +} diff --git a/yarn.lock b/yarn.lock index f80cac5e4f..e2b21e2d10 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3848,6 +3848,7 @@ __metadata: resolution: "@backstage/cli@workspace:packages/cli" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/catalog-model": "workspace:^" "@backstage/cli-common": "workspace:^" "@backstage/cli-node": "workspace:^" "@backstage/config": "workspace:^" @@ -3858,12 +3859,15 @@ __metadata: "@backstage/dev-utils": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/eslint-plugin": "workspace:^" + "@backstage/integration": "workspace:^" "@backstage/release-manifests": "workspace:^" "@backstage/test-utils": "workspace:^" "@backstage/theme": "workspace:^" "@backstage/types": "workspace:^" "@esbuild-kit/cjs-loader": ^2.4.1 "@manypkg/get-packages": ^1.1.3 + "@octokit/graphql": ^5.0.0 + "@octokit/graphql-schema": ^13.7.0 "@octokit/oauth-app": ^4.2.0 "@octokit/request": ^6.0.0 "@pmmmwh/react-refresh-webpack-plugin": ^0.5.7 @@ -3930,6 +3934,7 @@ __metadata: express: ^4.17.1 fork-ts-checker-webpack-plugin: ^7.0.0-alpha.8 fs-extra: 10.1.0 + git-url-parse: ^13.0.0 glob: ^7.1.7 global-agent: ^3.0.0 handlebars: ^4.7.3 @@ -13130,6 +13135,16 @@ __metadata: languageName: node linkType: hard +"@octokit/graphql-schema@npm:^13.7.0": + version: 13.10.0 + resolution: "@octokit/graphql-schema@npm:13.10.0" + dependencies: + graphql: ^16.0.0 + graphql-tag: ^2.10.3 + checksum: fdec9c9a4df1f90b733ea0e24964744faceaf65e5d350b1727892e8e0e5821df1d29aec5cfa039925a044c6f56d4ed2028505108db7fbc0c68011053853c2411 + languageName: node + linkType: hard + "@octokit/graphql@npm:^5.0.0": version: 5.0.6 resolution: "@octokit/graphql@npm:5.0.6" @@ -25543,7 +25558,7 @@ __metadata: languageName: node linkType: hard -"graphql-tag@npm:^2.11.0, graphql-tag@npm:^2.12.6": +"graphql-tag@npm:^2.10.3, graphql-tag@npm:^2.11.0, graphql-tag@npm:^2.12.6": version: 2.12.6 resolution: "graphql-tag@npm:2.12.6" dependencies: