Add discover feature to onboard command

Co-authored-by: Fredrik Adelöw <freben@gmail.com>
Co-authored-by: Johan Haals <johan.haals@gmail.com>
Co-authored-by: Philipp Hugenroth <philipph@spotify.com>
Signed-off-by: Marcus Eide <eide@spotify.com>

.changeset/two-mayflies-tell.md

@@ -0,0 +1,6 @@
+---
+'@backstage/integration': patch
+'@backstage/cli': patch
+---
+
+Add discovery feature to the onboard cli command.

packages/cli/package.json

@@ -30,16 +30,20 @@
     "backstage-cli": "bin/backstage-cli"
   },
   "dependencies": {
+    "@backstage/catalog-model": "workspace:^",
     "@backstage/cli-common": "workspace:^",
     "@backstage/cli-node": "workspace:^",
     "@backstage/config": "workspace:^",
     "@backstage/config-loader": "workspace:^",
     "@backstage/errors": "workspace:^",
     "@backstage/eslint-plugin": "workspace:^",
+    "@backstage/integration": "workspace:^",
     "@backstage/release-manifests": "workspace:^",
     "@backstage/types": "workspace:^",
     "@esbuild-kit/cjs-loader": "^2.4.1",
     "@manypkg/get-packages": "^1.1.3",
+    "@octokit/graphql": "^5.0.0",
+    "@octokit/graphql-schema": "^13.7.0",
     "@octokit/oauth-app": "^4.2.0",
     "@octokit/request": "^6.0.0",
     "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7",
@@ -88,6 +92,7 @@
     "express": "^4.17.1",
     "fork-ts-checker-webpack-plugin": "^7.0.0-alpha.8",
     "fs-extra": "10.1.0",
+    "git-url-parse": "^13.0.0",
     "glob": "^7.1.7",
     "global-agent": "^3.0.0",
     "handlebars": "^4.7.3",

packages/cli/src/commands/onboard/command.ts

@@ -19,11 +19,13 @@ import inquirer from 'inquirer';
 import { Task } from '../../lib/tasks';
 import { auth } from './auth';
 import { integrations } from './integrations';
+import { discover } from './discovery';
 
 export async function command(): Promise<void> {
   const answers = await inquirer.prompt<{
     shouldSetupAuth: boolean;
     shouldSetupScaffolder: boolean;
+    shouldDiscoverEntities: boolean;
   }>([
     {
       type: 'confirm',
@@ -37,9 +39,17 @@ export async function command(): Promise<void> {
       message: 'Do you want to use Software Templates in this project?',
       default: true,
     },
+    {
+      type: 'confirm',
+      name: 'shouldDiscoverEntities',
+      message:
+        'Do you want to discover entities and add them to the Software Catalog?',
+      default: true,
+    },
   ]);
 
-  const { shouldSetupAuth, shouldSetupScaffolder } = answers;
+  const { shouldSetupAuth, shouldSetupScaffolder, shouldDiscoverEntities } =
+    answers;
 
   let providerInfo;
   if (shouldSetupAuth) {
@@ -50,7 +60,11 @@ export async function command(): Promise<void> {
     await integrations(providerInfo);
   }
 
-  if (!shouldSetupAuth && !shouldSetupScaffolder) {
+  if (shouldDiscoverEntities) {
+    await discover(providerInfo);
+  }
+
+  if (!shouldSetupAuth && !shouldSetupScaffolder && !shouldDiscoverEntities) {
     Task.log(
       chalk.yellow(
         'If you change your mind, feel free to re-run this command.',

packages/cli/src/commands/onboard/discovery/Discovery.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import chalk from 'chalk';
+import { Entity } from '@backstage/catalog-model';
+import { Analyzer } from './analyzers/types';
+import { Provider } from './providers/types';
+import { DefaultAnalysisOutputs } from './analyzers/DefaultAnalysisOutputs';
+import { Task } from '../../../lib/tasks';
+
+export class Discovery {
+  readonly #providers: Provider[] = [];
+  readonly #analyzers: Analyzer[] = [];
+
+  addProvider(provider: Provider) {
+    this.#providers.push(provider);
+  }
+
+  addAnalyzer(analyzer: Analyzer) {
+    this.#analyzers.push(analyzer);
+  }
+
+  async run(url: string): Promise<{ entities: Entity[] }> {
+    Task.log(`Running discovery for ${chalk.cyan(url)}`);
+    const result: Entity[] = [];
+
+    for (const provider of this.#providers) {
+      const repositories = await provider.discover(url);
+      if (repositories && repositories.length) {
+        Task.log(
+          `Discovered ${chalk.cyan(
+            repositories.length,
+          )} repositories for ${chalk.cyan(provider.name())}`,
+        );
+
+        for (const repository of repositories) {
+          await Task.forItem('Analyzing', repository.name, async () => {
+            const output = new DefaultAnalysisOutputs();
+            for (const analyzer of this.#analyzers) {
+              await analyzer.analyzeRepository({ repository, output });
+            }
+
+            output
+              .list()
+              .filter(entry => entry.type === 'entity')
+              .forEach(({ entity }) => result.push(entity));
+          });
+        }
+
+        Task.log(`Produced ${chalk.cyan(result.length || 'no')} entities`);
+      }
+    }
+
+    return {
+      entities: result,
+    };
+  }
+}

packages/cli/src/commands/onboard/discovery/analyzers/BasicRepositoryAnalyzer.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ComponentEntity } from '@backstage/catalog-model';
+import { AnalysisOutputs, Analyzer } from './types';
+import { Repository } from '../providers/types';
+
+/**
+ * Naive analyzer that produces a single entity that represents the repository
+ * as a whole.
+ */
+export class BasicRepositoryAnalyzer implements Analyzer {
+  name(): string {
+    return BasicRepositoryAnalyzer.name;
+  }
+
+  async analyzeRepository(options: {
+    repository: Repository;
+    output: AnalysisOutputs;
+  }): Promise<void> {
+    const entity: ComponentEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      metadata: {
+        name: options.repository.name,
+        ...(options.repository.description
+          ? { description: options.repository.description }
+          : {}),
+      },
+      spec: {
+        type: 'service',
+        lifecycle: 'production',
+        owner: 'user:guest',
+      },
+    };
+
+    options.output.produce({
+      type: 'entity',
+      path: '/',
+      entity: entity,
+    });
+  }
+}

packages/cli/src/commands/onboard/discovery/analyzers/DefaultAnalysisOutputs.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { AnalysisOutput, AnalysisOutputs } from './types';
+
+export class DefaultAnalysisOutputs implements AnalysisOutputs {
+  readonly #outputs = new Map<string, AnalysisOutput>();
+
+  produce(output: AnalysisOutput) {
+    this.#outputs.set(output.entity.metadata.name, output);
+  }
+
+  list() {
+    return Array.from(this.#outputs).map(([_, output]) => output);
+  }
+}

packages/cli/src/commands/onboard/discovery/analyzers/PackageJsonAnalyzer.ts

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  ANNOTATION_SOURCE_LOCATION,
+  ComponentEntity,
+} from '@backstage/catalog-model';
+import z from 'zod';
+import { AnalysisOutputs, Analyzer } from './types';
+import { Repository, RepositoryFile } from '../providers/types';
+
+export class PackageJsonAnalyzer implements Analyzer {
+  name(): string {
+    return PackageJsonAnalyzer.name;
+  }
+
+  async analyzeRepository(options: {
+    repository: Repository;
+    output: AnalysisOutputs;
+  }): Promise<void> {
+    const packageJson = await options.repository.file('package.json');
+    if (!packageJson) {
+      return;
+    }
+
+    const content = await readPackageJson(packageJson);
+    if (!content) {
+      return;
+    }
+
+    const name = sanitizeName(content?.name) ?? options.repository.name;
+
+    const entity: ComponentEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      metadata: {
+        name,
+        ...(options.repository.description
+          ? { description: options.repository.description }
+          : {}),
+        tags: ['javascript'],
+        annotations: {
+          [ANNOTATION_SOURCE_LOCATION]: `url:${options.repository.url}`,
+        },
+      },
+      spec: {
+        type: 'website',
+        lifecycle: 'production',
+        owner: 'user:guest',
+      },
+    };
+
+    const decorate = options.output
+      .list()
+      .find(entry => entry.entity.metadata.name === name);
+
+    if (decorate) {
+      decorate.entity.spec = {
+        ...decorate.entity.spec,
+        type: 'website',
+      };
+
+      decorate.entity.metadata.tags = [
+        ...(decorate.entity.metadata.tags ?? []),
+        'javascript',
+      ];
+
+      decorate.entity.metadata.annotations = {
+        ...decorate.entity.metadata.annotations,
+        [ANNOTATION_SOURCE_LOCATION]: `url:${options.repository.url}`,
+      };
+
+      return;
+    }
+
+    options.output.produce({
+      type: 'entity',
+      path: '/',
+      entity,
+    });
+  }
+}
+
+const packageSchema = z.object({
+  name: z.string().optional(),
+});
+
+/**
+ * Makes sure that a name retrieved from a package.json file
+ * is reasonable and conforms to the catalog naming format.
+ */
+function sanitizeName(name?: string) {
+  return name && name !== 'root'
+    ? name.replace(/[^a-z0-9A-Z]/g, '_').substring(0, 62)
+    : undefined;
+}
+
+async function readPackageJson(
+  file: RepositoryFile,
+): Promise<z.infer<typeof packageSchema> | undefined> {
+  try {
+    const text = await file.text();
+    const result = packageSchema.safeParse(JSON.parse(text));
+    if (!result.success) {
+      return undefined;
+    }
+    return { name: result.data.name };
+  } catch (e) {
+    return undefined;
+  }
+}

packages/cli/src/commands/onboard/discovery/analyzers/types.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Entity } from '@backstage/catalog-model';
+import { Repository } from '../providers/types';
+
+export type AnalysisOutput = {
+  type: 'entity';
+  path: string;
+  entity: Entity;
+};
+
+export interface AnalysisOutputs {
+  produce(output: AnalysisOutput): void;
+  list(): AnalysisOutput[];
+}
+
+export interface Analyzer {
+  name(): string;
+  analyzeRepository(options: {
+    repository: Repository;
+    output: AnalysisOutputs;
+  }): Promise<void>;
+}

packages/cli/src/commands/onboard/discovery/index.ts

@@ -0,0 +1,141 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import yaml from 'yaml';
+import inquirer from 'inquirer';
+import chalk from 'chalk';
+import { loadCliConfig } from '../../../lib/config';
+import { updateConfigFile } from '../config';
+import { APP_CONFIG_FILE, DISCOVERED_ENTITIES_FILE } from '../files';
+import { Discovery } from './Discovery';
+import { BasicRepositoryAnalyzer } from './analyzers/BasicRepositoryAnalyzer';
+import { PackageJsonAnalyzer } from './analyzers/PackageJsonAnalyzer';
+import { GithubDiscoveryProvider } from './providers/github/GithubDiscoveryProvider';
+import { GitlabDiscoveryProvider } from './providers/gitlab/GitlabDiscoveryProvider';
+import { GitHubAnswers, GitLabAnswers } from '../auth';
+import { Task } from '../../../lib/tasks';
+
+export async function discover(providerInfo?: {
+  provider: string;
+  answers: GitHubAnswers | GitLabAnswers;
+}) {
+  Task.log(`
+    Would you like to scan for - and create - Software Catalog entities?
+
+    You will need to select which SCM (Source Code Management) provider you are using, 
+    and then which repository or organization you want to scan.
+
+    This will generate a new file in the root of your project containing discovered entities,
+    which will be included in the Software Catalog when you start up Backstage next time.
+
+    Note that this command requires an access token, which can be either added through the integration config or
+    provided as an environment variable.
+  `);
+
+  const answers = await inquirer.prompt<{
+    shouldContinue: boolean;
+    provider: string;
+    url: string;
+  }>([
+    {
+      type: 'confirm',
+      name: 'shouldContinue',
+      message: 'Do you want to continue?',
+    },
+    {
+      type: 'list',
+      name: 'provider',
+      message: 'Please select which SCM provider you want to use:',
+      choices: ['GitHub', 'GitLab'],
+      default: providerInfo?.provider,
+      when: ({ shouldContinue }) => shouldContinue,
+    },
+    {
+      type: 'input',
+      name: 'url',
+      message: `Which repository do you want to scan?`,
+      when: ({ shouldContinue }) => shouldContinue,
+      filter: (input, { provider }) => {
+        if (provider === 'GitLab') {
+          return `https://gitlab.com/${input}`;
+        }
+        if (provider === 'GitHub') {
+          return `https://github.com/${input}`;
+        }
+        return false;
+      },
+    },
+  ]);
+
+  if (!answers.shouldContinue) {
+    Task.log(
+      chalk.yellow(
+        'If you change your mind, feel free to re-run this command.',
+      ),
+    );
+    return;
+  }
+
+  const { fullConfig: config } = await loadCliConfig({ args: [] });
+
+  const discovery = new Discovery();
+
+  if (answers.provider === 'GitHub') {
+    discovery.addProvider(GithubDiscoveryProvider.fromConfig(config));
+  }
+  if (answers.provider === 'GitLab') {
+    discovery.addProvider(GitlabDiscoveryProvider.fromConfig(config));
+  }
+
+  discovery.addAnalyzer(new BasicRepositoryAnalyzer());
+  discovery.addAnalyzer(new PackageJsonAnalyzer());
+
+  const { entities } = await discovery.run(answers.url);
+
+  if (!entities.length) {
+    Task.log(
+      chalk.yellow(`
+      We could not find enough information to be able to generate any Software Catalog entities for you.
+      Perhaps you can try again with a different repository?`),
+    );
+    return;
+  }
+
+  await Task.forItem('Creating', DISCOVERED_ENTITIES_FILE, async () => {
+    const payload: string[] = [];
+    for (const entity of entities) {
+      payload.push('---\n', yaml.stringify(entity));
+    }
+    await fs.writeFile(DISCOVERED_ENTITIES_FILE, payload.join(''));
+  });
+
+  await Task.forItem(
+    'Updating',
+    APP_CONFIG_FILE,
+    async () =>
+      await updateConfigFile(APP_CONFIG_FILE, {
+        catalog: {
+          locations: [
+            {
+              type: 'file',
+              target: DISCOVERED_ENTITIES_FILE,
+            },
+          ],
+        },
+      }),
+  );
+}

packages/cli/src/commands/onboard/discovery/providers/github/GithubDiscoveryProvider.ts

@@ -0,0 +1,163 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+import {
+  DefaultGithubCredentialsProvider,
+  GithubCredentialsProvider,
+  ScmIntegrations,
+} from '@backstage/integration';
+import { graphql } from '@octokit/graphql';
+import {
+  Repository as GraphqlRepository,
+  Query as GraphqlQuery,
+} from '@octokit/graphql-schema';
+import parseGitUrl from 'git-url-parse';
+import { Provider, Repository } from '../types';
+import { GithubRepository } from './GithubRepository';
+
+export class GithubDiscoveryProvider implements Provider {
+  readonly #envToken: string | undefined;
+  readonly #scmIntegrations: ScmIntegrations;
+  readonly #credentialsProvider: GithubCredentialsProvider;
+
+  static fromConfig(config: Config): GithubDiscoveryProvider {
+    const envToken = process.env.GITHUB_TOKEN || undefined;
+    const scmIntegrations = ScmIntegrations.fromConfig(config);
+    const credentialsProvider =
+      DefaultGithubCredentialsProvider.fromIntegrations(scmIntegrations);
+    return new GithubDiscoveryProvider(
+      envToken,
+      scmIntegrations,
+      credentialsProvider,
+    );
+  }
+
+  private constructor(
+    envToken: string | undefined,
+    integrations: ScmIntegrations,
+    credentialsProvider: GithubCredentialsProvider,
+  ) {
+    this.#envToken = envToken;
+    this.#scmIntegrations = integrations;
+    this.#credentialsProvider = credentialsProvider;
+  }
+
+  name(): string {
+    return 'GitHub';
+  }
+
+  async discover(url: string): Promise<Repository[] | false> {
+    if (!url.startsWith('https://github.com/')) {
+      return false;
+    }
+
+    const scmIntegration = this.#scmIntegrations.github.byUrl(url);
+    if (!scmIntegration) {
+      throw new Error(`No GitHub integration found for ${url}`);
+    }
+
+    const parsed = parseGitUrl(url);
+    const { name, organization } = parsed;
+    const org = organization || name; // depends on if it's a repo url or an org url...
+
+    const client = graphql.defaults({
+      baseUrl: scmIntegration.config.apiBaseUrl,
+      headers: await this.#getRequestHeaders(url),
+    });
+
+    const { repositories } = await this.#getOrganizationRepositories(
+      client,
+      org,
+    );
+
+    return repositories
+      .filter(repo => repo.url.startsWith(url))
+      .map(repo => new GithubRepository(client, repo, org));
+  }
+
+  async #getRequestHeaders(url: string): Promise<Record<string, string>> {
+    const credentials = await this.#credentialsProvider.getCredentials({
+      url,
+    });
+
+    if (credentials.headers) {
+      return credentials.headers;
+    } else if (credentials.token) {
+      return { authorization: `token ${credentials.token}` };
+    }
+
+    if (this.#envToken) {
+      return { authorization: `token ${this.#envToken}` };
+    }
+
+    throw new Error(
+      'No token available for GitHub, please configure your integrations or set a GITHUB_TOKEN env variable',
+    );
+  }
+
+  async #getOrganizationRepositories(client: typeof graphql, org: string) {
+    const query = `query repositories($org: String!, $cursor: String) {
+      repositoryOwner(login: $org) {
+        login
+        repositories(first: 100, after: $cursor) {
+          nodes {
+            name
+            url
+            description
+            isArchived
+            isFork
+          }
+          pageInfo {
+            hasNextPage
+            endCursor
+          }
+        }
+      }
+    }`;
+
+    const result: GraphqlRepository[] = [];
+
+    let cursor: string | undefined | null = undefined;
+    let hasNextPage = true;
+
+    while (hasNextPage) {
+      const response: GraphqlQuery = await client(query, {
+        org,
+        cursor,
+      });
+
+      const { repositories: connection } = response.repositoryOwner ?? {};
+
+      if (!connection) {
+        throw new Error(`Found no repositories for ${org}`);
+      }
+
+      for (const repository of connection.nodes ?? []) {
+        if (repository && !repository.isArchived && !repository.isFork) {
+          result.push(repository);
+        }
+      }
+
+      cursor = connection.pageInfo.endCursor;
+      hasNextPage = connection.pageInfo.hasNextPage;
+    }
+
+    return {
+      repositories: result,
+    };
+  }
+}

packages/cli/src/commands/onboard/discovery/providers/github/GithubFile.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { RepositoryFile } from '../types';
+
+export class GithubFile implements RepositoryFile {
+  readonly #path: string;
+  readonly #content: string;
+
+  constructor(path: string, content: string) {
+    this.#path = path;
+    this.#content = content;
+  }
+
+  get path(): string {
+    return this.#path;
+  }
+
+  async text(): Promise<string> {
+    return this.#content;
+  }
+}

packages/cli/src/commands/onboard/discovery/providers/github/GithubRepository.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { graphql } from '@octokit/graphql';
+import {
+  Repository as GraphqlRepository,
+  Blob as GraphqlBlob,
+} from '@octokit/graphql-schema';
+import { Repository, RepositoryFile } from '../types';
+import { GithubFile } from './GithubFile';
+
+export class GithubRepository implements Repository {
+  readonly #client: typeof graphql;
+  readonly #repo: GraphqlRepository;
+  readonly #org: string;
+
+  constructor(client: typeof graphql, repo: GraphqlRepository, org: string) {
+    this.#client = client;
+    this.#repo = repo;
+    this.#org = org;
+  }
+
+  get url(): string {
+    return this.#repo.url;
+  }
+
+  get name(): string {
+    return this.#repo.name;
+  }
+
+  get owner(): string {
+    return this.#org;
+  }
+
+  get description(): string | undefined {
+    return this.#repo.description ?? undefined;
+  }
+
+  async file(filename: string): Promise<RepositoryFile | undefined> {
+    const content = await this.#getFileContent(filename);
+    if (!content || content.isBinary || !content.text) {
+      return undefined;
+    }
+
+    return new GithubFile(filename, content.text ?? '');
+  }
+
+  async #getFileContent(filename: string) {
+    const query = `query RepoFiles($owner: String!, $name: String!, $expr: String!) {
+      repository(owner: $owner, name: $name) {
+        object(expression: $expr) {
+          ...on Blob {
+            text
+            isBinary
+          }
+        }
+      }
+    }`;
+
+    const response = await this.#client<{ repository: GraphqlRepository }>(
+      query,
+      {
+        name: this.#repo.name,
+        owner: this.#org,
+        expr: `HEAD:${filename}`,
+      },
+    );
+
+    return response.repository.object as GraphqlBlob;
+  }
+}

packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabDiscoveryProvider.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+import {
+  DefaultGitlabCredentialsProvider,
+  GitlabCredentialsProvider,
+  ScmIntegrations,
+} from '@backstage/integration';
+import fetch from 'node-fetch';
+import { Provider } from '../types';
+import { GitlabProject, ProjectResponse } from './GitlabProject';
+
+export class GitlabDiscoveryProvider implements Provider {
+  readonly #envToken: string | undefined;
+  readonly #scmIntegrations: ScmIntegrations;
+  readonly #credentialsProvider: GitlabCredentialsProvider;
+
+  static fromConfig(config: Config): GitlabDiscoveryProvider {
+    const envToken = process.env.GITLAB_TOKEN || undefined;
+    const scmIntegrations = ScmIntegrations.fromConfig(config);
+    const credentialsProvider =
+      DefaultGitlabCredentialsProvider.fromIntegrations(scmIntegrations);
+
+    return new GitlabDiscoveryProvider(
+      envToken,
+      scmIntegrations,
+      credentialsProvider,
+    );
+  }
+
+  private constructor(
+    envToken: string | undefined,
+    integrations: ScmIntegrations,
+    credentialsProvider: GitlabCredentialsProvider,
+  ) {
+    this.#envToken = envToken;
+    this.#scmIntegrations = integrations;
+    this.#credentialsProvider = credentialsProvider;
+  }
+
+  name(): string {
+    return 'GitLab';
+  }
+
+  async discover(url: string): Promise<false | GitlabProject[]> {
+    const { origin, pathname } = new URL(url);
+    const [, user] = pathname.split('/');
+
+    const scmIntegration = this.#scmIntegrations.gitlab.byUrl(origin);
+    if (!scmIntegration) {
+      throw new Error(`No GitLab integration found for ${origin}`);
+    }
+
+    const headers = await this.#getRequestHeaders(origin);
+
+    const response = await fetch(
+      `${scmIntegration.config.apiBaseUrl}/users/${user}/projects`,
+      { headers },
+    );
+
+    if (!response.ok) {
+      throw new Error(`${response.status} ${response.statusText}`);
+    }
+
+    const projects: ProjectResponse[] = await response.json();
+
+    return projects.map(
+      project =>
+        new GitlabProject(project, scmIntegration.config.apiBaseUrl, headers),
+    );
+  }
+
+  async #getRequestHeaders(url: string): Promise<Record<string, string>> {
+    const credentials = await this.#credentialsProvider.getCredentials({
+      url,
+    });
+
+    if (credentials.headers) {
+      return credentials.headers;
+    } else if (credentials.token) {
+      return { authorization: `Bearer ${credentials.token}` };
+    }
+
+    if (this.#envToken) {
+      return { authorization: `Bearer ${this.#envToken}` };
+    }
+
+    throw new Error(
+      'No token available for GitLab, please set a GITLAB_TOKEN env variable',
+    );
+  }
+}

packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabFile.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { RepositoryFile } from '../types';
+
+/**
+ * A single file in a GitLab repository.
+ */
+export class GitlabFile implements RepositoryFile {
+  readonly #path: string;
+  readonly #content: string;
+
+  constructor(path: string, content: string) {
+    this.#path = path;
+    this.#content = content;
+  }
+
+  get path(): string {
+    return this.#path;
+  }
+
+  async text(): Promise<string> {
+    return this.#content;
+  }
+}

packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabProject.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fetch from 'node-fetch';
+import { GitlabFile } from './GitlabFile';
+import { Repository, RepositoryFile } from '../types';
+
+export type ProjectResponse = {
+  id: string;
+  name: string;
+  description: string;
+  owner: {
+    username: string;
+  };
+  web_url: string;
+};
+
+type BranchResponse = {
+  default: boolean;
+  name: string;
+};
+
+type FileContentResponse = {
+  content: string;
+};
+
+export class GitlabProject implements Repository {
+  constructor(
+    private readonly project: ProjectResponse,
+    private readonly apiBaseUrl: string,
+    private readonly headers: { [name: string]: string },
+  ) {}
+
+  get url(): string {
+    return this.project.web_url;
+  }
+
+  get name(): string {
+    return this.project.name;
+  }
+
+  get owner(): string {
+    return this.project.owner.username;
+  }
+
+  get description(): string {
+    return this.project.description;
+  }
+
+  async file(filename: string): Promise<RepositoryFile | undefined> {
+    const mainBranch = await this.#getMainBranch();
+    const content = await this.#getFileContent(filename, mainBranch);
+
+    return new GitlabFile(filename, content);
+  }
+
+  async #getFileContent(path: string, mainBranch: string): Promise<string> {
+    const response = await fetch(
+      `${this.apiBaseUrl}/projects/${this.project.id}/repository/files/${path}?ref=${mainBranch}`,
+      { headers: this.headers },
+    );
+    const { content }: FileContentResponse = await response.json();
+
+    return Buffer.from(content, 'base64').toString('ascii');
+  }
+
+  async #getMainBranch(): Promise<string> {
+    const response = await fetch(
+      `${this.apiBaseUrl}/projects/${this.project.id}/repository/branches`,
+      { headers: this.headers },
+    );
+    const branches: BranchResponse[] = await response.json();
+
+    return branches.find(branch => branch.default)?.name ?? 'main';
+  }
+}

packages/cli/src/commands/onboard/discovery/providers/types.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Abstraction for a single repository.
+ */
+export interface Repository {
+  url: string;
+
+  name: string;
+
+  owner: string;
+
+  description?: string;
+
+  file(filename: string): Promise<RepositoryFile | undefined>;
+}
+
+/**
+ * Abstraction for a single repository file.
+ */
+export interface RepositoryFile {
+  /**
+   * The filepath of the data.
+   */
+  path: string;
+
+  /**
+   * The textual contents of the file.
+   */
+  text(): Promise<string>;
+}
+
+/**
+ * One integration that supports discovery of repositories.
+ */
+export interface Provider {
+  name(): string;
+  discover(url: string): Promise<Repository[] | false>;
+}

packages/cli/src/commands/onboard/files.ts

@@ -19,7 +19,13 @@ import * as path from 'path';
 
 /* eslint-disable-next-line no-restricted-syntax */
 const { targetRoot, ownDir } = findPaths(__dirname);
+
 export const APP_CONFIG_FILE = path.join(targetRoot, 'app-config.local.yaml');
+export const DISCOVERED_ENTITIES_FILE = path.join(
+  targetRoot,
+  'examples',
+  'discovered-entities.yaml',
+);
 export const PATCH_FOLDER = path.join(
   ownDir,
   'src',

packages/integration/api-report.md

@@ -165,6 +165,18 @@ export class DefaultGithubCredentialsProvider
   getCredentials(opts: { url: string }): Promise<GithubCredentials>;
 }
 
+// @public
+export class DefaultGitlabCredentialsProvider
+  implements GitlabCredentialsProvider
+{
+  // (undocumented)
+  static fromIntegrations(
+    integrations: ScmIntegrationRegistry,
+  ): DefaultGitlabCredentialsProvider;
+  // (undocumented)
+  getCredentials(opts: { url: string }): Promise<GitlabCredentials>;
+}
+
 // @public
 export function defaultScmResolveUrl(options: {
   url: string;
@@ -478,6 +490,20 @@ export type GithubIntegrationConfig = {
   apps?: GithubAppConfig[];
 };
 
+// @public (undocumented)
+export type GitlabCredentials = {
+  headers?: {
+    [name: string]: string;
+  };
+  token?: string;
+};
+
+// @public (undocumented)
+export interface GitlabCredentialsProvider {
+  // (undocumented)
+  getCredentials(opts?: { url: string }): Promise<GitlabCredentials>;
+}
+
 // @public
 export class GitLabIntegration implements ScmIntegration {
   constructor(integrationConfig: GitLabIntegrationConfig);

packages/integration/src/gitlab/DefaultGitlabCredentialsProvider.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ScmIntegrationRegistry } from '../registry';
+import { SingleInstanceGitlabCredentialsProvider } from './SingleInstanceGitlabCredentialsProvider';
+import { GitlabCredentials, GitlabCredentialsProvider } from './types';
+
+/**
+ * Handles the creation and caching of credentials for GitLab integrations.
+ *
+ * @public
+ */
+export class DefaultGitlabCredentialsProvider
+  implements GitlabCredentialsProvider
+{
+  static fromIntegrations(integrations: ScmIntegrationRegistry) {
+    const credentialsProviders: Map<string, GitlabCredentialsProvider> =
+      new Map<string, GitlabCredentialsProvider>();
+
+    integrations.gitlab.list().forEach(integration => {
+      const credentialsProvider =
+        SingleInstanceGitlabCredentialsProvider.create(integration.config);
+      credentialsProviders.set(integration.config.host, credentialsProvider);
+    });
+    return new DefaultGitlabCredentialsProvider(credentialsProviders);
+  }
+
+  private constructor(
+    private readonly providers: Map<string, GitlabCredentialsProvider>,
+  ) {}
+
+  async getCredentials(opts: { url: string }): Promise<GitlabCredentials> {
+    const parsed = new URL(opts.url);
+    const provider = this.providers.get(parsed.host);
+
+    if (!provider) {
+      throw new Error(
+        `There is no GitLab integration that matches ${opts.url}. Please add a configuration for an integration.`,
+      );
+    }
+
+    return provider.getCredentials(opts);
+  }
+}

packages/integration/src/gitlab/SingleInstanceGitlabCredentialsProvider.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GitLabIntegrationConfig } from './config';
+import { GitlabCredentials, GitlabCredentialsProvider } from './types';
+
+export class SingleInstanceGitlabCredentialsProvider
+  implements GitlabCredentialsProvider
+{
+  static create: (
+    config: GitLabIntegrationConfig,
+  ) => GitlabCredentialsProvider = config => {
+    return new SingleInstanceGitlabCredentialsProvider(config.token);
+  };
+
+  constructor(private readonly token?: string) {}
+
+  async getCredentials(): Promise<GitlabCredentials> {
+    return {
+      headers: this.token
+        ? { Authorization: `Bearer ${this.token}` }
+        : undefined,
+      token: this.token,
+    };
+  }
+}

packages/integration/src/gitlab/index.ts

@@ -22,3 +22,5 @@ export {
 export type { GitLabIntegrationConfig } from './config';
 export { getGitLabFileFetchUrl, getGitLabRequestOptions } from './core';
 export { GitLabIntegration, replaceGitLabUrlType } from './GitLabIntegration';
+export { DefaultGitlabCredentialsProvider } from './DefaultGitlabCredentialsProvider';
+export type { GitlabCredentials, GitlabCredentialsProvider } from './types';

packages/integration/src/gitlab/types.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @public
+ */
+export type GitlabCredentials = {
+  headers?: { [name: string]: string };
+  token?: string;
+};
+
+/**
+ * @public
+ */
+export interface GitlabCredentialsProvider {
+  getCredentials(opts?: { url: string }): Promise<GitlabCredentials>;
+}

yarn.lock

@@ -3848,6 +3848,7 @@ __metadata:
   resolution: "@backstage/cli@workspace:packages/cli"
   dependencies:
     "@backstage/backend-common": "workspace:^"
+    "@backstage/catalog-model": "workspace:^"
     "@backstage/cli-common": "workspace:^"
     "@backstage/cli-node": "workspace:^"
     "@backstage/config": "workspace:^"
@@ -3858,12 +3859,15 @@ __metadata:
     "@backstage/dev-utils": "workspace:^"
     "@backstage/errors": "workspace:^"
     "@backstage/eslint-plugin": "workspace:^"
+    "@backstage/integration": "workspace:^"
     "@backstage/release-manifests": "workspace:^"
     "@backstage/test-utils": "workspace:^"
     "@backstage/theme": "workspace:^"
     "@backstage/types": "workspace:^"
     "@esbuild-kit/cjs-loader": ^2.4.1
     "@manypkg/get-packages": ^1.1.3
+    "@octokit/graphql": ^5.0.0
+    "@octokit/graphql-schema": ^13.7.0
     "@octokit/oauth-app": ^4.2.0
     "@octokit/request": ^6.0.0
     "@pmmmwh/react-refresh-webpack-plugin": ^0.5.7
@@ -3930,6 +3934,7 @@ __metadata:
     express: ^4.17.1
     fork-ts-checker-webpack-plugin: ^7.0.0-alpha.8
     fs-extra: 10.1.0
+    git-url-parse: ^13.0.0
     glob: ^7.1.7
     global-agent: ^3.0.0
     handlebars: ^4.7.3
@@ -13130,6 +13135,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@octokit/graphql-schema@npm:^13.7.0":
+  version: 13.10.0
+  resolution: "@octokit/graphql-schema@npm:13.10.0"
+  dependencies:
+    graphql: ^16.0.0
+    graphql-tag: ^2.10.3
+  checksum: fdec9c9a4df1f90b733ea0e24964744faceaf65e5d350b1727892e8e0e5821df1d29aec5cfa039925a044c6f56d4ed2028505108db7fbc0c68011053853c2411
+  languageName: node
+  linkType: hard
+
 "@octokit/graphql@npm:^5.0.0":
   version: 5.0.6
   resolution: "@octokit/graphql@npm:5.0.6"
@@ -25543,7 +25558,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"graphql-tag@npm:^2.11.0, graphql-tag@npm:^2.12.6":
+"graphql-tag@npm:^2.10.3, graphql-tag@npm:^2.11.0, graphql-tag@npm:^2.12.6":
   version: 2.12.6
   resolution: "graphql-tag@npm:2.12.6"
   dependencies: