diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index b83c9a16f3..9f38d782fe 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -15,11 +15,35 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'node-fetch'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; export class GithubReaderProcessor implements LocationProcessor { + private privateToken: string; + + constructor(config?: Config) { + this.privateToken = + config?.getOptionalString('catalog.processors.github.privateToken') ?? ''; + } + + getRequestOptions(): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (this.privateToken !== '') { + headers.Authorization = `token ${this.privateToken}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + async readLocation( location: LocationSpec, optional: boolean, @@ -34,7 +58,7 @@ export class GithubReaderProcessor implements LocationProcessor { // TODO(freben): Should "hard" errors thrown by this line be treated as // notFound instead of fatal? - const response = await fetch(url.toString()); + const response = await fetch(url.toString(), this.getRequestOptions()); if (response.ok) { const data = await response.buffer(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/github.test.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/github.test.ts index 0d29d9c7be..99a7d3bb5c 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/github.test.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/github.test.ts @@ -16,11 +16,13 @@ import { getVoidLogger } from '@backstage/backend-common'; import { GithubPreparer } from './github'; -import { checkoutGitRepository } from './helpers'; +import { checkoutGithubRepository } from './helpers'; jest.mock('./helpers', () => ({ ...jest.requireActual<{}>('./helpers'), - checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'), + checkoutGithubRepository: jest.fn( + () => '/tmp/backstage-repo/org/name/branch', + ), })); const createMockEntity = (annotations = {}) => { @@ -48,7 +50,7 @@ describe('github preparer', () => { }); const tempDocsPath = await preparer.prepare(mockEntity); - expect(checkoutGitRepository).toHaveBeenCalledTimes(1); + expect(checkoutGithubRepository).toHaveBeenCalledTimes(1); expect(tempDocsPath).toEqual( '/tmp/backstage-repo/org/name/branch/plugins/techdocs-backend/examples/documented-component', ); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/github.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/github.ts index 65c3e551cb..242b19c24d 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/github.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/github.ts @@ -18,7 +18,7 @@ import { Entity } from '@backstage/catalog-model'; import { InputError } from '@backstage/backend-common'; import { PreparerBase } from './types'; import parseGitUrl from 'git-url-parse'; -import { parseReferenceAnnotation, checkoutGitRepository } from './helpers'; +import { parseReferenceAnnotation, checkoutGithubRepository } from './helpers'; import { Logger } from 'winston'; export class GithubPreparer implements PreparerBase { @@ -39,7 +39,7 @@ export class GithubPreparer implements PreparerBase { } try { - const repoPath = await checkoutGitRepository(target); + const repoPath = await checkoutGithubRepository(target); const parsedGitLocation = parseGitUrl(target); return path.join(repoPath, parsedGitLocation.filepath); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/helpers.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/helpers.ts index 27e746590a..d962242262 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/helpers.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/helpers.ts @@ -94,3 +94,45 @@ export const checkoutGitRepository = async ( return repositoryTmpPath; }; + +// Could be merged with checkoutGitRepository +export const checkoutGithubRepository = async ( + repoUrl: string, +): Promise => { + const parsedGitLocation = parseGitUrl(repoUrl); + + // Should propably not be hardcoded names of env variables, but seems too hard to access config down here + const user = process.env.GITHUB_PRIVATE_TOKEN_USER || ''; + const token = process.env.GITHUB_PRIVATE_TOKEN || ''; + + const repositoryTmpPath = path.join( + // fs.realpathSync fixes a problem with macOS returning a path that is a symlink + fs.realpathSync(os.tmpdir()), + 'backstage-repo', + parsedGitLocation.source, + parsedGitLocation.owner, + parsedGitLocation.name, + parsedGitLocation.ref, + ); + + if (fs.existsSync(repositoryTmpPath)) { + const repository = await Repository.open(repositoryTmpPath); + const currentBranchName = (await repository.getCurrentBranch()).shorthand(); + await repository.mergeBranches( + currentBranchName, + `origin/${currentBranchName}`, + ); + return repositoryTmpPath; + } + + if (user && token) { + parsedGitLocation.token = `${user}:${token}`; + } + + const repositoryCheckoutUrl = parsedGitLocation.toString('https'); + + fs.mkdirSync(repositoryTmpPath, { recursive: true }); + await Clone.clone(repositoryCheckoutUrl, repositoryTmpPath); + + return repositoryTmpPath; +};