TechDocs & Catalog: Private Github repository support (#2247)

* Add private repo functionality to GhReaderProcesor

* Add github helper for private github repo

* Edit test

* lint

* Remove debugging console.log()

* Less specific if statement

* shorten get private token

* Actual shortening

* Change how current branch name is retrieved

plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts

@@ -15,11 +15,35 @@
  */
 
 import { LocationSpec } from '@backstage/catalog-model';
-import fetch from 'node-fetch';
+import fetch, { RequestInit, HeadersInit } from 'node-fetch';
 import * as result from './results';
 import { LocationProcessor, LocationProcessorEmit } from './types';
+import { Config } from '@backstage/config';
 
 export class GithubReaderProcessor implements LocationProcessor {
+  private privateToken: string;
+
+  constructor(config?: Config) {
+    this.privateToken =
+      config?.getOptionalString('catalog.processors.github.privateToken') ?? '';
+  }
+
+  getRequestOptions(): RequestInit {
+    const headers: HeadersInit = {
+      Accept: 'application/vnd.github.v3.raw',
+    };
+
+    if (this.privateToken !== '') {
+      headers.Authorization = `token ${this.privateToken}`;
+    }
+
+    const requestOptions: RequestInit = {
+      headers,
+    };
+
+    return requestOptions;
+  }
+
   async readLocation(
     location: LocationSpec,
     optional: boolean,
@@ -34,7 +58,7 @@ export class GithubReaderProcessor implements LocationProcessor {
 
       // TODO(freben): Should "hard" errors thrown by this line be treated as
       // notFound instead of fatal?
-      const response = await fetch(url.toString());
+      const response = await fetch(url.toString(), this.getRequestOptions());
 
       if (response.ok) {
         const data = await response.buffer();

plugins/techdocs-backend/src/techdocs/stages/prepare/github.test.ts

@@ -16,11 +16,13 @@
 
 import { getVoidLogger } from '@backstage/backend-common';
 import { GithubPreparer } from './github';
-import { checkoutGitRepository } from './helpers';
+import { checkoutGithubRepository } from './helpers';
 
 jest.mock('./helpers', () => ({
   ...jest.requireActual<{}>('./helpers'),
-  checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'),
+  checkoutGithubRepository: jest.fn(
+    () => '/tmp/backstage-repo/org/name/branch',
+  ),
 }));
 
 const createMockEntity = (annotations = {}) => {
@@ -48,7 +50,7 @@ describe('github preparer', () => {
     });
 
     const tempDocsPath = await preparer.prepare(mockEntity);
-    expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
+    expect(checkoutGithubRepository).toHaveBeenCalledTimes(1);
     expect(tempDocsPath).toEqual(
       '/tmp/backstage-repo/org/name/branch/plugins/techdocs-backend/examples/documented-component',
     );

plugins/techdocs-backend/src/techdocs/stages/prepare/github.ts

@@ -18,7 +18,7 @@ import { Entity } from '@backstage/catalog-model';
 import { InputError } from '@backstage/backend-common';
 import { PreparerBase } from './types';
 import parseGitUrl from 'git-url-parse';
-import { parseReferenceAnnotation, checkoutGitRepository } from './helpers';
+import { parseReferenceAnnotation, checkoutGithubRepository } from './helpers';
 import { Logger } from 'winston';
 
 export class GithubPreparer implements PreparerBase {
@@ -39,7 +39,7 @@ export class GithubPreparer implements PreparerBase {
     }
 
     try {
-      const repoPath = await checkoutGitRepository(target);
+      const repoPath = await checkoutGithubRepository(target);
 
       const parsedGitLocation = parseGitUrl(target);
       return path.join(repoPath, parsedGitLocation.filepath);

plugins/techdocs-backend/src/techdocs/stages/prepare/helpers.ts

@@ -94,3 +94,45 @@ export const checkoutGitRepository = async (
 
   return repositoryTmpPath;
 };
+
+// Could be merged with checkoutGitRepository
+export const checkoutGithubRepository = async (
+  repoUrl: string,
+): Promise<string> => {
+  const parsedGitLocation = parseGitUrl(repoUrl);
+
+  // Should propably not be hardcoded names of env variables, but seems too hard to access config down here
+  const user = process.env.GITHUB_PRIVATE_TOKEN_USER || '';
+  const token = process.env.GITHUB_PRIVATE_TOKEN || '';
+
+  const repositoryTmpPath = path.join(
+    // fs.realpathSync fixes a problem with macOS returning a path that is a symlink
+    fs.realpathSync(os.tmpdir()),
+    'backstage-repo',
+    parsedGitLocation.source,
+    parsedGitLocation.owner,
+    parsedGitLocation.name,
+    parsedGitLocation.ref,
+  );
+
+  if (fs.existsSync(repositoryTmpPath)) {
+    const repository = await Repository.open(repositoryTmpPath);
+    const currentBranchName = (await repository.getCurrentBranch()).shorthand();
+    await repository.mergeBranches(
+      currentBranchName,
+      `origin/${currentBranchName}`,
+    );
+    return repositoryTmpPath;
+  }
+
+  if (user && token) {
+    parsedGitLocation.token = `${user}:${token}`;
+  }
+
+  const repositoryCheckoutUrl = parsedGitLocation.toString('https');
+
+  fs.mkdirSync(repositoryTmpPath, { recursive: true });
+  await Clone.clone(repositoryCheckoutUrl, repositoryTmpPath);
+
+  return repositoryTmpPath;
+};