Merge pull request #3805 from backstage/blam/isomorphic-git

Remove the NodeGit Dependency in favour of isomorphic-git

packages/backend-common/package.json

@@ -45,6 +45,7 @@
     "fs-extra": "^9.0.1",
     "git-url-parse": "^11.4.3",
     "helmet": "^4.0.0",
+    "isomorphic-git": "^1.8.0",
     "knex": "^0.21.6",
     "lodash": "^4.17.15",
     "logform": "^2.1.1",

packages/backend-common/src/index.ts

@@ -24,3 +24,4 @@ export * from './reading';
 export * from './service';
 export * from './paths';
 export * from './hot';
+export * from './scm';

packages/backend-common/src/scm/git.test.ts

@@ -0,0 +1,321 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+jest.mock('isomorphic-git');
+jest.mock('isomorphic-git/http/node');
+jest.mock('fs-extra');
+
+import * as isomorphic from 'isomorphic-git';
+import { Git } from './git';
+import http from 'isomorphic-git/http/node';
+import fs from 'fs-extra';
+
+describe('Git', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+  describe('add', () => {
+    it('should call isomorphic-git add with the correct arguments', async () => {
+      const git = Git.fromAuth({});
+      const dir = 'mockdirectory';
+      const filepath = 'mockfile/path';
+
+      await git.add({ dir, filepath });
+
+      expect(isomorphic.add).toHaveBeenCalledWith({
+        fs,
+        dir,
+        filepath,
+      });
+    });
+  });
+
+  describe('addRemote', () => {
+    it('should call isomorphic-git with the correct arguments', async () => {
+      const git = Git.fromAuth({});
+      const dir = 'mockdirectory';
+      const remote = 'origin';
+      const url = 'git@github.com/something/sads';
+
+      await git.addRemote({ dir, remote, url });
+
+      expect(isomorphic.addRemote).toHaveBeenCalledWith({
+        fs,
+        dir,
+        remote,
+        url,
+      });
+    });
+  });
+
+  describe('commit', () => {
+    it('should call isomorphic-git with the correct arguments', async () => {
+      const git = Git.fromAuth({});
+      const dir = 'mockdirectory';
+      const message = 'Inital Commit';
+      const author = {
+        name: 'author',
+        email: 'test@backstage.io',
+      };
+      const committer = {
+        name: 'comitter',
+        email: 'test@backstage.io',
+      };
+
+      await git.commit({ dir, message, author, committer });
+
+      expect(isomorphic.commit).toHaveBeenCalledWith({
+        fs,
+        dir,
+        message,
+        author,
+        committer,
+      });
+    });
+  });
+
+  describe('clone', () => {
+    it('should call isomorphic-git with the correct arguments', async () => {
+      const url = 'http://github.com/some/repo';
+      const dir = '/some/mock/dir';
+      const auth = {
+        username: 'blob',
+        password: 'hunter2',
+      };
+      const git = Git.fromAuth(auth);
+
+      await git.clone({ url, dir });
+
+      expect(isomorphic.clone).toHaveBeenCalledWith({
+        fs,
+        http,
+        url,
+        dir,
+        singleBranch: true,
+        depth: 1,
+        onProgress: expect.any(Function),
+        headers: {
+          'user-agent': 'git/@isomorphic-git',
+        },
+        onAuth: expect.any(Function),
+      });
+    });
+    it('should pass a function that returns the authorization as the onAuth handler', async () => {
+      const url = 'http://github.com/some/repo';
+      const dir = '/some/mock/dir';
+      const auth = {
+        username: 'blob',
+        password: 'hunter2',
+      };
+      const git = Git.fromAuth(auth);
+
+      await git.clone({ url, dir });
+
+      const { onAuth } = ((isomorphic.clone as unknown) as jest.Mock<
+        typeof isomorphic['clone']
+      >).mock.calls[0][0]!;
+
+      expect(onAuth()).toEqual(auth);
+    });
+  });
+
+  describe('currentBranch', () => {
+    it('should call isomorphic-git with the correct arguments', async () => {
+      const dir = '/some/mock/dir';
+      const fullName = true;
+      const git = Git.fromAuth({});
+
+      await git.currentBranch({ dir, fullName });
+
+      expect(isomorphic.currentBranch).toHaveBeenCalledWith({
+        fs,
+        dir,
+        fullname: true,
+      });
+
+      await git.currentBranch({ dir });
+
+      expect(isomorphic.currentBranch).toHaveBeenCalledWith({
+        fs,
+        dir,
+        fullname: false,
+      });
+    });
+  });
+
+  describe('fetch', () => {
+    it('should call isomorphic-git with the correct arguments', async () => {
+      const remote = 'http://github.com/some/repo';
+      const dir = '/some/mock/dir';
+      const auth = {
+        username: 'blob',
+        password: 'hunter2',
+      };
+      const git = Git.fromAuth(auth);
+
+      await git.fetch({ remote, dir });
+
+      expect(isomorphic.fetch).toHaveBeenCalledWith({
+        fs,
+        http,
+        remote,
+        dir,
+        onProgress: expect.any(Function),
+        headers: {
+          'user-agent': 'git/@isomorphic-git',
+        },
+        onAuth: expect.any(Function),
+      });
+    });
+    it('should pass a function that returns the authorization as the onAuth handler', async () => {
+      const remote = 'http://github.com/some/repo';
+      const dir = '/some/mock/dir';
+      const auth = {
+        username: 'blob',
+        password: 'hunter2',
+      };
+      const git = Git.fromAuth(auth);
+
+      await git.fetch({ remote, dir });
+
+      const { onAuth } = ((isomorphic.fetch as unknown) as jest.Mock<
+        typeof isomorphic['fetch']
+      >).mock.calls[0][0]!;
+
+      expect(onAuth()).toEqual(auth);
+    });
+  });
+
+  describe('init', () => {
+    it('should call isomorphic-git with the correct arguments', async () => {
+      const dir = '/some/mock/dir';
+
+      const git = Git.fromAuth({});
+
+      await git.init({ dir });
+
+      expect(isomorphic.init).toHaveBeenCalledWith({
+        fs,
+        dir,
+      });
+    });
+  });
+
+  describe('merge', () => {
+    it('should call isomorphic-git with the correct arguments', async () => {
+      const dir = '/some/mock/dir';
+      const author = {
+        name: 'author',
+        email: 'test@backstage.io',
+      };
+      const committer = {
+        name: 'comitter',
+        email: 'test@backstage.io',
+      };
+      const theirs = 'master';
+      const ours = 'production';
+
+      const git = Git.fromAuth({});
+
+      await git.merge({ dir, theirs, ours, author, committer });
+
+      expect(isomorphic.merge).toHaveBeenCalledWith({
+        fs,
+        dir,
+        ours,
+        theirs,
+        author,
+        committer,
+      });
+    });
+  });
+
+  describe('push', () => {
+    it('should call isomorphic-git with the correct arguments', async () => {
+      const remote = 'origin';
+      const dir = '/some/mock/dir';
+      const auth = {
+        username: 'blob',
+        password: 'hunter2',
+      };
+      const git = Git.fromAuth(auth);
+
+      await git.push({ dir, remote });
+
+      expect(isomorphic.push).toHaveBeenCalledWith({
+        fs,
+        http,
+        remote,
+        dir,
+        onProgress: expect.any(Function),
+        headers: {
+          'user-agent': 'git/@isomorphic-git',
+        },
+        onAuth: expect.any(Function),
+      });
+    });
+    it('should pass a function that returns the authorization as the onAuth handler', async () => {
+      const remote = 'origin';
+      const dir = '/some/mock/dir';
+      const auth = {
+        username: 'blob',
+        password: 'hunter2',
+      };
+      const git = Git.fromAuth(auth);
+
+      await git.push({ remote, dir });
+
+      const { onAuth } = ((isomorphic.push as unknown) as jest.Mock<
+        typeof isomorphic['push']
+      >).mock.calls[0][0]!;
+
+      expect(onAuth()).toEqual(auth);
+    });
+  });
+
+  describe('readCommit', () => {
+    it('should call isomorphic-git with the correct arguments', async () => {
+      const dir = '/some/mock/dir';
+      const sha = 'as43bd7';
+
+      const git = Git.fromAuth({});
+
+      await git.readCommit({ dir, sha });
+
+      expect(isomorphic.readCommit).toHaveBeenCalledWith({
+        fs,
+        dir,
+        oid: sha,
+      });
+    });
+  });
+
+  describe('resolveRef', () => {
+    it('should call isomorphic-git with the correct arguments', async () => {
+      const dir = '/some/mock/dir';
+      const ref = 'as43bd7';
+
+      const git = Git.fromAuth({});
+
+      await git.resolveRef({ dir, ref });
+
+      expect(isomorphic.resolveRef).toHaveBeenCalledWith({
+        fs,
+        dir,
+        ref,
+      });
+    });
+  });
+});

packages/backend-common/src/scm/git.ts

@@ -0,0 +1,251 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import git, {
+  ProgressCallback,
+  MergeResult,
+  ReadCommitResult,
+} from 'isomorphic-git';
+import http from 'isomorphic-git/http/node';
+import fs from 'fs-extra';
+import { Logger } from 'winston';
+
+/*
+provider    username        password
+GitHub      token           'x-oauth-basic'
+GitHub App  token           'x-access-token'
+BitBucket   'x-token-auth'  token
+GitLab      'oauth2'        token
+From : https://isomorphic-git.org/docs/en/onAuth
+
+Azure       'notempty'      token
+*/
+export class Git {
+  private constructor(
+    private readonly config: {
+      username?: string;
+      password?: string;
+      logger?: Logger;
+    },
+  ) {}
+
+  async add({
+    dir,
+    filepath,
+  }: {
+    dir: string;
+    filepath: string;
+  }): Promise<void> {
+    this.config.logger?.info(`Adding file {dir=${dir},filepath=${filepath}}`);
+
+    return git.add({ fs, dir, filepath });
+  }
+
+  async addRemote({
+    dir,
+    url,
+    remote,
+  }: {
+    dir: string;
+    remote: string;
+    url: string;
+  }): Promise<void> {
+    this.config.logger?.info(
+      `Creating new remote {dir=${dir},remote=${remote},url=${url}}`,
+    );
+    return git.addRemote({ fs, dir, remote, url });
+  }
+
+  async commit({
+    dir,
+    message,
+    author,
+    committer,
+  }: {
+    dir: string;
+    message: string;
+    author: { name: string; email: string };
+    committer: { name: string; email: string };
+  }): Promise<string> {
+    this.config.logger?.info(
+      `Committing file to repo {dir=${dir},message=${message}}`,
+    );
+
+    return git.commit({ fs, dir, message, author, committer });
+  }
+
+  async clone({ url, dir }: { url: string; dir: string }): Promise<void> {
+    this.config.logger?.info(`Cloning repo {dir=${dir},url=${url}}`);
+    return git.clone({
+      fs,
+      http,
+      url,
+      dir,
+      singleBranch: true,
+      depth: 1,
+      onProgress: this.onProgressHandler(),
+      headers: {
+        'user-agent': 'git/@isomorphic-git',
+      },
+      onAuth: this.onAuth,
+    });
+  }
+
+  // https://isomorphic-git.org/docs/en/currentBranch
+  async currentBranch({
+    dir,
+    fullName,
+  }: {
+    dir: string;
+    fullName?: boolean;
+  }): Promise<string | undefined> {
+    const fullname = fullName ?? false;
+    return git.currentBranch({ fs, dir, fullname }) as Promise<
+      string | undefined
+    >;
+  }
+
+  // https://isomorphic-git.org/docs/en/fetch
+  async fetch({
+    dir,
+    remote,
+  }: {
+    dir: string;
+    remote?: string;
+  }): Promise<void> {
+    const remoteValue = remote ?? 'origin';
+    this.config.logger?.info(
+      `Fetching remote=${remoteValue} for repository {dir=${dir}}`,
+    );
+    await git.fetch({
+      fs,
+      http,
+      dir,
+      remote: remoteValue,
+      onProgress: this.onProgressHandler(),
+      headers: {
+        'user-agent': 'git/@isomorphic-git',
+      },
+      onAuth: this.onAuth,
+    });
+  }
+
+  async init({ dir }: { dir: string }): Promise<void> {
+    this.config.logger?.info(`Init git repository {dir=${dir}}`);
+
+    return git.init({
+      fs,
+      dir,
+    });
+  }
+
+  // https://isomorphic-git.org/docs/en/merge
+  async merge({
+    dir,
+    theirs,
+    ours,
+    author,
+    committer,
+  }: {
+    dir: string;
+    theirs: string;
+    ours?: string;
+    author: { name: string; email: string };
+    committer: { name: string; email: string };
+  }): Promise<MergeResult> {
+    this.config.logger?.info(
+      `Merging branch '${theirs}' into '${ours}' for repository {dir=${dir}}`,
+    );
+
+    // If ours is undefined, current branch is used.
+    return git.merge({
+      fs,
+      dir,
+      ours,
+      theirs,
+      author,
+      committer,
+    });
+  }
+
+  async push({ dir, remote }: { dir: string; remote: string }) {
+    this.config.logger?.info(
+      `Pushing directory to remote {dir=${dir},remote=${remote}}`,
+    );
+    return git.push({
+      fs,
+      dir,
+      http,
+      onProgress: this.onProgressHandler(),
+      headers: {
+        'user-agent': 'git/@isomorphic-git',
+      },
+      remote: remote,
+      onAuth: this.onAuth,
+    });
+  }
+
+  // https://isomorphic-git.org/docs/en/readCommit
+  async readCommit({
+    dir,
+    sha,
+  }: {
+    dir: string;
+    sha: string;
+  }): Promise<ReadCommitResult> {
+    return git.readCommit({ fs, dir, oid: sha });
+  }
+
+  // https://isomorphic-git.org/docs/en/resolveRef
+  async resolveRef({
+    dir,
+    ref,
+  }: {
+    dir: string;
+    ref: string;
+  }): Promise<string> {
+    return git.resolveRef({ fs, dir, ref });
+  }
+
+  private onAuth = () => ({
+    username: this.config.username,
+    password: this.config.password,
+  });
+
+  private onProgressHandler = (): ProgressCallback => {
+    let currentPhase = '';
+
+    return event => {
+      if (currentPhase !== event.phase) {
+        currentPhase = event.phase;
+        this.config.logger?.info(event.phase);
+      }
+      const total = event.total
+        ? `${Math.round((event.loaded / event.total) * 100)}%`
+        : event.loaded;
+      this.config.logger?.debug(`status={${event.phase},total={${total}}}`);
+    };
+  };
+
+  static fromAuth = ({
+    username,
+    password,
+    logger,
+  }: {
+    username?: string;
+    password?: string;
+    logger?: Logger;
+  }) => new Git({ username, password, logger });
+}

packages/backend-common/src/scm/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { Git } from './git';

packages/techdocs-common/package.json

@@ -51,7 +51,6 @@
     "js-yaml": "^3.14.0",
     "mime-types": "^2.1.27",
     "mock-fs": "^4.13.0",
-    "nodegit": "^0.27.0",
     "recursive-readdir": "^2.2.2",
     "winston": "^3.2.1"
   },
@@ -62,7 +61,6 @@
     "@types/js-yaml": "^3.12.5",
     "@types/mime-types": "^2.1.0",
     "@types/mock-fs": "^4.13.0",
-    "@types/nodegit": "^0.26.12",
     "@types/recursive-readdir": "^2.2.0"
   },
   "jest": {

packages/techdocs-common/src/helpers.ts

@@ -17,9 +17,8 @@
 import os from 'os';
 import path from 'path';
 import parseGitUrl from 'git-url-parse';
-import NodeGit, { Clone, Repository } from 'nodegit';
 import fs from 'fs-extra';
-import { InputError, UrlReader } from '@backstage/backend-common';
+import { InputError, UrlReader, Git } from '@backstage/backend-common';
 import { Entity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { getDefaultBranch } from './default-branch';
@@ -27,10 +26,6 @@ import { getGitRepoType, getTokenForGitRepo } from './git-auth';
 import { RemoteProtocol } from './stages/prepare/types';
 import { Logger } from 'winston';
 
-// Enables core.longpaths on windows to prevent crashing when checking out repos with long foldernames and/or deep nesting
-// @ts-ignore
-NodeGit.Libgit2.opts(28, 1);
-
 export type ParsedLocationAnnotation = {
   type: RemoteProtocol;
   target: string;
@@ -128,17 +123,61 @@ export const checkoutGitRepository = async (
   const repositoryTmpPath = await getGitRepositoryTempFolder(repoUrl, config);
   const token = await getTokenForGitRepo(repoUrl, config);
 
+  // Initialize a git client
+  let git = Git.fromAuth({ logger });
+
+  // Docs about why username and password are set to these specific values.
+  // https://isomorphic-git.org/docs/en/onAuth#oauth2-tokens
+  if (token) {
+    const type = getGitRepoType(repoUrl);
+    switch (type) {
+      case 'github':
+        git = Git.fromAuth({
+          username: token,
+          password: 'x-oauth-basic',
+          logger,
+        });
+        parsedGitLocation.token = `${token}:x-oauth-basic`;
+        break;
+      case 'gitlab':
+        git = Git.fromAuth({
+          username: 'oauth2',
+          password: token,
+          logger,
+        });
+        parsedGitLocation.token = `dummyUsername:${token}`;
+        parsedGitLocation.git_suffix = true;
+        break;
+      case 'azure/api':
+        git = Git.fromAuth({
+          username: 'notempty',
+          password: token,
+          logger: logger,
+        });
+        break;
+      default:
+        parsedGitLocation.token = `:${token}`;
+    }
+  }
+
+  // Pull from repository if it has already been cloned.
   if (fs.existsSync(repositoryTmpPath)) {
     try {
-      const repository = await Repository.open(repositoryTmpPath);
-      const currentBranchName = (
-        await repository.getCurrentBranch()
-      ).shorthand();
-      await repository.fetch('origin');
-      await repository.mergeBranches(
-        currentBranchName,
-        `origin/${currentBranchName}`,
-      );
+      const currentBranchName = await git.currentBranch({
+        dir: repositoryTmpPath,
+      });
+
+      await git.fetch({ dir: repositoryTmpPath, remote: 'origin' });
+      await git.merge({
+        dir: repositoryTmpPath,
+        theirs: `origin/${currentBranchName}`,
+        ours: currentBranchName || undefined,
+        author: { name: 'Backstage TechDocs', email: 'techdocs@backstage.io' },
+        committer: {
+          name: 'Backstage TechDocs',
+          email: 'techdocs@backstage.io',
+        },
+      });
       return repositoryTmpPath;
     } catch (e) {
       logger.info(
@@ -148,26 +187,10 @@ export const checkoutGitRepository = async (
     }
   }
 
-  if (token) {
-    const type = getGitRepoType(repoUrl);
-    switch (type) {
-      case 'gitlab':
-        // Personal Access Token
-        parsedGitLocation.token = `dummyUsername:${token}`;
-        parsedGitLocation.git_suffix = true;
-        break;
-      case 'github':
-        parsedGitLocation.token = `${token}:x-oauth-basic`;
-        break;
-      default:
-        parsedGitLocation.token = `:${token}`;
-    }
-  }
-
   const repositoryCheckoutUrl = parsedGitLocation.toString('https');
 
   fs.mkdirSync(repositoryTmpPath, { recursive: true });
-  await Clone.clone(repositoryCheckoutUrl, repositoryTmpPath);
+  await git.clone({ url: repositoryCheckoutUrl, dir: repositoryTmpPath });
 
   return repositoryTmpPath;
 };
@@ -183,10 +206,11 @@ export const getLastCommitTimestamp = async (
     logger,
   );
 
-  const repository = await Repository.open(repositoryLocation);
-  const commit = await repository.getReferenceCommit('HEAD');
+  const git = Git.fromAuth({ logger });
+  const sha = await git.resolveRef({ dir: repositoryLocation, ref: 'HEAD' });
+  const commit = await git.readCommit({ dir: repositoryLocation, sha });
 
-  return commit.date().getTime();
+  return commit.commit.committer.timestamp;
 };
 
 export const getDocFilesFromRepository = async (