From 07995f2ea042bb05879948148c6edd4b21390a87 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 19 Nov 2020 04:09:02 +0100 Subject: [PATCH 01/33] chore(git): messing around with git libraries --- plugins/scaffolder-backend/package.json | 7 +- .../src/scaffolder/stages/prepare/github.ts | 48 ++++++++----- yarn.lock | 72 ++++++++++++++++--- 3 files changed, 94 insertions(+), 33 deletions(-) diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 4bed7ea60c..09c4e0cd87 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -32,6 +32,7 @@ "command-exists-promise": "^2.0.2", "compression": "^1.7.4", "cors": "^2.8.5", + "cross-fetch": "^3.0.6", "dockerode": "^3.2.0", "express": "^4.17.1", "express-promise-router": "^3.0.3", @@ -39,20 +40,18 @@ "git-url-parse": "^11.4.0", "globby": "^11.0.0", "helmet": "^4.0.0", + "isomorphic-git": "^1.8.0", "jsonschema": "^1.2.6", "morgan": "^1.10.0", - "nodegit": "0.27.0", "uuid": "^8.2.0", "winston": "^3.2.1", - "yaml": "^1.10.0", - "cross-fetch": "^3.0.6" + "yaml": "^1.10.0" }, "devDependencies": { "@backstage/cli": "^0.2.0", "@octokit/types": "^5.4.1", "@types/fs-extra": "^9.0.1", "@types/git-url-parse": "^9.0.0", - "@types/nodegit": "0.26.11", "@types/supertest": "^2.0.8", "supertest": "^4.0.2", "yaml": "^1.10.0" diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index e6b41907be..a4ad4afe32 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -21,7 +21,8 @@ import { parseLocationAnnotation } from '../helpers'; import { InputError } from '@backstage/backend-common'; import { PreparerBase, PreparerOptions } from './types'; import GitUriParser from 'git-url-parse'; -import { Clone, CloneOptions, Cred } from 'nodegit'; +import git from 'isomorphic-git'; +import http from 'isomorphic-git/http/node'; export class GithubPreparer implements PreparerBase { token?: string; @@ -56,25 +57,36 @@ export class GithubPreparer implements PreparerBase { template.spec.path ?? '.', ); - let cloneOptions: CloneOptions = { - checkoutBranch: parsedGitLocation.ref, - }; - - if (token) { - cloneOptions = { - ...cloneOptions, - fetchOpts: { - callbacks: { - credentials() { - return Cred.userpassPlaintextNew(token, 'x-oauth-basic'); - }, - }, + console.warn(repositoryCheckoutUrl); + const finalplace = path.resolve(tempDir, templateDirectory); + try { + await git.clone({ + fs: require('fs'), + http, + url: repositoryCheckoutUrl, + dir: finalplace, + // corsProxy: 'https://cors.isomorphic-git.org', + singleBranch: true, + // need this header + headers: { + 'user-agent': 'git/@isomorphic-git/cors-proxy', }, - }; + depth: 1, + }); + // onAuth: () => ({ username: token, password: 'x-oauth-basic' }), + } catch (ex) { + console.warn(ex.data.url); + console.warn({ + fs: require('fs'), + http, + url: repositoryCheckoutUrl, + dir: templateDirectory, + depth: 1, + onAuth: () => ({ username: token, password: 'x-oauth-basic' }), + }); + throw ex; } - await Clone.clone(repositoryCheckoutUrl, tempDir, cloneOptions); - - return path.resolve(tempDir, templateDirectory); + return finalplace; } } diff --git a/yarn.lock b/yarn.lock index feacd32d97..3450a410c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5550,13 +5550,6 @@ resolved "https://registry.npmjs.org/@types/node/-/node-13.13.15.tgz#fe1cc3aa465a3ea6858b793fd380b66c39919766" integrity sha512-kwbcs0jySLxzLsa2nWUAGOd/s21WU1jebrEdtzhsj1D4Yps1EOuyI1Qcu+FD56dL7NRNIJtDDjcqIG22NwkgLw== -"@types/nodegit@0.26.11": - version "0.26.11" - resolved "https://registry.npmjs.org/@types/nodegit/-/nodegit-0.26.11.tgz#0cbc5e929f23e5ffc536920e3a887e0b3f46d1a4" - integrity sha512-BGrY9F8lBtfU+Ne1Pjb9k/PUsfSCUAqPXgKkTkM6mc5213H5VAoM12zG/sz/cr3jI3AXcngNbAYWlqSrXsWJug== - dependencies: - "@types/node" "*" - "@types/normalize-package-data@^2.4.0": version "2.4.0" resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" @@ -7134,6 +7127,11 @@ async-limiter@~1.0.0: resolved "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd" integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ== +async-lock@^1.1.0: + version "1.2.4" + resolved "https://registry.npmjs.org/async-lock/-/async-lock-1.2.4.tgz#80d0d612383045dd0c30eb5aad08510c1397cb91" + integrity sha512-UBQJC2pbeyGutIfYmErGc9RaJYnpZ1FHaxuKwb0ahvGiiCkPUf3p67Io+YLPmmv3RHY+mF6JEtNW8FlHsraAaA== + async-retry@^1.2.1: version "1.3.1" resolved "https://registry.npmjs.org/async-retry/-/async-retry-1.3.1.tgz#139f31f8ddce50c0870b0ba558a6079684aaed55" @@ -8534,6 +8532,11 @@ clean-css@^4.2.3: dependencies: source-map "~0.6.0" +clean-git-ref@^2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/clean-git-ref/-/clean-git-ref-2.0.1.tgz#dcc0ca093b90e527e67adb5a5e55b1af6816dcd9" + integrity sha512-bLSptAy2P0s6hU4PzuIMKmMJJSE6gLXGH1cntDu7bWJUksvuM+7ReOK61mozULErYvP6a15rnYl0zFDef+pyPw== + clean-stack@^2.0.0: version "2.2.0" resolved "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" @@ -9284,6 +9287,14 @@ cosmiconfig@^7.0.0: path-type "^4.0.0" yaml "^1.10.0" +crc-32@^1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208" + integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA== + dependencies: + exit-on-epipe "~1.0.1" + printj "~1.1.0" + create-ecdh@^4.0.0: version "4.0.3" resolved "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz#c9111b6f33045c4697f144787f9254cdc77c45ff" @@ -10330,6 +10341,11 @@ diff-sequences@^26.5.0: resolved "https://registry.npmjs.org/diff-sequences/-/diff-sequences-26.5.0.tgz#ef766cf09d43ed40406611f11c6d8d9dd8b2fefd" integrity sha512-ZXx86srb/iYy6jG71k++wBN9P9J05UNQ5hQHQd9MtMPvcqXPx/vKU69jfHV637D00Q2gSgPk2D+jSx3l1lDW/Q== +diff3@0.0.3: + version "0.0.3" + resolved "https://registry.npmjs.org/diff3/-/diff3-0.0.3.tgz#d4e5c3a4cdf4e5fe1211ab42e693fcb4321580fc" + integrity sha1-1OXDpM305f4SEatC5pP8tDIVgPw= + diff@^4.0.1, diff@^4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" @@ -11407,6 +11423,11 @@ exit-hook@^1.0.0: resolved "https://registry.npmjs.org/exit-hook/-/exit-hook-1.1.1.tgz#f05ca233b48c05d54fff07765df8507e95c02ff8" integrity sha1-8FyiM7SMBdVP/wd2XfhQfpXAL/g= +exit-on-epipe@~1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692" + integrity sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw== + exit@^0.1.2: version "0.1.2" resolved "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" @@ -14360,6 +14381,23 @@ isomorphic-form-data@~2.0.0: dependencies: form-data "^2.3.2" +isomorphic-git@^1.8.0: + version "1.8.0" + resolved "https://registry.npmjs.org/isomorphic-git/-/isomorphic-git-1.8.0.tgz#50440650a64706a321cbea1af955c1cf1110b238" + integrity sha512-TWJvQh+++eFrEG0IFS/jLhMwsBoCOX1/Dsw9q8no59Mp1K0jEjSHXFWv2P04PwkxcIpePkXVBI5YFcFT2nkuQg== + dependencies: + async-lock "^1.1.0" + clean-git-ref "^2.0.1" + crc-32 "^1.2.0" + diff3 "0.0.3" + ignore "^5.1.4" + minimisted "^2.0.0" + pako "^1.0.10" + pify "^4.0.1" + readable-stream "^3.4.0" + sha.js "^2.4.9" + simple-get "^3.0.2" + isomorphic-ws@^4.0.1: version "4.0.1" resolved "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz#55fd4cd6c5e6491e76dc125938dd863f5cd4f2dc" @@ -16629,6 +16667,13 @@ minimist@^1.1.1, minimist@^1.1.3, minimist@^1.2.0, minimist@^1.2.5: resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== +minimisted@^2.0.0: + version "2.0.1" + resolved "https://registry.npmjs.org/minimisted/-/minimisted-2.0.1.tgz#d059fb905beecf0774bc3b308468699709805cb1" + integrity sha512-1oPjfuLQa2caorJUM8HV8lGgWCc0qqAO1MNv/k05G4qslmsndV/5WdNZrqCiyqiz3wohia2Ij2B7w2Dr7/IyrA== + dependencies: + minimist "^1.2.5" + minipass-collect@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz#22b813bf745dc6edba2576b940022ad6edc8c617" @@ -17203,7 +17248,7 @@ node-request-interceptor@^0.5.1: debug "^4.1.1" headers-utils "^1.2.0" -nodegit@0.27.0, nodegit@^0.27.0: +nodegit@^0.27.0: version "0.27.0" resolved "https://registry.npmjs.org/nodegit/-/nodegit-0.27.0.tgz#4e8cc236f60e1c97324a5acff99056fe116a6ebe" integrity sha512-E9K4gPjWiA0b3Tx5lfWCzG7Cvodi2idl3V5UD2fZrOrHikIfrN7Fc2kWLtMUqqomyoToYJLeIC8IV7xb1CYRLA== @@ -17950,7 +17995,7 @@ packet-reader@1.0.0: resolved "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74" integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ== -pako@~1.0.5: +pako@^1.0.10, pako@~1.0.5: version "1.0.11" resolved "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== @@ -19111,6 +19156,11 @@ pretty-hrtime@^1.0.3: resolved "https://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz#b7e3ea42435a4c9b2759d99e0f201eb195802ee1" integrity sha1-t+PqQkNaTJsnWdmeDyAesZWALuE= +printj@~1.1.0: + version "1.1.2" + resolved "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222" + integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ== + prismjs@^1.21.0, prismjs@^1.8.4, prismjs@~1.21.0: version "1.21.0" resolved "https://registry.npmjs.org/prismjs/-/prismjs-1.21.0.tgz#36c086ec36b45319ec4218ee164c110f9fc015a3" @@ -21270,7 +21320,7 @@ setprototypeof@1.2.0: resolved "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== -sha.js@^2.4.0, sha.js@^2.4.11, sha.js@^2.4.8: +sha.js@^2.4.0, sha.js@^2.4.11, sha.js@^2.4.8, sha.js@^2.4.9: version "2.4.11" resolved "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz#37a5cf0b81ecbc6943de109ba2960d1b26584ae7" integrity sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ== @@ -21382,7 +21432,7 @@ simple-concat@^1.0.0: resolved "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f" integrity sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q== -simple-get@^3.0.3: +simple-get@^3.0.2, simple-get@^3.0.3: version "3.1.0" resolved "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz#b45be062435e50d159540b576202ceec40b9c6b3" integrity sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA== From 572b61825989fc42c4013beb7f9077f3ddf4092e Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Dec 2020 13:44:32 +0100 Subject: [PATCH 02/33] chore(scaffolder): remove extra logging stuff --- .../src/scaffolder/stages/prepare/github.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index a4ad4afe32..1eef5397de 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -57,24 +57,23 @@ export class GithubPreparer implements PreparerBase { template.spec.path ?? '.', ); - console.warn(repositoryCheckoutUrl); - const finalplace = path.resolve(tempDir, templateDirectory); + const checkoutLocation = path.resolve(tempDir, templateDirectory); + try { await git.clone({ fs: require('fs'), http, url: repositoryCheckoutUrl, - dir: finalplace, - // corsProxy: 'https://cors.isomorphic-git.org', + dir: checkoutLocation, singleBranch: true, - // need this header - headers: { - 'user-agent': 'git/@isomorphic-git/cors-proxy', - }, depth: 1, + onAuth: () => ({ username: token, password: 'x-oauth-basic' }), }); - // onAuth: () => ({ username: token, password: 'x-oauth-basic' }), } catch (ex) { + opts.logger.error( + `Failed checking out repository:${repositoryCheckoutUrl}`, + ); + opts.logger.error(ex.message); console.warn(ex.data.url); console.warn({ fs: require('fs'), @@ -87,6 +86,6 @@ export class GithubPreparer implements PreparerBase { throw ex; } - return finalplace; + return checkoutLocation; } } From f393f300e565a50fc20fca8c9cf2ac2abe14e978 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Dec 2020 20:04:36 +0100 Subject: [PATCH 03/33] chore: refactiring the on progress callback so we can see more ata and failures --- .../src/scaffolder/stages/prepare/github.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index 1eef5397de..ff814c20ba 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -67,22 +67,22 @@ export class GithubPreparer implements PreparerBase { dir: checkoutLocation, singleBranch: true, depth: 1, + onProgress: event => { + const total = event.total + ? `${event.loaded / event.total}%` + : event.loaded; + opts.logger.info({ status: event.phase, total }); + }, + headers: { + 'user-agent': 'git/@isomorphic-git', + }, onAuth: () => ({ username: token, password: 'x-oauth-basic' }), }); } catch (ex) { opts.logger.error( - `Failed checking out repository:${repositoryCheckoutUrl}`, + `Failed checking out repository: ${repositoryCheckoutUrl}`, ); opts.logger.error(ex.message); - console.warn(ex.data.url); - console.warn({ - fs: require('fs'), - http, - url: repositoryCheckoutUrl, - dir: templateDirectory, - depth: 1, - onAuth: () => ({ username: token, password: 'x-oauth-basic' }), - }); throw ex; } From a5414403bd15232a231fd44ba67c233852d9b05c Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Dec 2020 21:07:20 +0100 Subject: [PATCH 04/33] chore: reworking more of the github flow so that we can move to isomorphic git instead of nodegit :() --- .../src/scaffolder/stages/prepare/github.ts | 2 +- .../src/scaffolder/stages/publish/github.ts | 12 ++-- .../src/scaffolder/stages/publish/helpers.ts | 68 ++++++++++--------- 3 files changed, 42 insertions(+), 40 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index ff814c20ba..f202c609cc 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -61,7 +61,7 @@ export class GithubPreparer implements PreparerBase { try { await git.clone({ - fs: require('fs'), + fs, http, url: repositoryCheckoutUrl, dir: checkoutLocation, diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts index d5542e8800..0306bb72e9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -16,7 +16,7 @@ import { PublisherBase } from './types'; import { Octokit } from '@octokit/rest'; -import { pushToRemoteUserPass } from './helpers'; +import { pushToRemoteCred } from './helpers'; import { JsonValue } from '@backstage/config'; import { RequiredTemplateValues } from '../templater'; @@ -51,12 +51,10 @@ export class GithubPublisher implements PublisherBase { directory: string; }): Promise<{ remoteUrl: string }> { const remoteUrl = await this.createRemote(values); - await pushToRemoteUserPass( - directory, - remoteUrl, - this.token, - 'x-oauth-basic', - ); + await pushToRemoteCred(directory, remoteUrl, { + username: this.token, + password: 'x-auth-basic', + }); return { remoteUrl }; } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts index 51f5a4c854..61e76a31be 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts @@ -13,43 +13,47 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Repository, Remote, Signature, Cred } from 'nodegit'; + +import git from 'isomorphic-git'; +import globby from 'globby'; +import fs from 'fs'; +import http from 'isomorphic-git/http/node'; export async function pushToRemoteCred( - directory: string, + dir: string, remote: string, - credentialsProvider: () => Cred, + auth?: { username: string; password: string }, ): Promise { - const repo = await Repository.init(directory, 0); - const index = await repo.refreshIndex(); - await index.addAll(); - await index.write(); - const oid = await index.writeTree(); - await repo.createCommit( - 'HEAD', - Signature.now('Scaffolder', 'scaffolder@backstage.io'), - Signature.now('Scaffolder', 'scaffolder@backstage.io'), - 'initial commit', - oid, - [], - ); + await git.init({ + fs, + dir, + }); - const remoteRepo = await Remote.create(repo, 'origin', remote); + const paths = await globby(['./**', './**/.*'], { gitignore: true }); + for (const filepath of paths) { + await git.add({ fs, dir, filepath }); + } - await remoteRepo.push(['refs/heads/master:refs/heads/master'], { - callbacks: { - credentials: credentialsProvider, - }, + await git.commit({ + fs, + dir, + message: 'Initial commit', + author: { name: 'Scaffolder', email: 'scaffolder@backstage.io' }, + committer: { name: 'Scaffolder', email: 'scaffolder@backstage.io' }, + }); + + await git.addRemote({ + fs, + dir, + remote: 'origin', + url: remote, + }); + + await git.push({ + fs, + dir, + http, + remote: 'origin', + onAuth: () => auth, }); } - -export async function pushToRemoteUserPass( - directory: string, - remote: string, - username: string, - password: string, -): Promise { - return pushToRemoteCred(directory, remote, () => - Cred.userpassPlaintextNew(username, password), - ); -} From d1f588a38b956128339b8833fa5b05dc2d9c17e1 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Dec 2020 21:36:35 +0100 Subject: [PATCH 05/33] chore: pushing works with github! --- .../src/scaffolder/stages/publish/github.ts | 1 + .../src/scaffolder/stages/publish/helpers.ts | 21 ++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts index 0306bb72e9..1b7852adcc 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -54,6 +54,7 @@ export class GithubPublisher implements PublisherBase { await pushToRemoteCred(directory, remoteUrl, { username: this.token, password: 'x-auth-basic', + token: this.token, }); return { remoteUrl }; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts index 61e76a31be..8e76649989 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts @@ -19,17 +19,28 @@ import globby from 'globby'; import fs from 'fs'; import http from 'isomorphic-git/http/node'; +/* +username password +GitHub | token 'x-oauth-basic' +GitHub App | token 'x-access-token' +BitBucket | 'x-token-auth' token +GitLab | 'oauth2' token +From : https://isomorphic-git.org/docs/en/onAuth +*/ export async function pushToRemoteCred( dir: string, remote: string, - auth?: { username: string; password: string }, + auth?: { username: string; password: string; token: string }, ): Promise { await git.init({ fs, dir, }); - const paths = await globby(['./**', './**/.*'], { gitignore: true }); + const paths = await globby(['./**', './**/.*'], { + cwd: dir, + gitignore: true, + }); for (const filepath of paths) { await git.add({ fs, dir, filepath }); } @@ -49,11 +60,15 @@ export async function pushToRemoteCred( url: remote, }); + console.warn({ username: auth.token, password: 'x-oauth-basic' }); await git.push({ fs, dir, http, + headers: { + 'user-agent': 'git/@isomorphic-git', + }, remote: 'origin', - onAuth: () => auth, + onAuth: () => ({ username: auth.token, password: 'x-oauth-basic' }), }); } From f9f4846a58876d9b2f9bdc99497713309d140296 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 4 Dec 2020 18:10:39 +0100 Subject: [PATCH 06/33] chore: add back nodegit for testing --- plugins/scaffolder-backend/package.json | 2 ++ yarn.lock | 23 ++++++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 09c4e0cd87..ce740f0ec2 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -43,6 +43,7 @@ "isomorphic-git": "^1.8.0", "jsonschema": "^1.2.6", "morgan": "^1.10.0", + "nodegit": "^0.27.0", "uuid": "^8.2.0", "winston": "^3.2.1", "yaml": "^1.10.0" @@ -52,6 +53,7 @@ "@octokit/types": "^5.4.1", "@types/fs-extra": "^9.0.1", "@types/git-url-parse": "^9.0.0", + "@types/nodegit": "^0.26.12", "@types/supertest": "^2.0.8", "supertest": "^4.0.2", "yaml": "^1.10.0" diff --git a/yarn.lock b/yarn.lock index 3450a410c2..26bf43335c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1290,40 +1290,34 @@ to-fast-properties "^2.0.0" "@backstage/core@^0.2.0": - version "0.3.0" + version "0.2.0" + resolved "https://registry.npmjs.org/@backstage/core/-/core-0.2.0.tgz#543246b2d87563c9aa4d9fb96e40fdfc7e827520" + integrity sha512-75m2u3FoUngBOvt9l65xZcYTzzB+49OXpY1A9VNFUR1+jMs3cL/0HDfByQV2H0xXaHzMngQ8C5u/sWhkQsij1w== dependencies: "@backstage/config" "^0.1.1" - "@backstage/core-api" "^0.2.1" - "@backstage/theme" "^0.2.1" + "@backstage/core-api" "^0.2.0" + "@backstage/theme" "^0.2.0" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" - "@types/dagre" "^0.7.44" "@types/react" "^16.9" "@types/react-sparklines" "^1.7.0" classnames "^2.2.6" clsx "^1.1.0" - d3-selection "^2.0.0" - d3-shape "^2.0.0" - d3-zoom "^2.0.0" - dagre "^0.8.5" immer "^7.0.9" lodash "^4.17.15" material-table "^1.69.1" prop-types "^15.7.2" - qs "^6.9.4" rc-progress "^3.0.0" react "^16.12.0" react-dom "^16.12.0" react-helmet "6.1.0" react-hook-form "^6.6.0" - react-markdown "^5.0.2" react-router "6.0.0-beta.0" react-router-dom "6.0.0-beta.0" react-sparklines "^1.7.0" react-syntax-highlighter "^13.5.1" react-use "^15.3.3" - remark-gfm "^1.0.0" "@bcoe/v8-coverage@^0.2.3": version "0.2.3" @@ -5550,6 +5544,13 @@ resolved "https://registry.npmjs.org/@types/node/-/node-13.13.15.tgz#fe1cc3aa465a3ea6858b793fd380b66c39919766" integrity sha512-kwbcs0jySLxzLsa2nWUAGOd/s21WU1jebrEdtzhsj1D4Yps1EOuyI1Qcu+FD56dL7NRNIJtDDjcqIG22NwkgLw== +"@types/nodegit@^0.26.12": + version "0.26.12" + resolved "https://registry.npmjs.org/@types/nodegit/-/nodegit-0.26.12.tgz#93afb4cb85d3a48d392c3232699c9c07d8251a36" + integrity sha512-4YpeTImFZNJ1cve4lEueHFVS8rAs8XpZqlmx+Bm9bMc+XMiCrcwaUf6peN7pod7Rl3esVlGP1zdBB7Z12eMVAA== + dependencies: + "@types/node" "*" + "@types/normalize-package-data@^2.4.0": version "2.4.0" resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" From 15e5d6a76398eb414b8d41e533eb1aa1f392c710 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 7 Dec 2020 13:26:02 +0100 Subject: [PATCH 07/33] feat: use better logging for the publish step and the new helper with isomorphic-git --- .../src/scaffolder/stages/prepare/github.ts | 4 ++-- .../src/scaffolder/stages/publish/github.ts | 6 +++--- .../src/scaffolder/stages/publish/helpers.ts | 19 +++++++++++++++---- .../src/scaffolder/stages/publish/types.ts | 2 ++ 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index f202c609cc..8e5581d63d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -69,9 +69,9 @@ export class GithubPreparer implements PreparerBase { depth: 1, onProgress: event => { const total = event.total - ? `${event.loaded / event.total}%` + ? `${Math.round((event.loaded / event.total) * 100)}%` : event.loaded; - opts.logger.info({ status: event.phase, total }); + opts.logger.info(`status={${event.phase},total={${total}}}`); }, headers: { 'user-agent': 'git/@isomorphic-git', diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts index 612e676207..e262c59274 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -46,13 +46,13 @@ export class GithubPublisher implements PublisherBase { async publish({ values, directory, + logger, }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); - await pushToRemoteCred(directory, remoteUrl, { + await pushToRemoteCred(directory, remoteUrl, logger, { username: this.token, - password: 'x-auth-basic', - token: this.token, + password: 'x-oauth-basic', }); const catalogInfoUrl = remoteUrl.replace( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts index 8e76649989..39b1513d2e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts @@ -18,7 +18,7 @@ import git from 'isomorphic-git'; import globby from 'globby'; import fs from 'fs'; import http from 'isomorphic-git/http/node'; - +import { Logger } from 'winston'; /* username password GitHub | token 'x-oauth-basic' @@ -30,8 +30,10 @@ From : https://isomorphic-git.org/docs/en/onAuth export async function pushToRemoteCred( dir: string, remote: string, - auth?: { username: string; password: string; token: string }, + logger: Logger, + auth: { username: string; password: string }, ): Promise { + logger.info('Initializing Git Repo', dir); await git.init({ fs, dir, @@ -41,10 +43,13 @@ export async function pushToRemoteCred( cwd: dir, gitignore: true, }); + + logger.info('Adding files to repository', dir); for (const filepath of paths) { await git.add({ fs, dir, filepath }); } + logger.info('Creating commit', dir); await git.commit({ fs, dir, @@ -60,7 +65,7 @@ export async function pushToRemoteCred( url: remote, }); - console.warn({ username: auth.token, password: 'x-oauth-basic' }); + logger.info('Pushing code to remote', remote); await git.push({ fs, dir, @@ -68,7 +73,13 @@ export async function pushToRemoteCred( headers: { 'user-agent': 'git/@isomorphic-git', }, + onProgress: event => { + const total = event.total + ? `${Math.round((event.loaded / event.total) * 100)}%` + : event.loaded; + logger.info(`status={${event.phase},total={${total}}}`); + }, remote: 'origin', - onAuth: () => ({ username: auth.token, password: 'x-oauth-basic' }), + onAuth: () => auth, }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts index f3bc59e19d..bc6b63cc57 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts @@ -17,6 +17,7 @@ import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { RequiredTemplateValues } from '../templater'; import { JsonValue } from '@backstage/config'; import { RemoteProtocol } from '../types'; +import { Logger } from 'winston'; /** * Publisher is in charge of taking a folder created by @@ -34,6 +35,7 @@ export type PublisherBase = { export type PublisherOptions = { values: RequiredTemplateValues & Record; + logger: Logger; directory: string; }; From e2beaf5940a2a453844d70817d7eecfff53e0bdf Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 17 Dec 2020 10:18:52 +0100 Subject: [PATCH 08/33] chore: create scm from auth --- packages/backend-common/src/scm/git.ts | 44 ++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index 5bd721acd5..acfe53c1f8 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -15,3 +15,47 @@ */ import git from 'isomorphic-git'; import http from 'isomorphic-git/http/node'; +import fs from 'fs-extra'; + +class SCM { + constructor( + private readonly config: { username: string; password: string }, + private readonly logger: Logger, + ) {} + + async clone({ url, dir }) { + return git.clone({ + fs, + http, + url, + dir, + singleBranch: true, + depth: 1, + onProgress: event => { + const total = event.total + ? `${Math.round((event.loaded / event.total) * 100)}%` + : event.loaded; + this.logger.info(`status={${event.phase},total={${total}}}`); + }, + headers: { + 'user-agent': 'git/@isomorphic-git', + }, + onAuth: () => ({ + username: this.config.username, + password: this.config.password, + }), + }); + } +} + +export const fromAuth = ({ + username, + password, + logger, +}: { + username: string; + password: string; + logger: Logger; +}) => { + return new SCM({ username, password }); +}; From 70ceab3db447e0df58fd44d5daba2356ece14f77 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 21 Dec 2020 15:38:53 +0100 Subject: [PATCH 09/33] feat: adding in all the functions needed for isomorphic git feature --- packages/backend-common/package.json | 1 + packages/backend-common/src/scm/git.ts | 90 ++++++++++++++++++++++++-- yarn.lock | 8 ++- 3 files changed, 91 insertions(+), 8 deletions(-) diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 13b827762c..db5750dd70 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -45,6 +45,7 @@ "fs-extra": "^9.0.1", "git-url-parse": "^11.4.0", "helmet": "^4.0.0", + "isomorphic-git": "^1.8.0", "knex": "^0.21.6", "lodash": "^4.17.15", "logform": "^2.1.1", diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index acfe53c1f8..eecfd36612 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -16,14 +16,19 @@ import git from 'isomorphic-git'; import http from 'isomorphic-git/http/node'; import fs from 'fs-extra'; +import { Logger } from 'winston'; class SCM { constructor( - private readonly config: { username: string; password: string }, - private readonly logger: Logger, + private readonly config: { + username: string; + password: string; + logger?: Logger; + }, ) {} - async clone({ url, dir }) { + async clone({ url, dir }: { url: string; dir: string }) { + this.config.logger?.info(`Cloning repo {dir=${dir},url=${url}}`); return git.clone({ fs, http, @@ -35,7 +40,7 @@ class SCM { const total = event.total ? `${Math.round((event.loaded / event.total) * 100)}%` : event.loaded; - this.logger.info(`status={${event.phase},total={${total}}}`); + this.config.logger?.info(`status={${event.phase},total={${total}}}`); }, headers: { 'user-agent': 'git/@isomorphic-git', @@ -46,6 +51,79 @@ class SCM { }), }); } + + async init({ dir }: { dir: string }) { + this.config.logger?.info(`Init git repository {dir=${dir}}`); + + return git.init({ + fs, + dir, + }); + } + + async add({ dir, filepath }: { dir: string; filepath: string }) { + this.config.logger?.info(`Adding file {dir=${dir},filepath=${filepath}}`); + + return git.add({ fs, dir, filepath }); + } + + async commit({ + dir, + message, + author, + committer, + }: { + dir: string; + message: string; + author: { name: string; email: string }; + committer: { name: string; email: string }; + }) { + this.config.logger?.info( + `Committing file to repo {dir=${dir},message=${message}}`, + ); + + return git.commit({ fs, dir, message, author, committer }); + } + + async addRemote({ + dir, + url, + remoteName, + }: { + dir: string; + remoteName: string; + url: string; + }) { + this.config.logger?.info( + `Creating new remote {dir=${dir},remoteName=${remoteName},url=${url}}`, + ); + return git.addRemote({ fs, dir, remote: remoteName, url }); + } + + async push({ dir, remoteName }: { dir: string; remoteName: string }) { + this.config.logger?.info( + `Pushing directory to remote {dir=${dir},remoteName=${remoteName}}`, + ); + git.push({ + fs, + dir, + http, + onProgress: event => { + const total = event.total + ? `${Math.round((event.loaded / event.total) * 100)}%` + : event.loaded; + this.config.logger?.info(`status={${event.phase},total={${total}}}`); + }, + headers: { + 'user-agent': 'git/@isomorphic-git', + }, + remote: remoteName, + onAuth: () => ({ + username: this.config.username, + password: this.config.password, + }), + }); + } } export const fromAuth = ({ @@ -55,7 +133,7 @@ export const fromAuth = ({ }: { username: string; password: string; - logger: Logger; + logger?: Logger; }) => { - return new SCM({ username, password }); + return new SCM({ username, password, logger }); }; diff --git a/yarn.lock b/yarn.lock index f16e1326ab..4cb027e631 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1630,7 +1630,9 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.2.0": - version "0.4.0" + version "0.2.0" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.2.0.tgz#e3fe2a4ddeb6a9b6ec480c80cb2b9c39cb245576" + integrity sha512-Y1ocdRpBlxK/VrJQjHlQd0bgADECd1B2NRjwd8ss46ibT5hwLvMOfD80+Fa7oPLu0ktJrH4lq0pNIIJIml48zA== dependencies: "@backstage/config" "^0.1.1" "@types/json-schema" "^7.0.5" @@ -1641,7 +1643,9 @@ yup "^0.29.3" "@backstage/catalog-model@^0.3.0": - version "0.4.0" + version "0.3.1" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.3.1.tgz#45d08e2f333c9c566b2bf2629fd707fe989bb404" + integrity sha512-9XhV7c4rmVW+Yzj2PiwTQ7DsegWGB3C4ELsDRExuEVZONdqNcC02cyJtrt3fT5F31ZS3tHkB9bMUymFOBLqUSA== dependencies: "@backstage/config" "^0.1.1" "@types/json-schema" "^7.0.5" From d0a972eb63bb727f321504e485b5d8730740453d Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 21 Dec 2020 16:05:10 +0100 Subject: [PATCH 10/33] chore: reworking how we deal with exporting things from the main package --- packages/backend-common/src/index.ts | 1 + packages/backend-common/src/scm/git.ts | 23 +++-- packages/backend-common/src/scm/index.ts | 16 ++++ .../src/scaffolder/stages/prepare/github.ts | 40 +++------ .../src/scaffolder/stages/publish/github.ts | 17 +++- .../src/scaffolder/stages/publish/helpers.ts | 85 ------------------- 6 files changed, 57 insertions(+), 125 deletions(-) create mode 100644 packages/backend-common/src/scm/index.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index e968edef69..3f04f32b34 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -24,3 +24,4 @@ export * from './reading'; export * from './service'; export * from './paths'; export * from './hot'; +export * from './scm'; diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index eecfd36612..25115bab26 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -18,11 +18,19 @@ import http from 'isomorphic-git/http/node'; import fs from 'fs-extra'; import { Logger } from 'winston'; +/* +provider username password +GitHub token 'x-oauth-basic' +GitHub App token 'x-access-token' +BitBucket 'x-token-auth' token +GitLab 'oauth2' token +From : https://isomorphic-git.org/docs/en/onAuth +*/ class SCM { constructor( private readonly config: { - username: string; - password: string; + username?: string; + password?: string; logger?: Logger; }, ) {} @@ -126,14 +134,15 @@ class SCM { } } +// TODO(blam): This could potentially become something like for URL +// and use the integrations config for URLReading instead. +// But for now, I don't want to do all that in this PR. export const fromAuth = ({ username, password, logger, }: { - username: string; - password: string; + username?: string; + password?: string; logger?: Logger; -}) => { - return new SCM({ username, password, logger }); -}; +}) => new SCM({ username, password, logger }); diff --git a/packages/backend-common/src/scm/index.ts b/packages/backend-common/src/scm/index.ts new file mode 100644 index 0000000000..76af535fd0 --- /dev/null +++ b/packages/backend-common/src/scm/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * as Git from './git'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index 8e5581d63d..761259c0c5 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -18,11 +18,9 @@ import fs from 'fs-extra'; import path from 'path'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { parseLocationAnnotation } from '../helpers'; -import { InputError } from '@backstage/backend-common'; +import { InputError, Git } from '@backstage/backend-common'; import { PreparerBase, PreparerOptions } from './types'; import GitUriParser from 'git-url-parse'; -import git from 'isomorphic-git'; -import http from 'isomorphic-git/http/node'; export class GithubPreparer implements PreparerBase { token?: string; @@ -37,7 +35,6 @@ export class GithubPreparer implements PreparerBase { ): Promise { const { protocol, location } = parseLocationAnnotation(template); const workingDirectory = opts?.workingDirectory ?? os.tmpdir(); - const { token } = this; if (!['github', 'url'].includes(protocol)) { throw new InputError( @@ -58,33 +55,16 @@ export class GithubPreparer implements PreparerBase { ); const checkoutLocation = path.resolve(tempDir, templateDirectory); + const git = Git.fromAuth({ + username: this.token, + password: 'x-oauth-basic', + logger: opts.logger, + }); - try { - await git.clone({ - fs, - http, - url: repositoryCheckoutUrl, - dir: checkoutLocation, - singleBranch: true, - depth: 1, - onProgress: event => { - const total = event.total - ? `${Math.round((event.loaded / event.total) * 100)}%` - : event.loaded; - opts.logger.info(`status={${event.phase},total={${total}}}`); - }, - headers: { - 'user-agent': 'git/@isomorphic-git', - }, - onAuth: () => ({ username: token, password: 'x-oauth-basic' }), - }); - } catch (ex) { - opts.logger.error( - `Failed checking out repository: ${repositoryCheckoutUrl}`, - ); - opts.logger.error(ex.message); - throw ex; - } + await git.clone({ + url: repositoryCheckoutUrl, + dir: checkoutLocation, + }); return checkoutLocation; } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts index e262c59274..6e99590e15 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -16,7 +16,7 @@ import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { Octokit } from '@octokit/rest'; -import { pushToRemoteCred } from './helpers'; +import { Git } from '@backstage/backend-common'; import { JsonValue } from '@backstage/config'; import { RequiredTemplateValues } from '../templater'; @@ -49,10 +49,21 @@ export class GithubPublisher implements PublisherBase { logger, }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); - - await pushToRemoteCred(directory, remoteUrl, logger, { + const git = Git.fromAuth({ username: this.token, password: 'x-oauth-basic', + logger, + }); + + await git.addRemote({ + dir: directory, + url: remoteUrl, + remoteName: 'origin', + }); + + await git.push({ + dir: directory, + remoteName: 'origin', }); const catalogInfoUrl = remoteUrl.replace( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts deleted file mode 100644 index 1dcf3fdee1..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import git from 'isomorphic-git'; -import globby from 'globby'; -import fs from 'fs'; -import http from 'isomorphic-git/http/node'; -import { Logger } from 'winston'; -/* -username password -GitHub | token 'x-oauth-basic' -GitHub App | token 'x-access-token' -BitBucket | 'x-token-auth' token -GitLab | 'oauth2' token -From : https://isomorphic-git.org/docs/en/onAuth -*/ -export async function push( - dir: string, - remote: string, - logger: Logger, - auth: { username: string; password: string }, -): Promise { - logger.info('Initializing Git Repo', dir); - await git.init({ - fs, - dir, - }); - - const paths = await globby(['./**', './**/.*'], { - cwd: dir, - gitignore: true, - }); - - logger.info('Adding files to repository', dir); - for (const filepath of paths) { - await git.add({ fs, dir, filepath }); - } - - logger.info('Creating commit', dir); - await git.commit({ - fs, - dir, - message: 'Initial commit', - author: { name: 'Scaffolder', email: 'scaffolder@backstage.io' }, - committer: { name: 'Scaffolder', email: 'scaffolder@backstage.io' }, - }); - - await git.addRemote({ - fs, - dir, - remote: 'origin', - url: remote, - }); - - logger.info('Pushing code to remote', remote); - await git.push({ - fs, - dir, - http, - headers: { - 'user-agent': 'git/@isomorphic-git', - }, - onProgress: event => { - const total = event.total - ? `${Math.round((event.loaded / event.total) * 100)}%` - : event.loaded; - logger.info(`status={${event.phase},total={${total}}}`); - }, - remote: 'origin', - onAuth: () => auth, - }); -} From 54a19d444497a2c53bdeefe1dba64ccaf2ac66cd Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 21 Dec 2020 16:05:24 +0100 Subject: [PATCH 11/33] feat: move all publishers over to the backend common way to interact with git scm --- packages/backend-common/src/scm/git.ts | 2 + packages/backend-common/src/scm/index.ts | 4 +- .../src/scaffolder/stages/publish/azure.ts | 21 ++++- .../src/scaffolder/stages/publish/gitlab.ts | 21 ++++- .../src/scaffolder/stages/publish/helpers.ts | 85 +++++++++++++++++++ 5 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index 25115bab26..7e31b4e8e1 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -25,6 +25,8 @@ GitHub App token 'x-access-token' BitBucket 'x-token-auth' token GitLab 'oauth2' token From : https://isomorphic-git.org/docs/en/onAuth + +Azure 'notempty' token */ class SCM { constructor( diff --git a/packages/backend-common/src/scm/index.ts b/packages/backend-common/src/scm/index.ts index 76af535fd0..83a59310cd 100644 --- a/packages/backend-common/src/scm/index.ts +++ b/packages/backend-common/src/scm/index.ts @@ -13,4 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export * as Git from './git'; +import * as Git from './git'; + +export { Git }; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts index 1e962bf223..16956cc8fb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts @@ -16,8 +16,8 @@ import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { GitApi } from 'azure-devops-node-api/GitApi'; +import { Git } from '@backstage/backend-common'; import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; -import { pushToRemoteUserPass } from './helpers'; import { JsonValue } from '@backstage/config'; import { RequiredTemplateValues } from '../templater'; @@ -33,11 +33,28 @@ export class AzurePublisher implements PublisherBase { async publish({ values, directory, + logger, }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); - await pushToRemoteUserPass(directory, remoteUrl, 'notempty', this.token); const catalogInfoUrl = `${remoteUrl}?path=%2Fcatalog-info.yaml`; + const git = Git.fromAuth({ + username: 'notempty', + password: this.token, + logger, + }); + + await git.addRemote({ + dir: directory, + url: remoteUrl, + remoteName: 'origin', + }); + + await git.push({ + dir: directory, + remoteName: 'origin', + }); + return { remoteUrl, catalogInfoUrl }; } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts index 4fae8e8a5c..32b203bf51 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts @@ -16,7 +16,7 @@ import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { Gitlab } from '@gitbeaker/core'; -import { pushToRemoteUserPass } from './helpers'; +import { Git } from '@backstage/backend-common'; import { JsonValue } from '@backstage/config'; import { RequiredTemplateValues } from '../templater'; @@ -32,9 +32,26 @@ export class GitlabPublisher implements PublisherBase { async publish({ values, directory, + logger, }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); - await pushToRemoteUserPass(directory, remoteUrl, 'oauth2', this.token); + + const git = Git.fromAuth({ + password: this.token, + username: 'oauth2', + logger, + }); + + await git.addRemote({ + dir: directory, + url: remoteUrl, + remoteName: 'origin', + }); + + await git.push({ + dir: directory, + remoteName: 'origin', + }); return { remoteUrl }; } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts new file mode 100644 index 0000000000..1dcf3fdee1 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts @@ -0,0 +1,85 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import git from 'isomorphic-git'; +import globby from 'globby'; +import fs from 'fs'; +import http from 'isomorphic-git/http/node'; +import { Logger } from 'winston'; +/* +username password +GitHub | token 'x-oauth-basic' +GitHub App | token 'x-access-token' +BitBucket | 'x-token-auth' token +GitLab | 'oauth2' token +From : https://isomorphic-git.org/docs/en/onAuth +*/ +export async function push( + dir: string, + remote: string, + logger: Logger, + auth: { username: string; password: string }, +): Promise { + logger.info('Initializing Git Repo', dir); + await git.init({ + fs, + dir, + }); + + const paths = await globby(['./**', './**/.*'], { + cwd: dir, + gitignore: true, + }); + + logger.info('Adding files to repository', dir); + for (const filepath of paths) { + await git.add({ fs, dir, filepath }); + } + + logger.info('Creating commit', dir); + await git.commit({ + fs, + dir, + message: 'Initial commit', + author: { name: 'Scaffolder', email: 'scaffolder@backstage.io' }, + committer: { name: 'Scaffolder', email: 'scaffolder@backstage.io' }, + }); + + await git.addRemote({ + fs, + dir, + remote: 'origin', + url: remote, + }); + + logger.info('Pushing code to remote', remote); + await git.push({ + fs, + dir, + http, + headers: { + 'user-agent': 'git/@isomorphic-git', + }, + onProgress: event => { + const total = event.total + ? `${Math.round((event.loaded / event.total) * 100)}%` + : event.loaded; + logger.info(`status={${event.phase},total={${total}}}`); + }, + remote: 'origin', + onAuth: () => auth, + }); +} From ab82709595acdeae1bfcc2ecacd7a7d203e086fb Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 21 Dec 2020 16:31:50 +0100 Subject: [PATCH 12/33] bug: need to actually use the tmpDirectory for checkout not the end location --- .../scaffolder-backend/src/scaffolder/stages/prepare/github.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index 761259c0c5..87612bb0fb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -55,6 +55,7 @@ export class GithubPreparer implements PreparerBase { ); const checkoutLocation = path.resolve(tempDir, templateDirectory); + const git = Git.fromAuth({ username: this.token, password: 'x-oauth-basic', @@ -63,7 +64,7 @@ export class GithubPreparer implements PreparerBase { await git.clone({ url: repositoryCheckoutUrl, - dir: checkoutLocation, + dir: tempDir, }); return checkoutLocation; From 3727e3b36739fc672f652bebd7bdc25b7798c620 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 22 Dec 2020 13:00:36 +0100 Subject: [PATCH 13/33] feat: update all preparers to use isomorphic git --- packages/backend-common/src/scm/git.ts | 2 +- plugins/scaffolder-backend/package.json | 1 - .../src/scaffolder/stages/prepare/azure.ts | 29 ++++++++++--------- .../src/scaffolder/stages/prepare/github.ts | 13 +++++---- .../src/scaffolder/stages/prepare/gitlab.ts | 25 ++++++++-------- .../src/scaffolder/stages/publish/github.ts | 22 ++++++++++++++ 6 files changed, 59 insertions(+), 33 deletions(-) diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index 7e31b4e8e1..3b31d7c60b 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -114,7 +114,7 @@ class SCM { this.config.logger?.info( `Pushing directory to remote {dir=${dir},remoteName=${remoteName}}`, ); - git.push({ + return git.push({ fs, dir, http, diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 992a9efc3b..a51e9edfcb 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -53,7 +53,6 @@ "isomorphic-git": "^1.8.0", "jsonschema": "^1.2.6", "morgan": "^1.10.0", - "nodegit": "^0.27.0", "uuid": "^8.2.0", "winston": "^3.2.1", "yaml": "^1.10.0" diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts index 333bf9116c..ce1c18e048 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts @@ -18,10 +18,9 @@ import fs from 'fs-extra'; import path from 'path'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { parseLocationAnnotation } from '../helpers'; -import { InputError } from '@backstage/backend-common'; +import { InputError, Git } from '@backstage/backend-common'; import { PreparerBase, PreparerOptions } from './types'; import GitUriParser from 'git-url-parse'; -import { Clone, Cred } from 'nodegit'; import { Config } from '@backstage/config'; export class AzurePreparer implements PreparerBase { @@ -38,6 +37,7 @@ export class AzurePreparer implements PreparerBase { ): Promise { const { protocol, location } = parseLocationAnnotation(template); const workingDirectory = opts?.workingDirectory ?? os.tmpdir(); + const { logger } = opts; if (!['azure/api', 'url'].includes(protocol)) { throw new InputError( @@ -57,19 +57,20 @@ export class AzurePreparer implements PreparerBase { template.spec.path ?? '.', ); - const options = this.privateToken - ? { - fetchOpts: { - callbacks: { - credentials: () => - // Username can anything but the empty string according to: https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat - Cred.userpassPlaintextNew('notempty', this.privateToken), - }, - }, - } - : {}; + // Username can anything but the empty string according to: + // https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat + const git = this.privateToken + ? Git.fromAuth({ + password: this.privateToken, + username: 'notempty', + logger, + }) + : Git.fromAuth({ logger }); - await Clone.clone(repositoryCheckoutUrl, tempDir, options); + await git.clone({ + url: repositoryCheckoutUrl, + dir: tempDir, + }); return path.resolve(tempDir, templateDirectory); } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index 87612bb0fb..e7fa0c9b72 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -35,6 +35,7 @@ export class GithubPreparer implements PreparerBase { ): Promise { const { protocol, location } = parseLocationAnnotation(template); const workingDirectory = opts?.workingDirectory ?? os.tmpdir(); + const { logger } = opts; if (!['github', 'url'].includes(protocol)) { throw new InputError( @@ -56,11 +57,13 @@ export class GithubPreparer implements PreparerBase { const checkoutLocation = path.resolve(tempDir, templateDirectory); - const git = Git.fromAuth({ - username: this.token, - password: 'x-oauth-basic', - logger: opts.logger, - }); + const git = this.token + ? Git.fromAuth({ + username: this.token, + password: 'x-oauth-basic', + logger, + }) + : Git.fromAuth({ logger }); await git.clone({ url: repositoryCheckoutUrl, diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts index c2c2dcd3fd..6e169baa98 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { InputError } from '@backstage/backend-common'; +import { InputError, Git } from '@backstage/backend-common'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { @@ -22,7 +22,6 @@ import { } from '@backstage/integration'; import fs from 'fs-extra'; import GitUriParser from 'git-url-parse'; -import { Clone, Cred } from 'nodegit'; import os from 'os'; import path from 'path'; import { parseLocationAnnotation } from '../helpers'; @@ -46,6 +45,7 @@ export class GitlabPreparer implements PreparerBase { opts: PreparerOptions, ): Promise { const { protocol, location } = parseLocationAnnotation(template); + const { logger } = opts; const workingDirectory = opts?.workingDirectory ?? os.tmpdir(); if (!['gitlab', 'gitlab/api', 'url'].includes(protocol)) { @@ -67,17 +67,18 @@ export class GitlabPreparer implements PreparerBase { ); const token = this.getToken(parsedGitLocation.resource); - const options = token - ? { - fetchOpts: { - callbacks: { - credentials: () => Cred.userpassPlaintextNew('oauth2', token), - }, - }, - } - : {}; + const git = token + ? Git.fromAuth({ + password: token, + username: 'oauth2', + logger, + }) + : Git.fromAuth({ logger }); - await Clone.clone(repositoryCheckoutUrl, tempDir, options); + await git.clone({ + url: repositoryCheckoutUrl, + dir: tempDir, + }); return path.resolve(tempDir, templateDirectory); } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts index 6e99590e15..7abcee193c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -19,6 +19,7 @@ import { Octokit } from '@octokit/rest'; import { Git } from '@backstage/backend-common'; import { JsonValue } from '@backstage/config'; import { RequiredTemplateValues } from '../templater'; +import globby from 'globby'; export type RepoVisibilityOptions = 'private' | 'internal' | 'public'; @@ -55,6 +56,27 @@ export class GithubPublisher implements PublisherBase { logger, }); + await git.init({ + dir: directory, + }); + + const paths = await globby(['./**', './**/.*'], { + cwd: directory, + gitignore: true, + dot: true, + }); + + for (const filepath of paths) { + await git.add({ dir: directory, filepath }); + } + + await git.commit({ + dir: directory, + message: 'Initial commit', + author: { name: 'Scaffolder', email: 'scaffolder@backstage.io' }, + committer: { name: 'Scaffolder', email: 'scaffolder@backstage.io' }, + }); + await git.addRemote({ dir: directory, url: remoteUrl, From 1e7f526bd3e0a231c56f43d0329e37c4283d89e3 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 22 Dec 2020 13:09:27 +0100 Subject: [PATCH 14/33] feat: update all publishers to use isomorphic-gint --- .../src/scaffolder/stages/publish/azure.ts | 23 +++---- .../src/scaffolder/stages/publish/github.ts | 46 +++---------- .../src/scaffolder/stages/publish/gitlab.ts | 23 +++---- .../src/scaffolder/stages/publish/helpers.ts | 64 +++++++------------ 4 files changed, 49 insertions(+), 107 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts index 16956cc8fb..ffe6845f01 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts @@ -16,10 +16,10 @@ import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { GitApi } from 'azure-devops-node-api/GitApi'; -import { Git } from '@backstage/backend-common'; import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; import { JsonValue } from '@backstage/config'; import { RequiredTemplateValues } from '../templater'; +import { initRepoAndPush } from './helpers'; export class AzurePublisher implements PublisherBase { private readonly client: GitApi; @@ -38,23 +38,16 @@ export class AzurePublisher implements PublisherBase { const remoteUrl = await this.createRemote(values); const catalogInfoUrl = `${remoteUrl}?path=%2Fcatalog-info.yaml`; - const git = Git.fromAuth({ - username: 'notempty', - password: this.token, + await initRepoAndPush({ + dir: directory, + remoteUrl, + auth: { + username: 'notempty', + password: this.token, + }, logger, }); - await git.addRemote({ - dir: directory, - url: remoteUrl, - remoteName: 'origin', - }); - - await git.push({ - dir: directory, - remoteName: 'origin', - }); - return { remoteUrl, catalogInfoUrl }; } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts index 7abcee193c..39a588882f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -16,10 +16,9 @@ import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { Octokit } from '@octokit/rest'; -import { Git } from '@backstage/backend-common'; +import { initRepoAndPush } from './helpers'; import { JsonValue } from '@backstage/config'; import { RequiredTemplateValues } from '../templater'; -import globby from 'globby'; export type RepoVisibilityOptions = 'private' | 'internal' | 'public'; @@ -50,44 +49,17 @@ export class GithubPublisher implements PublisherBase { logger, }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); - const git = Git.fromAuth({ - username: this.token, - password: 'x-oauth-basic', + + await initRepoAndPush({ + dir: directory, + remoteUrl, + auth: { + username: this.token, + password: 'x-oauth-basic', + }, logger, }); - await git.init({ - dir: directory, - }); - - const paths = await globby(['./**', './**/.*'], { - cwd: directory, - gitignore: true, - dot: true, - }); - - for (const filepath of paths) { - await git.add({ dir: directory, filepath }); - } - - await git.commit({ - dir: directory, - message: 'Initial commit', - author: { name: 'Scaffolder', email: 'scaffolder@backstage.io' }, - committer: { name: 'Scaffolder', email: 'scaffolder@backstage.io' }, - }); - - await git.addRemote({ - dir: directory, - url: remoteUrl, - remoteName: 'origin', - }); - - await git.push({ - dir: directory, - remoteName: 'origin', - }); - const catalogInfoUrl = remoteUrl.replace( /\.git$/, '/blob/master/catalog-info.yaml', diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts index 32b203bf51..156a4a249c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts @@ -16,8 +16,8 @@ import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { Gitlab } from '@gitbeaker/core'; -import { Git } from '@backstage/backend-common'; import { JsonValue } from '@backstage/config'; +import { initRepoAndPush } from './helpers'; import { RequiredTemplateValues } from '../templater'; export class GitlabPublisher implements PublisherBase { @@ -36,23 +36,16 @@ export class GitlabPublisher implements PublisherBase { }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); - const git = Git.fromAuth({ - password: this.token, - username: 'oauth2', + await initRepoAndPush({ + dir: directory, + remoteUrl, + auth: { + username: 'oauth2', + password: this.token, + }, logger, }); - await git.addRemote({ - dir: directory, - url: remoteUrl, - remoteName: 'origin', - }); - - await git.push({ - dir: directory, - remoteName: 'origin', - }); - return { remoteUrl }; } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts index 1dcf3fdee1..bf19c2f56d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts @@ -14,44 +14,42 @@ * limitations under the License. */ -import git from 'isomorphic-git'; import globby from 'globby'; -import fs from 'fs'; -import http from 'isomorphic-git/http/node'; import { Logger } from 'winston'; -/* -username password -GitHub | token 'x-oauth-basic' -GitHub App | token 'x-access-token' -BitBucket | 'x-token-auth' token -GitLab | 'oauth2' token -From : https://isomorphic-git.org/docs/en/onAuth -*/ -export async function push( - dir: string, - remote: string, - logger: Logger, - auth: { username: string; password: string }, -): Promise { - logger.info('Initializing Git Repo', dir); +import { Git } from '@backstage/backend-common'; + +export async function initRepoAndPush({ + dir, + remoteUrl, + auth, + logger, +}: { + dir: string; + remoteUrl: string; + auth: { username: string; password: string }; + logger: Logger; +}): Promise { + const git = Git.fromAuth({ + username: auth.username, + password: auth.password, + logger, + }); + await git.init({ - fs, dir, }); const paths = await globby(['./**', './**/.*'], { cwd: dir, gitignore: true, + dot: true, }); - logger.info('Adding files to repository', dir); for (const filepath of paths) { - await git.add({ fs, dir, filepath }); + await git.add({ dir, filepath }); } - logger.info('Creating commit', dir); await git.commit({ - fs, dir, message: 'Initial commit', author: { name: 'Scaffolder', email: 'scaffolder@backstage.io' }, @@ -59,27 +57,13 @@ export async function push( }); await git.addRemote({ - fs, dir, - remote: 'origin', - url: remote, + url: remoteUrl, + remoteName: 'origin', }); - logger.info('Pushing code to remote', remote); await git.push({ - fs, dir, - http, - headers: { - 'user-agent': 'git/@isomorphic-git', - }, - onProgress: event => { - const total = event.total - ? `${Math.round((event.loaded / event.total) * 100)}%` - : event.loaded; - logger.info(`status={${event.phase},total={${total}}}`); - }, - remote: 'origin', - onAuth: () => auth, + remoteName: 'origin', }); } From 9ca586e579feb2a450b48246801506f6db485c27 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 23 Dec 2020 00:12:20 +0100 Subject: [PATCH 15/33] techdocs: Replace nodegit with isomorphic-git TechDocs needs 4 functions from the git library 1. To clone 2. To get the commit timestamp for HEAD 3. To fetch 4. To merge two branches Created necessary commands in the new SCM client and reordered them alphabetically for convenience --- packages/backend-common/src/scm/git.ts | 122 +++++++++--- packages/techdocs-common/package.json | 2 - packages/techdocs-common/src/helpers.ts | 89 +++++---- .../src/scaffolder/stages/prepare/azure.ts | 2 +- yarn.lock | 182 +----------------- 5 files changed, 154 insertions(+), 243 deletions(-) diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index 3b31d7c60b..b744040cb2 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -37,6 +37,45 @@ class SCM { }, ) {} + async add({ dir, filepath }: { dir: string; filepath: string }) { + this.config.logger?.info(`Adding file {dir=${dir},filepath=${filepath}}`); + + return git.add({ fs, dir, filepath }); + } + + async addRemote({ + dir, + url, + remoteName, + }: { + dir: string; + remoteName: string; + url: string; + }) { + this.config.logger?.info( + `Creating new remote {dir=${dir},remoteName=${remoteName},url=${url}}`, + ); + return git.addRemote({ fs, dir, remote: remoteName, url }); + } + + async commit({ + dir, + message, + author, + committer, + }: { + dir: string; + message: string; + author: { name: string; email: string }; + committer: { name: string; email: string }; + }) { + this.config.logger?.info( + `Committing file to repo {dir=${dir},message=${message}}`, + ); + + return git.commit({ fs, dir, message, author, committer }); + } + async clone({ url, dir }: { url: string; dir: string }) { this.config.logger?.info(`Cloning repo {dir=${dir},url=${url}}`); return git.clone({ @@ -62,6 +101,39 @@ class SCM { }); } + // https://isomorphic-git.org/docs/en/currentBranch + async currentBranch({ dir, fullName }: { dir: string; fullName?: boolean }) { + const fullname = fullName ?? false; + return git.currentBranch({ fs, dir, fullname }); + } + + // https://isomorphic-git.org/docs/en/fetch + async fetch({ dir, remote }: { dir: string; remote?: string }) { + const remoteValue = remote ?? 'origin'; + this.config.logger?.info( + `Fetching remote=${remoteValue} for repository {dir=${dir}}`, + ); + return git.fetch({ + fs, + http, + dir, + remote: remoteValue, + onProgress: event => { + const total = event.total + ? `${Math.round((event.loaded / event.total) * 100)}%` + : event.loaded; + this.config.logger?.info(`status={${event.phase},total={${total}}}`); + }, + headers: { + 'user-agent': 'git/@isomorphic-git', + }, + onAuth: () => ({ + username: this.config.username, + password: this.config.password, + }), + }); + } + async init({ dir }: { dir: string }) { this.config.logger?.info(`Init git repository {dir=${dir}}`); @@ -71,43 +143,21 @@ class SCM { }); } - async add({ dir, filepath }: { dir: string; filepath: string }) { - this.config.logger?.info(`Adding file {dir=${dir},filepath=${filepath}}`); - - return git.add({ fs, dir, filepath }); - } - - async commit({ + // https://isomorphic-git.org/docs/en/merge + async merge({ dir, - message, - author, - committer, + headBranch, + baseBranch, }: { dir: string; - message: string; - author: { name: string; email: string }; - committer: { name: string; email: string }; + headBranch: string; + baseBranch?: string; }) { this.config.logger?.info( - `Committing file to repo {dir=${dir},message=${message}}`, + `Merging branch '${headBranch}' into '${baseBranch}' for repository {dir=${dir}}`, ); - - return git.commit({ fs, dir, message, author, committer }); - } - - async addRemote({ - dir, - url, - remoteName, - }: { - dir: string; - remoteName: string; - url: string; - }) { - this.config.logger?.info( - `Creating new remote {dir=${dir},remoteName=${remoteName},url=${url}}`, - ); - return git.addRemote({ fs, dir, remote: remoteName, url }); + // If baseBranch is undefined, current branch is used. + return git.merge({ fs, dir, ours: baseBranch, theirs: headBranch }); } async push({ dir, remoteName }: { dir: string; remoteName: string }) { @@ -134,6 +184,16 @@ class SCM { }), }); } + + // https://isomorphic-git.org/docs/en/readCommit + async readCommit({ dir, sha }: { dir: string; sha: string }) { + return git.readCommit({ fs, dir, oid: sha }); + } + + // https://isomorphic-git.org/docs/en/resolveRef + async resolveRef({ dir, ref }: { dir: string; ref: string }) { + return git.resolveRef({ fs, dir, ref }); + } } // TODO(blam): This could potentially become something like for URL diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index d0c09919a0..632dbc2b66 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -50,7 +50,6 @@ "js-yaml": "^3.14.0", "mime-types": "^2.1.27", "mock-fs": "^4.13.0", - "nodegit": "^0.27.0", "recursive-readdir": "^2.2.2", "winston": "^3.2.1" }, @@ -61,7 +60,6 @@ "@types/js-yaml": "^3.12.5", "@types/mime-types": "^2.1.0", "@types/mock-fs": "^4.13.0", - "@types/nodegit": "^0.26.12", "@types/recursive-readdir": "^2.2.0" }, "jest": { diff --git a/packages/techdocs-common/src/helpers.ts b/packages/techdocs-common/src/helpers.ts index e08d58d58e..749c91adc0 100644 --- a/packages/techdocs-common/src/helpers.ts +++ b/packages/techdocs-common/src/helpers.ts @@ -17,19 +17,14 @@ import os from 'os'; import path from 'path'; import parseGitUrl from 'git-url-parse'; -import NodeGit, { Clone, Repository } from 'nodegit'; import fs from 'fs-extra'; import { getDefaultBranch } from './default-branch'; import { getGitRepoType, getTokenForGitRepo } from './git-auth'; import { Entity } from '@backstage/catalog-model'; -import { InputError, UrlReader } from '@backstage/backend-common'; +import { InputError, UrlReader, Git } from '@backstage/backend-common'; import { RemoteProtocol } from './stages/prepare/types'; import { Logger } from 'winston'; -// Enables core.longpaths on windows to prevent crashing when checking out repos with long foldernames and/or deep nesting -// @ts-ignore -NodeGit.Libgit2.opts(28, 1); - export type ParsedLocationAnnotation = { type: RemoteProtocol; target: string; @@ -124,17 +119,56 @@ export const checkoutGitRepository = async ( const repositoryTmpPath = await getGitRepositoryTempFolder(repoUrl); const token = await getTokenForGitRepo(repoUrl); + // Initialize a git client + let git = Git.fromAuth({ logger }); + + // Docs about why username and password are set to these specific values. + // https://isomorphic-git.org/docs/en/onAuth#oauth2-tokens + if (token) { + const type = getGitRepoType(repoUrl); + switch (type) { + case 'github': + git = Git.fromAuth({ + username: token, + password: 'x-oauth-basic', + logger, + }); + parsedGitLocation.token = `${token}:x-oauth-basic`; + break; + case 'gitlab': + git = Git.fromAuth({ + username: 'oauth2', + password: token, + logger, + }); + parsedGitLocation.token = `dummyUsername:${token}`; + parsedGitLocation.git_suffix = true; + break; + case 'azure/api': + git = Git.fromAuth({ + username: 'notempty', + password: token, + logger: logger, + }); + break; + default: + parsedGitLocation.token = `:${token}`; + } + } + + // Pull from repository if it has already been cloned. if (fs.existsSync(repositoryTmpPath)) { try { - const repository = await Repository.open(repositoryTmpPath); - const currentBranchName = ( - await repository.getCurrentBranch() - ).shorthand(); - await repository.fetch('origin'); - await repository.mergeBranches( - currentBranchName, - `origin/${currentBranchName}`, - ); + const currentBranchName = await git.currentBranch({ + dir: repositoryTmpPath, + }); + + await git.fetch({ dir: repositoryTmpPath, remote: 'origin' }); + await git.merge({ + dir: repositoryTmpPath, + headBranch: `origin/${currentBranchName}`, + baseBranch: currentBranchName || undefined, + }); return repositoryTmpPath; } catch (e) { logger.info( @@ -144,26 +178,10 @@ export const checkoutGitRepository = async ( } } - if (token) { - const type = getGitRepoType(repoUrl); - switch (type) { - case 'gitlab': - // Personal Access Token - parsedGitLocation.token = `dummyUsername:${token}`; - parsedGitLocation.git_suffix = true; - break; - case 'github': - parsedGitLocation.token = `${token}:x-oauth-basic`; - break; - default: - parsedGitLocation.token = `:${token}`; - } - } - const repositoryCheckoutUrl = parsedGitLocation.toString('https'); fs.mkdirSync(repositoryTmpPath, { recursive: true }); - await Clone.clone(repositoryCheckoutUrl, repositoryTmpPath); + await git.clone({ url: repositoryCheckoutUrl, dir: repositoryTmpPath }); return repositoryTmpPath; }; @@ -174,10 +192,11 @@ export const getLastCommitTimestamp = async ( ): Promise => { const repositoryLocation = await checkoutGitRepository(repositoryUrl, logger); - const repository = await Repository.open(repositoryLocation); - const commit = await repository.getReferenceCommit('HEAD'); + const git = Git.fromAuth({ logger }); + const sha = await git.resolveRef({ dir: repositoryLocation, ref: 'HEAD' }); + const commit = await git.readCommit({ dir: repositoryLocation, sha }); - return commit.date().getTime(); + return commit.commit.committer.timestamp; }; export const getDocFilesFromRepository = async ( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts index ce1c18e048..a38c5f4fdc 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts @@ -57,7 +57,7 @@ export class AzurePreparer implements PreparerBase { template.spec.path ?? '.', ); - // Username can anything but the empty string according to: + // Username can be anything but the empty string according to: // https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat const git = this.privateToken ? Git.fromAuth({ diff --git a/yarn.lock b/yarn.lock index 99c691297a..43da6893ed 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4348,11 +4348,6 @@ resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ== -"@sindresorhus/is@^2.0.0": - version "2.1.1" - resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.1.tgz#ceff6a28a5b4867c2dd4a1ba513de278ccbe8bb1" - integrity sha512-/aPsuoj/1Dw/kzhkgz+ES6TxG0zfTMGLwuK2ZG00k/iJzYHTLCE8mVU8EPqEOp/lmxPoq1C1C9RYToRKb2KEfg== - "@sindresorhus/is@^3.1.1": version "3.1.2" resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-3.1.2.tgz#548650de521b344e3781fbdb0ece4aa6f729afb8" @@ -5110,7 +5105,7 @@ dependencies: defer-to-connect "^1.0.1" -"@szmarczak/http-timer@^4.0.0", "@szmarczak/http-timer@^4.0.5": +"@szmarczak/http-timer@^4.0.5": version "4.0.5" resolved "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz#bfbd50211e9dfa51ba07da58a14cdfd333205152" integrity sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ== @@ -5822,7 +5817,7 @@ resolved "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.2.tgz#513abfd256d7ad0bf1ee1873606317b33b1b2a72" integrity sha512-GJhpTepz2udxGexqos8wgaBx4I/zWIDPh/KOGEwAqtuGDkOUJu5eFvwmdBX4AmB8Odsr+9pHCQqiAqDL/yKMKw== -"@types/keyv@*", "@types/keyv@^3.1.1": +"@types/keyv@*": version "3.1.1" resolved "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.1.tgz#e45a45324fca9dab716ab1230ee249c9fb52cfa7" integrity sha512-MPtoySlAZQ37VoLaPcTHCu1RWJ4llDkULYZIzOYxlhxBqYPB0RsRlmMU0R6tahtFe27mIdkHV+551ZWV4PLmVw== @@ -8278,14 +8273,6 @@ bindings@^1.5.0: dependencies: file-uri-to-path "1.0.0" -bl@^1.0.0: - version "1.2.2" - resolved "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz#a160911717103c07410cef63ef51b397c025af9c" - integrity sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA== - dependencies: - readable-stream "^2.3.5" - safe-buffer "^5.1.1" - bl@^4.0.3: version "4.0.3" resolved "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz#12d6287adc29080e22a705e5764b2a9522cdc489" @@ -8544,19 +8531,6 @@ btoa@^1.2.1: resolved "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz#01a9909f8b2c93f6bf680ba26131eb30f7fa3d73" integrity sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g== -buffer-alloc-unsafe@^1.1.0: - version "1.1.0" - resolved "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz#bd7dc26ae2972d0eda253be061dba992349c19f0" - integrity sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg== - -buffer-alloc@^1.2.0: - version "1.2.0" - resolved "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz#890dd90d923a873e08e10e5fd51a57e5b7cce0ec" - integrity sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow== - dependencies: - buffer-alloc-unsafe "^1.1.0" - buffer-fill "^1.0.0" - buffer-crc32@^0.2.1, buffer-crc32@^0.2.13, buffer-crc32@~0.2.3: version "0.2.13" resolved "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" @@ -8567,11 +8541,6 @@ buffer-equal-constant-time@1.0.1: resolved "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" integrity sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk= -buffer-fill@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c" - integrity sha1-+PeLdniYiO858gXNY39o5wISKyw= - buffer-from@1.x, buffer-from@^1.0.0: version "1.1.1" resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" @@ -8740,14 +8709,6 @@ cache-base@^1.0.1: union-value "^1.0.0" unset-value "^1.0.0" -cacheable-lookup@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-2.0.1.tgz#87be64a18b925234875e10a9bb1ebca4adce6b38" - integrity sha512-EMMbsiOTcdngM/K6gV/OxF2x0t07+vMOWxZNSCRQMjO2MY2nhZQ6OYhOOpyQrbhqsgtvKGI7hcq6xjnA92USjg== - dependencies: - "@types/keyv" "^3.1.1" - keyv "^4.0.0" - cacheable-lookup@^5.0.3: version "5.0.3" resolved "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.3.tgz#049fdc59dffdd4fc285e8f4f82936591bd59fec3" @@ -9055,7 +9016,7 @@ chokidar@^3.2.2, chokidar@^3.3.0, chokidar@^3.3.1, chokidar@^3.4.1, chokidar@^3. optionalDependencies: fsevents "~2.1.2" -chownr@^1.0.1, chownr@^1.1.1, chownr@^1.1.2: +chownr@^1.1.1, chownr@^1.1.2: version "1.1.4" resolved "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== @@ -10719,13 +10680,6 @@ decompress-response@^4.2.0: dependencies: mimic-response "^2.0.0" -decompress-response@^5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/decompress-response/-/decompress-response-5.0.0.tgz#7849396e80e3d1eba8cb2f75ef4930f76461cb0f" - integrity sha512-TLZWWybuxWgoW7Lykv+gq9xvzOsUjQ9tF09Tj6NSTYGMTCHNXzrPnD6Hi+TgZq19PyTAGH4Ll/NIM/eTGglnMw== - dependencies: - mimic-response "^2.0.0" - decompress-response@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" @@ -12874,7 +12828,7 @@ fs-extra@^0.30.0: path-is-absolute "^1.0.0" rimraf "^2.2.8" -fs-extra@^7.0.0, fs-extra@^7.0.1: +fs-extra@^7.0.1: version "7.0.1" resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9" integrity sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw== @@ -13446,27 +13400,6 @@ google-p12-pem@^3.0.3: dependencies: node-forge "^0.10.0" -got@^10.7.0: - version "10.7.0" - resolved "https://registry.npmjs.org/got/-/got-10.7.0.tgz#62889dbcd6cca32cd6a154cc2d0c6895121d091f" - integrity sha512-aWTDeNw9g+XqEZNcTjMMZSy7B7yE9toWOFYip7ofFTLleJhvZwUxxTxkTpKvF+p1SAA4VHmuEy7PiHTHyq8tJg== - dependencies: - "@sindresorhus/is" "^2.0.0" - "@szmarczak/http-timer" "^4.0.0" - "@types/cacheable-request" "^6.0.1" - cacheable-lookup "^2.0.0" - cacheable-request "^7.0.1" - decompress-response "^5.0.0" - duplexer3 "^0.1.4" - get-stream "^5.0.0" - lowercase-keys "^2.0.0" - mimic-response "^2.1.0" - p-cancelable "^2.0.0" - p-event "^4.0.0" - responselike "^2.0.0" - to-readable-stream "^2.0.0" - type-fest "^0.10.0" - got@^11.5.2: version "11.6.0" resolved "https://registry.npmjs.org/got/-/got-11.6.0.tgz#4978c78f3cbc3a45ee95381f8bb6efd1db1f4638" @@ -15969,7 +15902,7 @@ json3@^3.3.2: resolved "https://registry.npmjs.org/json3/-/json3-3.3.3.tgz#7fc10e375fc5ae42c4705a5cc0aa6f62be305b81" integrity sha512-c7/8mbUsKigAbLkD5B010BK4D9LZm7A1pNItkEwiUZRpIN66exu/e7YQWysGun+TRKaJp8MhemM+VkfWv42aCA== -json5@2.x, json5@^2.1.0, json5@^2.1.1, json5@^2.1.2: +json5@2.x, json5@^2.1.1, json5@^2.1.2: version "2.1.3" resolved "https://registry.npmjs.org/json5/-/json5-2.1.3.tgz#c9b0f7fa9233bfe5807fe66fcf3a5617ed597d43" integrity sha512-KXPvOm8K9IJKFM0bmdn8QXh7udDh1g/giieX0NLCaMnb4hEiVFqnop2ImTXCc5e0/oHz3LTqmHGtExn5hfMkOA== @@ -17481,7 +17414,7 @@ mimic-response@^1.0.0, mimic-response@^1.0.1: resolved "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== -mimic-response@^2.0.0, mimic-response@^2.1.0: +mimic-response@^2.0.0: version "2.1.0" resolved "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43" integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA== @@ -18001,23 +17934,6 @@ node-gyp@3.x: tar "^2.0.0" which "1" -node-gyp@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/node-gyp/-/node-gyp-4.0.0.tgz#972654af4e5dd0cd2a19081b4b46fe0442ba6f45" - integrity sha512-2XiryJ8sICNo6ej8d0idXDEMKfVfFK7kekGCtJAuelGsYHQxhj13KTf95swTCN2dZ/4lTfZ84Fu31jqJEEgjWA== - dependencies: - glob "^7.0.3" - graceful-fs "^4.1.2" - mkdirp "^0.5.0" - nopt "2 || 3" - npmlog "0 || 1 || 2 || 3 || 4" - osenv "0" - request "^2.87.0" - rimraf "2" - semver "~5.3.0" - tar "^4.4.8" - which "1" - node-gyp@^5.0.2: version "5.1.0" resolved "https://registry.npmjs.org/node-gyp/-/node-gyp-5.1.0.tgz#8e31260a7af4a2e2f994b0673d4e0b3866156332" @@ -18107,22 +18023,6 @@ node-pre-gyp@^0.11.0: semver "^5.3.0" tar "^4" -node-pre-gyp@^0.13.0: - version "0.13.0" - resolved "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.13.0.tgz#df9ab7b68dd6498137717838e4f92a33fc9daa42" - integrity sha512-Md1D3xnEne8b/HGVQkZZwV27WUi1ZRuZBij24TNaZwUPU3ZAFtvT6xxJGaUVillfmMKnn5oD1HoGsp2Ftik7SQ== - dependencies: - detect-libc "^1.0.2" - mkdirp "^0.5.1" - needle "^2.2.1" - nopt "^4.0.1" - npm-packlist "^1.1.6" - npmlog "^4.0.2" - rc "^1.2.7" - rimraf "^2.6.1" - semver "^5.3.0" - tar "^4" - node-releases@^1.1.52, node-releases@^1.1.58: version "1.1.60" resolved "https://registry.npmjs.org/node-releases/-/node-releases-1.1.60.tgz#6948bdfce8286f0b5d0e5a88e8384e954dfe7084" @@ -18151,21 +18051,6 @@ node-request-interceptor@^0.5.1: debug "^4.1.1" headers-utils "^1.2.0" -nodegit@^0.27.0: - version "0.27.0" - resolved "https://registry.npmjs.org/nodegit/-/nodegit-0.27.0.tgz#4e8cc236f60e1c97324a5acff99056fe116a6ebe" - integrity sha512-E9K4gPjWiA0b3Tx5lfWCzG7Cvodi2idl3V5UD2fZrOrHikIfrN7Fc2kWLtMUqqomyoToYJLeIC8IV7xb1CYRLA== - dependencies: - fs-extra "^7.0.0" - got "^10.7.0" - json5 "^2.1.0" - lodash "^4.17.14" - nan "^2.14.0" - node-gyp "^4.0.0" - node-pre-gyp "^0.13.0" - ramda "^0.25.0" - tar-fs "^1.16.3" - nodemon@^2.0.2: version "2.0.6" resolved "https://registry.npmjs.org/nodemon/-/nodemon-2.0.6.tgz#1abe1937b463aaf62f0d52e2b7eaadf28cc2240d" @@ -18750,7 +18635,7 @@ p-each-series@^2.1.0: resolved "https://registry.npmjs.org/p-each-series/-/p-each-series-2.1.0.tgz#961c8dd3f195ea96c747e636b262b800a6b1af48" integrity sha512-ZuRs1miPT4HrjFa+9fRfOFXxGJfORgelKV9f9nNOWw2gl6gVsRaVDOQP0+MI0G0wGKns1Yacsu0GjOFbTK0JFQ== -p-event@^4.0.0, p-event@^4.1.0: +p-event@^4.1.0: version "4.2.0" resolved "https://registry.npmjs.org/p-event/-/p-event-4.2.0.tgz#af4b049c8acd91ae81083ebd1e6f5cae2044c1b5" integrity sha512-KXatOjCRXXkSePPb1Nbi0p0m+gQAwdlbhi4wQKJPI1HsMQS9g+Sqp2o+QHziPr7eYJyOZet836KoHEVM1mwOrQ== @@ -20285,14 +20170,6 @@ public-encrypt@^4.0.0: randombytes "^2.0.1" safe-buffer "^5.1.2" -pump@^1.0.0: - version "1.0.3" - resolved "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz#5dfe8311c33bbf6fc18261f9f34702c47c08a954" - integrity sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw== - dependencies: - end-of-stream "^1.1.0" - once "^1.3.1" - pump@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz#12399add6e4cf7526d973cbc8b5ce2e2908b3909" @@ -20438,11 +20315,6 @@ ramda@^0.21.0: resolved "https://registry.npmjs.org/ramda/-/ramda-0.21.0.tgz#a001abedb3ff61077d4ff1d577d44de77e8d0a35" integrity sha1-oAGr7bP/YQd9T/HVd9RN536NCjU= -ramda@^0.25.0: - version "0.25.0" - resolved "https://registry.npmjs.org/ramda/-/ramda-0.25.0.tgz#8fdf68231cffa90bc2f9460390a0cb74a29b29a9" - integrity sha512-GXpfrYVPwx3K7RQ6aYT8KPS8XViSXUVJT1ONhoKPE9VAleW42YE+U+8VEyGWt41EnEQW7gwecYJriTI0pKoecQ== - ramda@^0.26, ramda@~0.26.1: version "0.26.1" resolved "https://registry.npmjs.org/ramda/-/ramda-0.26.1.tgz#8d41351eb8111c55353617fc3bbffad8e4d35d06" @@ -21201,7 +21073,7 @@ read@1, read@~1.0.1: dependencies: mute-stream "~0.0.4" -"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.3, readable-stream@^2.3.5, readable-stream@^2.3.6, readable-stream@~2.3.6: +"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.3, readable-stream@^2.3.5, readable-stream@^2.3.6, readable-stream@~2.3.6: version "2.3.7" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -23495,16 +23367,6 @@ tapable@^1.0.0, tapable@^1.1.3: resolved "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== -tar-fs@^1.16.3: - version "1.16.3" - resolved "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz#966a628841da2c4010406a82167cbd5e0c72d509" - integrity sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw== - dependencies: - chownr "^1.0.1" - mkdirp "^0.5.1" - pump "^1.0.0" - tar-stream "^1.1.2" - tar-fs@~2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz#e44086c1c60d31a4f0cf893b1c4e155dabfae9e2" @@ -23515,19 +23377,6 @@ tar-fs@~2.0.1: pump "^3.0.0" tar-stream "^2.0.0" -tar-stream@^1.1.2: - version "1.6.2" - resolved "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz#8ea55dab37972253d9a9af90fdcd559ae435c555" - integrity sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A== - dependencies: - bl "^1.0.0" - buffer-alloc "^1.2.0" - end-of-stream "^1.0.0" - fs-constants "^1.0.0" - readable-stream "^2.3.0" - to-buffer "^1.1.1" - xtend "^4.0.0" - tar-stream@^2.0.0, tar-stream@^2.1.4: version "2.1.4" resolved "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz#c4fb1a11eb0da29b893a5b25476397ba2d053bfa" @@ -23867,11 +23716,6 @@ to-arraybuffer@^1.0.0: resolved "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz#7d229b1fcc637e466ca081180836a7aabff83f43" integrity sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M= -to-buffer@^1.1.1: - version "1.1.1" - resolved "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz#493bd48f62d7c43fcded313a03dcadb2e1213a80" - integrity sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg== - to-fast-properties@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" @@ -23889,11 +23733,6 @@ to-readable-stream@^1.0.0: resolved "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz#ce0aa0c2f3df6adf852efb404a783e77c0475771" integrity sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q== -to-readable-stream@^2.0.0: - version "2.1.0" - resolved "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-2.1.0.tgz#82880316121bea662cdc226adb30addb50cb06e8" - integrity sha512-o3Qa6DGg1CEXshSdvWNX2sN4QHqg03SPq7U6jPXRahlQdl5dK8oXjkU/2/sGrnOZKeGV1zLSO8qPwyKklPPE7w== - to-regex-range@^2.1.0: version "2.1.1" resolved "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz#7c80c17b9dfebe599e27367e0d4dd5590141db38" @@ -24194,11 +24033,6 @@ type-detect@4.0.8: resolved "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g== -type-fest@^0.10.0: - version "0.10.0" - resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.10.0.tgz#7f06b2b9fbfc581068d1341ffabd0349ceafc642" - integrity sha512-EUV9jo4sffrwlg8s0zDhP0T2WD3pru5Xi0+HTE3zTUmBaZNhfkite9PdSJwdXLwPVW0jnAHT56pZHIOYckPEiw== - type-fest@^0.11.0: version "0.11.0" resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.11.0.tgz#97abf0872310fed88a5c466b25681576145e33f1" From cb28e1a6a0ceede54d6fa9a2397ef3a87cdc327b Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 23 Dec 2020 00:34:12 +0100 Subject: [PATCH 16/33] chore(git): Need author and commiter for doing git merge --- packages/backend-common/src/scm/git.ts | 13 ++++++++++++- packages/techdocs-common/src/helpers.ts | 5 +++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index b744040cb2..871a623b19 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -148,16 +148,27 @@ class SCM { dir, headBranch, baseBranch, + author, + committer, }: { dir: string; headBranch: string; baseBranch?: string; + author: { name: string; email: string }; + committer: { name: string; email: string }; }) { this.config.logger?.info( `Merging branch '${headBranch}' into '${baseBranch}' for repository {dir=${dir}}`, ); // If baseBranch is undefined, current branch is used. - return git.merge({ fs, dir, ours: baseBranch, theirs: headBranch }); + return git.merge({ + fs, + dir, + ours: baseBranch, + theirs: headBranch, + author, + committer, + }); } async push({ dir, remoteName }: { dir: string; remoteName: string }) { diff --git a/packages/techdocs-common/src/helpers.ts b/packages/techdocs-common/src/helpers.ts index 749c91adc0..9685218bfa 100644 --- a/packages/techdocs-common/src/helpers.ts +++ b/packages/techdocs-common/src/helpers.ts @@ -168,6 +168,11 @@ export const checkoutGitRepository = async ( dir: repositoryTmpPath, headBranch: `origin/${currentBranchName}`, baseBranch: currentBranchName || undefined, + author: { name: 'Backstage TechDocs', email: 'techdocs@backstage.io' }, + committer: { + name: 'Backstage TechDocs', + email: 'techdocs@backstage.io', + }, }); return repositoryTmpPath; } catch (e) { From 5118cb472fa37d12583d9f3c840452ace7d31977 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 28 Dec 2020 16:16:18 +0100 Subject: [PATCH 17/33] test(backend-common/git): Adding tests for all the git helpers in backend common --- packages/backend-common/src/scm/git.test.ts | 321 ++++++++++++++++++++ packages/backend-common/src/scm/git.ts | 50 ++- 2 files changed, 340 insertions(+), 31 deletions(-) create mode 100644 packages/backend-common/src/scm/git.test.ts diff --git a/packages/backend-common/src/scm/git.test.ts b/packages/backend-common/src/scm/git.test.ts new file mode 100644 index 0000000000..7c38eafa0c --- /dev/null +++ b/packages/backend-common/src/scm/git.test.ts @@ -0,0 +1,321 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('isomorphic-git'); +jest.mock('isomorphic-git/http/node'); +jest.mock('fs-extra'); + +import * as isomorphic from 'isomorphic-git'; +import * as Git from './git'; +import http from 'isomorphic-git/http/node'; +import fs from 'fs-extra'; + +describe('Git', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + describe('add', () => { + it('should call isomorphic-git add with the correct arguments', async () => { + const git = Git.fromAuth({}); + const dir = 'mockdirectory'; + const filepath = 'mockfile/path'; + + await git.add({ dir, filepath }); + + expect(isomorphic.add).toHaveBeenCalledWith({ + fs, + dir, + filepath, + }); + }); + }); + + describe('addRemote', () => { + it('should call isomorphic-git with the correct arguments', async () => { + const git = Git.fromAuth({}); + const dir = 'mockdirectory'; + const remoteName = 'origin'; + const url = 'git@github.com/something/sads'; + + await git.addRemote({ dir, remoteName, url }); + + expect(isomorphic.addRemote).toHaveBeenCalledWith({ + fs, + dir, + remote: remoteName, + url, + }); + }); + }); + + describe('commit', () => { + it('should call isomorphic-git with the correct arguments', async () => { + const git = Git.fromAuth({}); + const dir = 'mockdirectory'; + const message = 'Inital Commit'; + const author = { + name: 'author', + email: 'test@backstage.io', + }; + const committer = { + name: 'comitter', + email: 'test@backstage.io', + }; + + await git.commit({ dir, message, author, committer }); + + expect(isomorphic.commit).toHaveBeenCalledWith({ + fs, + dir, + message, + author, + committer, + }); + }); + }); + + describe('clone', () => { + it('should call isomorphic-git with the correct arguments', async () => { + const url = 'http://github.com/some/repo'; + const dir = '/some/mock/dir'; + const auth = { + username: 'blob', + password: 'hunter2', + }; + const git = Git.fromAuth(auth); + + await git.clone({ url, dir }); + + expect(isomorphic.clone).toHaveBeenCalledWith({ + fs, + http, + url, + dir, + singleBranch: true, + depth: 1, + onProgress: expect.any(Function), + headers: { + 'user-agent': 'git/@isomorphic-git', + }, + onAuth: expect.any(Function), + }); + }); + it('should pass a function that returns the authorization as the onAuth handler', async () => { + const url = 'http://github.com/some/repo'; + const dir = '/some/mock/dir'; + const auth = { + username: 'blob', + password: 'hunter2', + }; + const git = Git.fromAuth(auth); + + await git.clone({ url, dir }); + + const { onAuth } = ((isomorphic.clone as unknown) as jest.Mock< + typeof isomorphic['clone'] + >).mock.calls[0][0]!; + + expect(onAuth()).toEqual(auth); + }); + }); + + describe('currentBranch', () => { + it('should call isomorphic-git with the correct arguments', async () => { + const dir = '/some/mock/dir'; + const fullName = true; + const git = Git.fromAuth({}); + + await git.currentBranch({ dir, fullName }); + + expect(isomorphic.currentBranch).toHaveBeenCalledWith({ + fs, + dir, + fullname: true, + }); + + await git.currentBranch({ dir }); + + expect(isomorphic.currentBranch).toHaveBeenCalledWith({ + fs, + dir, + fullname: false, + }); + }); + }); + + describe('fetch', () => { + it('should call isomorphic-git with the correct arguments', async () => { + const remote = 'http://github.com/some/repo'; + const dir = '/some/mock/dir'; + const auth = { + username: 'blob', + password: 'hunter2', + }; + const git = Git.fromAuth(auth); + + await git.fetch({ remote, dir }); + + expect(isomorphic.fetch).toHaveBeenCalledWith({ + fs, + http, + remote, + dir, + onProgress: expect.any(Function), + headers: { + 'user-agent': 'git/@isomorphic-git', + }, + onAuth: expect.any(Function), + }); + }); + it('should pass a function that returns the authorization as the onAuth handler', async () => { + const remote = 'http://github.com/some/repo'; + const dir = '/some/mock/dir'; + const auth = { + username: 'blob', + password: 'hunter2', + }; + const git = Git.fromAuth(auth); + + await git.fetch({ remote, dir }); + + const { onAuth } = ((isomorphic.fetch as unknown) as jest.Mock< + typeof isomorphic['fetch'] + >).mock.calls[0][0]!; + + expect(onAuth()).toEqual(auth); + }); + }); + + describe('init', () => { + it('should call isomorphic-git with the correct arguments', async () => { + const dir = '/some/mock/dir'; + + const git = Git.fromAuth({}); + + await git.init({ dir }); + + expect(isomorphic.init).toHaveBeenCalledWith({ + fs, + dir, + }); + }); + }); + + describe('merge', () => { + it('should call isomorphic-git with the correct arguments', async () => { + const dir = '/some/mock/dir'; + const author = { + name: 'author', + email: 'test@backstage.io', + }; + const committer = { + name: 'comitter', + email: 'test@backstage.io', + }; + const headBranch = 'master'; + const baseBranch = 'production'; + + const git = Git.fromAuth({}); + + await git.merge({ dir, headBranch, baseBranch, author, committer }); + + expect(isomorphic.merge).toHaveBeenCalledWith({ + fs, + dir, + ours: baseBranch, + theirs: headBranch, + author, + committer, + }); + }); + }); + + describe('push', () => { + it('should call isomorphic-git with the correct arguments', async () => { + const remoteName = 'origin'; + const dir = '/some/mock/dir'; + const auth = { + username: 'blob', + password: 'hunter2', + }; + const git = Git.fromAuth(auth); + + await git.push({ dir, remoteName }); + + expect(isomorphic.push).toHaveBeenCalledWith({ + fs, + http, + remote: remoteName, + dir, + onProgress: expect.any(Function), + headers: { + 'user-agent': 'git/@isomorphic-git', + }, + onAuth: expect.any(Function), + }); + }); + it('should pass a function that returns the authorization as the onAuth handler', async () => { + const remoteName = 'origin'; + const dir = '/some/mock/dir'; + const auth = { + username: 'blob', + password: 'hunter2', + }; + const git = Git.fromAuth(auth); + + await git.push({ remoteName, dir }); + + const { onAuth } = ((isomorphic.push as unknown) as jest.Mock< + typeof isomorphic['push'] + >).mock.calls[0][0]!; + + expect(onAuth()).toEqual(auth); + }); + }); + + describe('readCommit', () => { + it('should call isomorphic-git with the correct arguments', async () => { + const dir = '/some/mock/dir'; + const sha = 'as43bd7'; + + const git = Git.fromAuth({}); + + await git.readCommit({ dir, sha }); + + expect(isomorphic.readCommit).toHaveBeenCalledWith({ + fs, + dir, + oid: sha, + }); + }); + }); + + describe('resolveRef', () => { + it('should call isomorphic-git with the correct arguments', async () => { + const dir = '/some/mock/dir'; + const ref = 'as43bd7'; + + const git = Git.fromAuth({}); + + await git.resolveRef({ dir, ref }); + + expect(isomorphic.resolveRef).toHaveBeenCalledWith({ + fs, + dir, + ref, + }); + }); + }); +}); diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index 871a623b19..5d3f95d42c 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import git from 'isomorphic-git'; +import git, { ProgressCallback } from 'isomorphic-git'; import http from 'isomorphic-git/http/node'; import fs from 'fs-extra'; import { Logger } from 'winston'; @@ -85,19 +85,11 @@ class SCM { dir, singleBranch: true, depth: 1, - onProgress: event => { - const total = event.total - ? `${Math.round((event.loaded / event.total) * 100)}%` - : event.loaded; - this.config.logger?.info(`status={${event.phase},total={${total}}}`); - }, + onProgress: this.onProgressHandler, headers: { 'user-agent': 'git/@isomorphic-git', }, - onAuth: () => ({ - username: this.config.username, - password: this.config.password, - }), + onAuth: this.onAuth, }); } @@ -118,19 +110,11 @@ class SCM { http, dir, remote: remoteValue, - onProgress: event => { - const total = event.total - ? `${Math.round((event.loaded / event.total) * 100)}%` - : event.loaded; - this.config.logger?.info(`status={${event.phase},total={${total}}}`); - }, + onProgress: this.onProgressHandler, headers: { 'user-agent': 'git/@isomorphic-git', }, - onAuth: () => ({ - username: this.config.username, - password: this.config.password, - }), + onAuth: this.onAuth, }); } @@ -179,20 +163,12 @@ class SCM { fs, dir, http, - onProgress: event => { - const total = event.total - ? `${Math.round((event.loaded / event.total) * 100)}%` - : event.loaded; - this.config.logger?.info(`status={${event.phase},total={${total}}}`); - }, + onProgress: this.onProgressHandler, headers: { 'user-agent': 'git/@isomorphic-git', }, remote: remoteName, - onAuth: () => ({ - username: this.config.username, - password: this.config.password, - }), + onAuth: this.onAuth, }); } @@ -205,6 +181,18 @@ class SCM { async resolveRef({ dir, ref }: { dir: string; ref: string }) { return git.resolveRef({ fs, dir, ref }); } + + private onAuth = () => ({ + username: this.config.username, + password: this.config.password, + }); + + private onProgressHandler: ProgressCallback = event => { + const total = event.total + ? `${Math.round((event.loaded / event.total) * 100)}%` + : event.loaded; + this.config.logger?.info(`status={${event.phase},total={${total}}}`); + }; } // TODO(blam): This could potentially become something like for URL From 8022655e0bf4cedfd088aae49fd9dee5bc8a4520 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 28 Dec 2020 16:42:11 +0100 Subject: [PATCH 18/33] chore: removing the dependency on the azure preparer for nodegit --- .../scaffolder/stages/prepare/azure.test.ts | 71 +++++++++---------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts index 4d19069b5e..39efd1c62a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts @@ -14,11 +14,6 @@ * limitations under the License. */ -const mocks = { - Clone: { clone: jest.fn() }, - CheckoutOptions: jest.fn(() => {}), -}; -jest.doMock('nodegit', () => mocks); jest.doMock('fs-extra', () => ({ promises: { mkdtemp: jest.fn(dir => `${dir}-static`), @@ -30,10 +25,16 @@ import { TemplateEntityV1alpha1, LOCATION_ANNOTATION, } from '@backstage/catalog-model'; -import { getVoidLogger } from '@backstage/backend-common'; +import { getVoidLogger, Git } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; describe('AzurePreparer', () => { + const mockGitClient = { + clone: jest.fn(), + }; + + jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any); + let mockEntity: TemplateEntityV1alpha1; beforeEach(() => { jest.clearAllMocks(); @@ -77,18 +78,7 @@ describe('AzurePreparer', () => { }; }); - it('calls the clone command with the correct arguments for a repository', async () => { - const preparer = new AzurePreparer(new ConfigReader({})); - await preparer.prepare(mockEntity, { logger: getVoidLogger() }); - expect(mocks.Clone.clone).toHaveBeenNthCalledWith( - 1, - 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', - expect.any(String), - {}, - ); - }); - - it('calls the clone command with the correct arguments if an access token is provided for a repository', async () => { + it('initializes git client with the correct arguments if an access token is provided for a repository', async () => { const preparer = new AzurePreparer( new ConfigReader({ scaffolder: { @@ -100,36 +90,44 @@ describe('AzurePreparer', () => { }, }), ); + const logger = getVoidLogger(); + await preparer.prepare(mockEntity, { logger }); + + expect(Git.fromAuth).toHaveBeenCalledWith({ + username: 'notempty', + password: 'fake-token', + logger, + }); + }); + it('calls the clone command with the correct arguments for a repository', async () => { + const preparer = new AzurePreparer(new ConfigReader({})); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); - expect(mocks.Clone.clone).toHaveBeenNthCalledWith( - 1, - 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', - expect.any(String), - { - fetchOpts: { - callbacks: { - credentials: expect.anything(), - }, - }, - }, - ); + + expect(mockGitClient.clone).toHaveBeenCalledWith({ + url: + 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', + dir: expect.any(String), + }); }); it('calls the clone command with the correct arguments for a repository when no path is provided', async () => { const preparer = new AzurePreparer(new ConfigReader({})); delete mockEntity.spec.path; + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); - expect(mocks.Clone.clone).toHaveBeenNthCalledWith( - 1, - 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', - expect.any(String), - {}, - ); + + expect(mockGitClient.clone).toHaveBeenCalledWith({ + url: + 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', + dir: expect.any(String), + }); }); it('return the temp directory with the path to the folder if it is specified', async () => { const preparer = new AzurePreparer(new ConfigReader({})); mockEntity.spec.path = './template/test/1/2/3'; + const response = await preparer.prepare(mockEntity, { logger: getVoidLogger(), }); @@ -142,6 +140,7 @@ describe('AzurePreparer', () => { it('return the working directory with the path to the folder if it is specified', async () => { const preparer = new AzurePreparer(new ConfigReader({})); mockEntity.spec.path = './template/test/1/2/3'; + const response = await preparer.prepare(mockEntity, { logger: getVoidLogger(), workingDirectory: '/workDir', From 9a9604408c1e65163691929a881145114d27c05e Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 28 Dec 2020 16:48:02 +0100 Subject: [PATCH 19/33] chore: removing dependency for githubn preparer --- .../scaffolder/stages/prepare/github.test.ts | 64 ++++++++----------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts index 86a870330e..aa040bfe58 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts @@ -14,11 +14,6 @@ * limitations under the License. */ -const mocks = { - Clone: { clone: jest.fn() }, - CheckoutOptions: jest.fn(() => {}), -}; -jest.doMock('nodegit', () => mocks); jest.doMock('fs-extra', () => ({ promises: { mkdtemp: jest.fn(dir => `${dir}-static`), @@ -30,10 +25,16 @@ import { TemplateEntityV1alpha1, LOCATION_ANNOTATION, } from '@backstage/catalog-model'; -import { getVoidLogger } from '@backstage/backend-common'; +import { getVoidLogger, Git } from '@backstage/backend-common'; describe('GitHubPreparer', () => { let mockEntity: TemplateEntityV1alpha1; + const mockGitClient = { + clone: jest.fn(), + }; + + jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any); + beforeEach(() => { jest.clearAllMocks(); mockEntity = { @@ -77,28 +78,24 @@ describe('GitHubPreparer', () => { }); it('calls the clone command with the correct arguments for a repository', async () => { const preparer = new GithubPreparer(); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); - expect(mocks.Clone.clone).toHaveBeenNthCalledWith( - 1, - 'https://github.com/benjdlambert/backstage-graphql-template', - expect.any(String), - { - checkoutBranch: 'master', - }, - ); + + expect(mockGitClient.clone).toHaveBeenCalledWith({ + url: 'https://github.com/benjdlambert/backstage-graphql-template', + dir: expect.any(String), + }); }); it('calls the clone command with the correct arguments for a repository when no path is provided', async () => { const preparer = new GithubPreparer(); delete mockEntity.spec.path; + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); - expect(mocks.Clone.clone).toHaveBeenNthCalledWith( - 1, - 'https://github.com/benjdlambert/backstage-graphql-template', - expect.any(String), - { - checkoutBranch: 'master', - }, - ); + + expect(mockGitClient.clone).toHaveBeenCalledWith({ + url: 'https://github.com/benjdlambert/backstage-graphql-template', + dir: expect.any(String), + }); }); it('return the temp directory with the path to the folder if it is specified', async () => { @@ -128,19 +125,14 @@ describe('GitHubPreparer', () => { it('calls the clone command with the token when provided', async () => { const preparer = new GithubPreparer({ token: 'abc' }); - await preparer.prepare(mockEntity, { logger: getVoidLogger() }); - expect(mocks.Clone.clone).toHaveBeenNthCalledWith( - 1, - 'https://github.com/benjdlambert/backstage-graphql-template', - expect.any(String), - { - checkoutBranch: 'master', - fetchOpts: { - callbacks: { - credentials: expect.any(Function), - }, - }, - }, - ); + const logger = getVoidLogger(); + + await preparer.prepare(mockEntity, { logger }); + + expect(Git.fromAuth).toHaveBeenCalledWith({ + logger, + username: 'abc', + password: 'x-oauth-basic', + }); }); }); From 84e937e32448951f5aa2bef884cc38def86360a4 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 28 Dec 2020 17:08:53 +0100 Subject: [PATCH 20/33] chore: fix gitlab preparer tests to remove the node-git dependency --- .../scaffolder/stages/prepare/gitlab.test.ts | 81 +++++++++---------- 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts index 8712507026..8dad349a13 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts @@ -13,11 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const mocks = { - Clone: { clone: jest.fn() }, - CheckoutOptions: jest.fn(() => {}), -}; -jest.doMock('nodegit', () => mocks); jest.doMock('fs-extra', () => ({ promises: { mkdtemp: jest.fn(dir => `${dir}-static`), @@ -30,7 +25,7 @@ import { LOCATION_ANNOTATION, } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; -import { getVoidLogger } from '@backstage/backend-common'; +import { getVoidLogger, Git } from '@backstage/backend-common'; const mockEntityWithProtocol = (protocol: string): TemplateEntityV1alpha1 => ({ apiVersion: 'backstage.io/v1alpha1', @@ -72,6 +67,12 @@ const mockEntityWithProtocol = (protocol: string): TemplateEntityV1alpha1 => ({ describe('GitLabPreparer', () => { let mockEntity: TemplateEntityV1alpha1; + const mockGitClient = { + clone: jest.fn(), + }; + + jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any); + beforeEach(() => { jest.clearAllMocks(); }); @@ -80,13 +81,13 @@ describe('GitLabPreparer', () => { it(`calls the clone command with the correct arguments for a repository using the ${protocol} protocol`, async () => { const preparer = new GitlabPreparer(new ConfigReader({})); mockEntity = mockEntityWithProtocol(protocol); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); - expect(mocks.Clone.clone).toHaveBeenNthCalledWith( - 1, - 'https://gitlab.com/benjdlambert/backstage-graphql-template', - expect.any(String), - {}, - ); + + expect(mockGitClient.clone).toHaveBeenCalledWith({ + url: 'https://gitlab.com/benjdlambert/backstage-graphql-template', + dir: expect.any(String), + }); }); it(`calls the clone command with the correct arguments if an access token is provided in integrations for a repository using the ${protocol} protocol`, async () => { @@ -103,19 +104,15 @@ describe('GitLabPreparer', () => { }), ); mockEntity = mockEntityWithProtocol(protocol); - await preparer.prepare(mockEntity, { logger: getVoidLogger() }); - expect(mocks.Clone.clone).toHaveBeenNthCalledWith( - 1, - 'https://gitlab.com/benjdlambert/backstage-graphql-template', - expect.any(String), - { - fetchOpts: { - callbacks: { - credentials: expect.anything(), - }, - }, - }, - ); + const logger = getVoidLogger(); + + await preparer.prepare(mockEntity, { logger }); + + expect(Git.fromAuth).toHaveBeenCalledWith({ + logger, + username: 'oauth2', + password: 'fake-token', + }); }); it(`calls the clone command with the correct arguments if an access token is provided in scaffolder for a repository using the ${protocol} protocol`, async () => { @@ -127,32 +124,28 @@ describe('GitLabPreparer', () => { }), ); mockEntity = mockEntityWithProtocol(protocol); - await preparer.prepare(mockEntity, { logger: getVoidLogger() }); - expect(mocks.Clone.clone).toHaveBeenNthCalledWith( - 1, - 'https://gitlab.com/benjdlambert/backstage-graphql-template', - expect.any(String), - { - fetchOpts: { - callbacks: { - credentials: expect.anything(), - }, - }, - }, - ); + const logger = getVoidLogger(); + + await preparer.prepare(mockEntity, { logger }); + + expect(Git.fromAuth).toHaveBeenCalledWith({ + logger, + username: 'oauth2', + password: 'fake-token', + }); }); it(`calls the clone command with the correct arguments for a repository when no path is provided using the ${protocol} protocol`, async () => { const preparer = new GitlabPreparer(new ConfigReader({})); mockEntity = mockEntityWithProtocol(protocol); delete mockEntity.spec.path; + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); - expect(mocks.Clone.clone).toHaveBeenNthCalledWith( - 1, - 'https://gitlab.com/benjdlambert/backstage-graphql-template', - expect.any(String), - {}, - ); + + expect(mockGitClient.clone).toHaveBeenCalledWith({ + url: 'https://gitlab.com/benjdlambert/backstage-graphql-template', + dir: expect.any(String), + }); }); it(`return the temp directory with the path to the folder if it is specified using the ${protocol} protocol`, async () => { From 438948f9bbc57d85b45383a57f6180bc31b90e2a Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 28 Dec 2020 17:23:39 +0100 Subject: [PATCH 21/33] chore: remove nodegit dependency on azure publisher --- .../scaffolder/stages/publish/azure.test.ts | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts index 9ea7d2de5f..f1e0cd6175 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts @@ -13,17 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -jest.mock('nodegit'); -jest.mock('azure-devops-node-api/GitApi'); -jest.mock('azure-devops-node-api/interfaces/GitInterfaces'); -jest.mock('./helpers', () => ({ - pushToRemoteUserPass: jest.fn(), -})); +jest.mock('./helpers'); import { AzurePublisher } from './azure'; import { GitApi } from 'azure-devops-node-api/GitApi'; -import { pushToRemoteUserPass } from './helpers'; +import * as helpers from './helpers'; +import { getVoidLogger } from '@backstage/backend-common'; const { mockGitApi } = require('azure-devops-node-api/GitApi') as { mockGitApi: { @@ -33,6 +28,7 @@ const { mockGitApi } = require('azure-devops-node-api/GitApi') as { describe('Azure Publisher', () => { const publisher = new AzurePublisher(new GitApi('', []), 'fake-token'); + const logger = getVoidLogger(); beforeEach(() => { jest.clearAllMocks(); @@ -50,6 +46,7 @@ describe('Azure Publisher', () => { owner: 'bob', }, directory: '/tmp/test', + logger, }); expect(result).toEqual({ @@ -63,12 +60,12 @@ describe('Azure Publisher', () => { }, 'project', ); - expect(pushToRemoteUserPass).toHaveBeenCalledWith( - '/tmp/test', - 'https://dev.azure.com/organization/project/_git/repo', - 'notempty', - 'fake-token', - ); + expect(helpers.initRepoAndPush).toHaveBeenCalledWith({ + dir: '/tmp/test', + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + auth: { username: 'notempty', password: 'fake-token' }, + logger, + }); }); }); }); From 783ac57ce7b344a0a04b0cccb03c7efed360a915 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 28 Dec 2020 17:33:04 +0100 Subject: [PATCH 22/33] chore: removing nodegit dependency on the github publisher --- .../scaffolder/stages/publish/github.test.ts | 75 ++++++++++--------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts index cda64faf25..22f1cd3172 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts @@ -15,10 +15,7 @@ */ jest.mock('@octokit/rest'); -jest.mock('nodegit'); -jest.mock('./helpers', () => ({ - pushToRemoteUserPass: jest.fn(), -})); +jest.mock('./helpers'); import { Octokit } from '@octokit/rest'; import { @@ -27,7 +24,8 @@ import { UsersGetByUsernameResponseData, } from '@octokit/types'; import { GithubPublisher } from './github'; -import { pushToRemoteUserPass } from './helpers'; +import { initRepoAndPush } from './helpers'; +import { getVoidLogger } from '@backstage/backend-common'; const { mockGithubClient } = require('@octokit/rest') as { mockGithubClient: { @@ -38,6 +36,7 @@ const { mockGithubClient } = require('@octokit/rest') as { }; describe('GitHub Publisher', () => { + const logger = getVoidLogger(); beforeEach(() => { jest.clearAllMocks(); }); @@ -69,6 +68,7 @@ describe('GitHub Publisher', () => { access: 'blam/team', }, directory: '/tmp/test', + logger, }); expect(result).toEqual({ @@ -91,12 +91,12 @@ describe('GitHub Publisher', () => { repo: 'test', permission: 'admin', }); - expect(pushToRemoteUserPass).toHaveBeenCalledWith( - '/tmp/test', - 'https://github.com/backstage/backstage.git', - 'abc', - 'x-oauth-basic', - ); + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: '/tmp/test', + remoteUrl: 'https://github.com/backstage/backstage.git', + auth: { username: 'abc', password: 'x-oauth-basic' }, + logger, + }); }); it('should use octokit to create a repo in the authed user if the organisation property is not set', async () => { @@ -118,6 +118,7 @@ describe('GitHub Publisher', () => { access: 'blam', }, directory: '/tmp/test', + logger, }); expect(result).toEqual({ @@ -132,12 +133,13 @@ describe('GitHub Publisher', () => { private: false, }); expect(mockGithubClient.repos.addCollaborator).not.toHaveBeenCalled(); - expect(pushToRemoteUserPass).toHaveBeenCalledWith( - '/tmp/test', - 'https://github.com/backstage/backstage.git', - 'abc', - 'x-oauth-basic', - ); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: '/tmp/test', + remoteUrl: 'https://github.com/backstage/backstage.git', + auth: { username: 'abc', password: 'x-oauth-basic' }, + logger, + }); }); }); @@ -161,6 +163,7 @@ describe('GitHub Publisher', () => { description: 'description', }, directory: '/tmp/test', + logger, }); expect(result).toEqual({ @@ -181,12 +184,12 @@ describe('GitHub Publisher', () => { username: 'bob', permission: 'admin', }); - expect(pushToRemoteUserPass).toHaveBeenCalledWith( - '/tmp/test', - 'https://github.com/backstage/backstage.git', - 'abc', - 'x-oauth-basic', - ); + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: '/tmp/test', + remoteUrl: 'https://github.com/backstage/backstage.git', + auth: { username: 'abc', password: 'x-oauth-basic' }, + logger, + }); }); }); @@ -216,6 +219,7 @@ describe('GitHub Publisher', () => { owner: 'bob', }, directory: '/tmp/test', + logger, }); expect(result).toEqual({ @@ -229,12 +233,12 @@ describe('GitHub Publisher', () => { private: true, visibility: 'internal', }); - expect(pushToRemoteUserPass).toHaveBeenCalledWith( - '/tmp/test', - 'https://github.com/backstage/backstage.git', - 'abc', - 'x-oauth-basic', - ); + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: '/tmp/test', + remoteUrl: 'https://github.com/backstage/backstage.git', + auth: { username: 'abc', password: 'x-oauth-basic' }, + logger, + }); }); }); @@ -263,6 +267,7 @@ describe('GitHub Publisher', () => { owner: 'bob', }, directory: '/tmp/test', + logger, }); expect(result).toEqual({ @@ -276,12 +281,12 @@ describe('GitHub Publisher', () => { name: 'test', private: true, }); - expect(pushToRemoteUserPass).toHaveBeenCalledWith( - '/tmp/test', - 'https://github.com/backstage/backstage.git', - 'abc', - 'x-oauth-basic', - ); + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: '/tmp/test', + remoteUrl: 'https://github.com/backstage/backstage.git', + auth: { username: 'abc', password: 'x-oauth-basic' }, + logger, + }); }); }); }); From 4311d59521f52e33691b113901991f8052448f48 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 28 Dec 2020 17:37:11 +0100 Subject: [PATCH 23/33] chore: remove nodegit dependency from gitlab publisher --- .../scaffolder/stages/publish/gitlab.test.ts | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts index a87fa6c6d9..61113003d4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts @@ -16,14 +16,13 @@ jest.mock('nodegit'); jest.mock('@gitbeaker/node'); -jest.mock('./helpers', () => ({ - pushToRemoteUserPass: jest.fn(), -})); +jest.mock('./helpers'); import { GitlabPublisher } from './gitlab'; import { Gitlab as GitlabAPI } from '@gitbeaker/core'; import { Gitlab } from '@gitbeaker/node'; -import { pushToRemoteUserPass } from './helpers'; +import { initRepoAndPush } from './helpers'; +import { getVoidLogger } from '@backstage/backend-common'; const { mockGitlabClient } = require('@gitbeaker/node') as { mockGitlabClient: { @@ -34,6 +33,7 @@ const { mockGitlabClient } = require('@gitbeaker/node') as { }; describe('GitLab Publisher', () => { + const logger = getVoidLogger(); const publisher = new GitlabPublisher(new Gitlab({}), 'fake-token'); beforeEach(() => { @@ -56,6 +56,7 @@ describe('GitLab Publisher', () => { owner: 'bob', }, directory: '/tmp/test', + logger, }); expect(result).toEqual({ remoteUrl: 'mockclone' }); @@ -63,12 +64,12 @@ describe('GitLab Publisher', () => { namespace_id: 42, name: 'test', }); - expect(pushToRemoteUserPass).toHaveBeenCalledWith( - '/tmp/test', - 'mockclone', - 'oauth2', - 'fake-token', - ); + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: '/tmp/test', + remoteUrl: 'mockclone', + auth: { username: 'oauth2', password: 'fake-token' }, + logger, + }); }); it('should use gitbeaker to create a repo in the authed user if the namespace property is not set', async () => { @@ -86,6 +87,7 @@ describe('GitLab Publisher', () => { owner: 'bob', }, directory: '/tmp/test', + logger, }); expect(result).toEqual({ remoteUrl: 'mockclone' }); @@ -94,12 +96,12 @@ describe('GitLab Publisher', () => { namespace_id: 21, name: 'test', }); - expect(pushToRemoteUserPass).toHaveBeenCalledWith( - '/tmp/test', - 'mockclone', - 'oauth2', - 'fake-token', - ); + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: '/tmp/test', + remoteUrl: 'mockclone', + auth: { username: 'oauth2', password: 'fake-token' }, + logger, + }); }); }); }); From 5c6da874de80d0366c531a36b39fe7bb4b51db3e Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 28 Dec 2020 17:38:52 +0100 Subject: [PATCH 24/33] chore: removing old helper tests --- .../scaffolder/stages/publish/helpers.test.ts | 112 ------------------ 1 file changed, 112 deletions(-) delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts deleted file mode 100644 index 3a317adacd..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -jest.mock('nodegit'); -import * as NodeGit from 'nodegit'; -import { pushToRemoteCred } from './helpers'; - -const { - Repository, - mockRepo, - mockIndex, - Signature, - Remote, - mockRemote, - Cred, -} = require('nodegit') as { - Repository: jest.Mocked<{ init: any }>; - Signature: jest.Mocked<{ now: any }>; - Cred: jest.Mocked<{ userpassPlaintextNew: any }>; - Remote: jest.Mocked<{ create: any }>; - - mockIndex: jest.Mocked; - mockRepo: jest.Mocked; - mockRemote: jest.Mocked; -}; - -describe('pushToRemoteCred', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - const directory = '/tmp/test/dir'; - const remote = 'mockclone'; - const credentialsProvider = () => - NodeGit.Cred.userpassPlaintextNew('username', 'password'); - - it('should call init on the repo with the directory', async () => { - await pushToRemoteCred(directory, remote, credentialsProvider); - - expect(Repository.init).toHaveBeenCalledWith(directory, 0); - }); - - it('should call refresh index on the index and write the new files', async () => { - await pushToRemoteCred(directory, remote, credentialsProvider); - - expect(mockRepo.refreshIndex).toHaveBeenCalled(); - }); - - it('should call add all files and write', async () => { - await pushToRemoteCred(directory, remote, credentialsProvider); - - expect(mockIndex.addAll).toHaveBeenCalled(); - expect(mockIndex.write).toHaveBeenCalled(); - expect(mockIndex.writeTree).toHaveBeenCalled(); - }); - - it('should create a commit with on head with the right name and commiter', async () => { - const mockSignature = { mockSignature: 'bloblly' }; - Signature.now.mockReturnValue(mockSignature); - - await pushToRemoteCred(directory, remote, credentialsProvider); - - expect(Signature.now).toHaveBeenCalledTimes(2); - expect(Signature.now).toHaveBeenCalledWith( - 'Scaffolder', - 'scaffolder@backstage.io', - ); - - expect(mockRepo.createCommit).toHaveBeenCalledWith( - 'HEAD', - mockSignature, - mockSignature, - 'initial commit', - 'mockoid', - [], - ); - }); - - it('creates a remote with the repo and remote', async () => { - await pushToRemoteCred(directory, remote, credentialsProvider); - - expect(Remote.create).toHaveBeenCalledWith(mockRepo, 'origin', 'mockclone'); - }); - - it('shoud push to the remote repo', async () => { - await pushToRemoteCred(directory, remote, credentialsProvider); - - const [remotes, { callbacks }] = mockRemote.push.mock - .calls[0] as NodeGit.PushOptions[]; - - expect(remotes).toEqual(['refs/heads/master:refs/heads/master']); - - callbacks?.credentials?.(); - - expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith( - 'username', - 'password', - ); - }); -}); From e203b56d13123037f71b4ed9e6b49070cb2a943b Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 28 Dec 2020 17:41:25 +0100 Subject: [PATCH 25/33] chore: fix typescript issues with scaffolder-backend now the publishers require a logger :) --- plugins/scaffolder-backend/src/service/router.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index ac3b1c80ec..ffd4d6487c 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -159,6 +159,7 @@ export async function createRouter( const result = await publisher.publish({ values: ctx.values, directory: ctx.resultDir, + logger: ctx.logger, }); return result; }, From 7839e0fb42bd3d7d04cc542e534123b05d65f47c Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 29 Dec 2020 10:05:27 +0100 Subject: [PATCH 26/33] chore: removing last occurrence of nodegit --- plugins/scaffolder-backend/package.json | 1 - .../src/scaffolder/stages/publish/gitlab.test.ts | 1 - yarn.lock | 11 ++--------- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index d53e04fe3e..1a443ceb7c 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -62,7 +62,6 @@ "@octokit/types": "^5.4.1", "@types/fs-extra": "^9.0.1", "@types/git-url-parse": "^9.0.0", - "@types/nodegit": "^0.26.12", "@types/supertest": "^2.0.8", "supertest": "^4.0.2", "yaml": "^1.10.0" diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts index 61113003d4..2f33358829 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -jest.mock('nodegit'); jest.mock('@gitbeaker/node'); jest.mock('./helpers'); diff --git a/yarn.lock b/yarn.lock index 528c653b38..7a49095f70 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1666,10 +1666,10 @@ yup "^0.29.3" "@backstage/core@^0.3.0": - version "0.4.1" + version "0.4.2" dependencies: "@backstage/config" "^0.1.2" - "@backstage/core-api" "^0.2.6" + "@backstage/core-api" "^0.2.7" "@backstage/theme" "^0.2.2" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" @@ -5972,13 +5972,6 @@ resolved "https://registry.npmjs.org/@types/node/-/node-13.13.15.tgz#fe1cc3aa465a3ea6858b793fd380b66c39919766" integrity sha512-kwbcs0jySLxzLsa2nWUAGOd/s21WU1jebrEdtzhsj1D4Yps1EOuyI1Qcu+FD56dL7NRNIJtDDjcqIG22NwkgLw== -"@types/nodegit@^0.26.12": - version "0.26.12" - resolved "https://registry.npmjs.org/@types/nodegit/-/nodegit-0.26.12.tgz#93afb4cb85d3a48d392c3232699c9c07d8251a36" - integrity sha512-4YpeTImFZNJ1cve4lEueHFVS8rAs8XpZqlmx+Bm9bMc+XMiCrcwaUf6peN7pod7Rl3esVlGP1zdBB7Z12eMVAA== - dependencies: - "@types/node" "*" - "@types/normalize-package-data@^2.4.0": version "2.4.0" resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" From ba6f2b8a40ad9dce9f606e64378990527e4f55ca Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 29 Dec 2020 11:46:36 +0100 Subject: [PATCH 27/33] chore: reworking the API exports and fixing the logging to be a little less verbose --- packages/backend-common/src/scm/git.test.ts | 2 +- packages/backend-common/src/scm/git.ts | 61 +++++++++++---------- packages/backend-common/src/scm/index.ts | 3 +- 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/packages/backend-common/src/scm/git.test.ts b/packages/backend-common/src/scm/git.test.ts index 7c38eafa0c..96c0f919a7 100644 --- a/packages/backend-common/src/scm/git.test.ts +++ b/packages/backend-common/src/scm/git.test.ts @@ -18,7 +18,7 @@ jest.mock('isomorphic-git/http/node'); jest.mock('fs-extra'); import * as isomorphic from 'isomorphic-git'; -import * as Git from './git'; +import { Git } from './git'; import http from 'isomorphic-git/http/node'; import fs from 'fs-extra'; diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index 5d3f95d42c..ca0bc7853a 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -19,17 +19,17 @@ import fs from 'fs-extra'; import { Logger } from 'winston'; /* -provider username password +provider username password GitHub token 'x-oauth-basic' -GitHub App token 'x-access-token' -BitBucket 'x-token-auth' token -GitLab 'oauth2' token +GitHub App token 'x-access-token' +BitBucket 'x-token-auth' token +GitLab 'oauth2' token From : https://isomorphic-git.org/docs/en/onAuth Azure 'notempty' token */ -class SCM { - constructor( +export class Git { + private constructor( private readonly config: { username?: string; password?: string; @@ -85,7 +85,7 @@ class SCM { dir, singleBranch: true, depth: 1, - onProgress: this.onProgressHandler, + onProgress: this.onProgressHandler(), headers: { 'user-agent': 'git/@isomorphic-git', }, @@ -110,7 +110,7 @@ class SCM { http, dir, remote: remoteValue, - onProgress: this.onProgressHandler, + onProgress: this.onProgressHandler(), headers: { 'user-agent': 'git/@isomorphic-git', }, @@ -163,7 +163,7 @@ class SCM { fs, dir, http, - onProgress: this.onProgressHandler, + onProgress: this.onProgressHandler(), headers: { 'user-agent': 'git/@isomorphic-git', }, @@ -187,23 +187,28 @@ class SCM { password: this.config.password, }); - private onProgressHandler: ProgressCallback = event => { - const total = event.total - ? `${Math.round((event.loaded / event.total) * 100)}%` - : event.loaded; - this.config.logger?.info(`status={${event.phase},total={${total}}}`); - }; -} + private onProgressHandler = (): ProgressCallback => { + let currentPhase = ''; -// TODO(blam): This could potentially become something like for URL -// and use the integrations config for URLReading instead. -// But for now, I don't want to do all that in this PR. -export const fromAuth = ({ - username, - password, - logger, -}: { - username?: string; - password?: string; - logger?: Logger; -}) => new SCM({ username, password, logger }); + return event => { + if (currentPhase !== event.phase) { + currentPhase = event.phase; + this.config.logger?.info(event.phase); + } + const total = event.total + ? `${Math.round((event.loaded / event.total) * 100)}%` + : event.loaded; + this.config.logger?.debug(`status={${event.phase},total={${total}}}`); + }; + }; + + static fromAuth = ({ + username, + password, + logger, + }: { + username?: string; + password?: string; + logger?: Logger; + }) => new Git({ username, password, logger }); +} diff --git a/packages/backend-common/src/scm/index.ts b/packages/backend-common/src/scm/index.ts index 83a59310cd..e967fffb44 100644 --- a/packages/backend-common/src/scm/index.ts +++ b/packages/backend-common/src/scm/index.ts @@ -13,6 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import * as Git from './git'; -export { Git }; +export { Git } from './git'; From 610b3a205a123c1a51a5b768a9943b824b314196 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 29 Dec 2020 11:54:30 +0100 Subject: [PATCH 28/33] chore: try to fix the alignment --- packages/backend-common/src/scm/git.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index ca0bc7853a..690f464d35 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -20,8 +20,8 @@ import { Logger } from 'winston'; /* provider username password -GitHub token 'x-oauth-basic' -GitHub App token 'x-access-token' +GitHub token 'x-oauth-basic' +GitHub App token 'x-access-token' BitBucket 'x-token-auth' token GitLab 'oauth2' token From : https://isomorphic-git.org/docs/en/onAuth From 8bccc6bec034ac5f21e7303823bffb09ab020585 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 29 Dec 2020 13:32:24 +0100 Subject: [PATCH 29/33] chore: refactor variable names for isomorphic git. --- packages/backend-common/src/scm/git.test.ts | 26 ++++++++-------- packages/backend-common/src/scm/git.ts | 31 ++++++++++--------- packages/techdocs-common/src/helpers.ts | 4 +-- .../src/scaffolder/stages/publish/helpers.ts | 4 +-- 4 files changed, 33 insertions(+), 32 deletions(-) diff --git a/packages/backend-common/src/scm/git.test.ts b/packages/backend-common/src/scm/git.test.ts index 96c0f919a7..9af080a782 100644 --- a/packages/backend-common/src/scm/git.test.ts +++ b/packages/backend-common/src/scm/git.test.ts @@ -46,15 +46,15 @@ describe('Git', () => { it('should call isomorphic-git with the correct arguments', async () => { const git = Git.fromAuth({}); const dir = 'mockdirectory'; - const remoteName = 'origin'; + const remote = 'origin'; const url = 'git@github.com/something/sads'; - await git.addRemote({ dir, remoteName, url }); + await git.addRemote({ dir, remote, url }); expect(isomorphic.addRemote).toHaveBeenCalledWith({ fs, dir, - remote: remoteName, + remote, url, }); }); @@ -224,18 +224,18 @@ describe('Git', () => { name: 'comitter', email: 'test@backstage.io', }; - const headBranch = 'master'; - const baseBranch = 'production'; + const theirs = 'master'; + const ours = 'production'; const git = Git.fromAuth({}); - await git.merge({ dir, headBranch, baseBranch, author, committer }); + await git.merge({ dir, theirs, ours, author, committer }); expect(isomorphic.merge).toHaveBeenCalledWith({ fs, dir, - ours: baseBranch, - theirs: headBranch, + ours, + theirs, author, committer, }); @@ -244,7 +244,7 @@ describe('Git', () => { describe('push', () => { it('should call isomorphic-git with the correct arguments', async () => { - const remoteName = 'origin'; + const remote = 'origin'; const dir = '/some/mock/dir'; const auth = { username: 'blob', @@ -252,12 +252,12 @@ describe('Git', () => { }; const git = Git.fromAuth(auth); - await git.push({ dir, remoteName }); + await git.push({ dir, remote }); expect(isomorphic.push).toHaveBeenCalledWith({ fs, http, - remote: remoteName, + remote, dir, onProgress: expect.any(Function), headers: { @@ -267,7 +267,7 @@ describe('Git', () => { }); }); it('should pass a function that returns the authorization as the onAuth handler', async () => { - const remoteName = 'origin'; + const remote = 'origin'; const dir = '/some/mock/dir'; const auth = { username: 'blob', @@ -275,7 +275,7 @@ describe('Git', () => { }; const git = Git.fromAuth(auth); - await git.push({ remoteName, dir }); + await git.push({ remote, dir }); const { onAuth } = ((isomorphic.push as unknown) as jest.Mock< typeof isomorphic['push'] diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index 690f464d35..31b19ac327 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -46,16 +46,16 @@ export class Git { async addRemote({ dir, url, - remoteName, + remote, }: { dir: string; - remoteName: string; + remote: string; url: string; }) { this.config.logger?.info( - `Creating new remote {dir=${dir},remoteName=${remoteName},url=${url}}`, + `Creating new remote {dir=${dir},remote=${remote},url=${url}}`, ); - return git.addRemote({ fs, dir, remote: remoteName, url }); + return git.addRemote({ fs, dir, remote, url }); } async commit({ @@ -130,34 +130,35 @@ export class Git { // https://isomorphic-git.org/docs/en/merge async merge({ dir, - headBranch, - baseBranch, + theirs, + ours, author, committer, }: { dir: string; - headBranch: string; - baseBranch?: string; + theirs: string; + ours?: string; author: { name: string; email: string }; committer: { name: string; email: string }; }) { this.config.logger?.info( - `Merging branch '${headBranch}' into '${baseBranch}' for repository {dir=${dir}}`, + `Merging branch '${theirs}' into '${ours}' for repository {dir=${dir}}`, ); - // If baseBranch is undefined, current branch is used. + + // If ours is undefined, current branch is used. return git.merge({ fs, dir, - ours: baseBranch, - theirs: headBranch, + ours, + theirs, author, committer, }); } - async push({ dir, remoteName }: { dir: string; remoteName: string }) { + async push({ dir, remote }: { dir: string; remote: string }) { this.config.logger?.info( - `Pushing directory to remote {dir=${dir},remoteName=${remoteName}}`, + `Pushing directory to remote {dir=${dir},remote=${remote}}`, ); return git.push({ fs, @@ -167,7 +168,7 @@ export class Git { headers: { 'user-agent': 'git/@isomorphic-git', }, - remote: remoteName, + remote: remote, onAuth: this.onAuth, }); } diff --git a/packages/techdocs-common/src/helpers.ts b/packages/techdocs-common/src/helpers.ts index 8f59ead6fc..f70b0f3840 100644 --- a/packages/techdocs-common/src/helpers.ts +++ b/packages/techdocs-common/src/helpers.ts @@ -170,8 +170,8 @@ export const checkoutGitRepository = async ( await git.fetch({ dir: repositoryTmpPath, remote: 'origin' }); await git.merge({ dir: repositoryTmpPath, - headBranch: `origin/${currentBranchName}`, - baseBranch: currentBranchName || undefined, + theirs: `origin/${currentBranchName}`, + ours: currentBranchName || undefined, author: { name: 'Backstage TechDocs', email: 'techdocs@backstage.io' }, committer: { name: 'Backstage TechDocs', diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts index bf19c2f56d..796e48b8f8 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts @@ -59,11 +59,11 @@ export async function initRepoAndPush({ await git.addRemote({ dir, url: remoteUrl, - remoteName: 'origin', + remote: 'origin', }); await git.push({ dir, - remoteName: 'origin', + remote: 'origin', }); } From 00042e73c0b13e445d5847d98e2c9081d83a4951 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 29 Dec 2020 13:37:12 +0100 Subject: [PATCH 30/33] chore: added changeset for changed files --- .changeset/kind-buses-change.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/kind-buses-change.md diff --git a/.changeset/kind-buses-change.md b/.changeset/kind-buses-change.md new file mode 100644 index 0000000000..1d93e27dff --- /dev/null +++ b/.changeset/kind-buses-change.md @@ -0,0 +1,7 @@ +--- +'@backstage/backend-common': patch +'@backstage/techdocs-common': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Moving the Git actions to isomorphic-git instead of the node binding version of nodegit From 3af133465a89528c6cb81bc670ab8ffdcaa8764a Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 29 Dec 2020 13:42:54 +0100 Subject: [PATCH 31/33] chore: update value dictionary --- .github/styles/vocab.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index b4be37c057..e2b7f36067 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -134,6 +134,7 @@ neuro newrelic nginx Niklas +nodegit nohoist nonces npm From 82469a5d5a686c0a14da1d2a4712c54184b1a1f0 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 29 Dec 2020 14:13:07 +0100 Subject: [PATCH 32/33] chore: explicitly set the types returned from the functions --- packages/backend-common/src/scm/git.ts | 58 ++++++++++++++++++++------ 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index 31b19ac327..9007449d8b 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -13,7 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import git, { ProgressCallback } from 'isomorphic-git'; +import git, { + ProgressCallback, + MergeResult, + ReadCommitResult, +} from 'isomorphic-git'; import http from 'isomorphic-git/http/node'; import fs from 'fs-extra'; import { Logger } from 'winston'; @@ -37,7 +41,13 @@ export class Git { }, ) {} - async add({ dir, filepath }: { dir: string; filepath: string }) { + async add({ + dir, + filepath, + }: { + dir: string; + filepath: string; + }): Promise { this.config.logger?.info(`Adding file {dir=${dir},filepath=${filepath}}`); return git.add({ fs, dir, filepath }); @@ -51,7 +61,7 @@ export class Git { dir: string; remote: string; url: string; - }) { + }): Promise { this.config.logger?.info( `Creating new remote {dir=${dir},remote=${remote},url=${url}}`, ); @@ -68,7 +78,7 @@ export class Git { message: string; author: { name: string; email: string }; committer: { name: string; email: string }; - }) { + }): Promise { this.config.logger?.info( `Committing file to repo {dir=${dir},message=${message}}`, ); @@ -76,7 +86,7 @@ export class Git { return git.commit({ fs, dir, message, author, committer }); } - async clone({ url, dir }: { url: string; dir: string }) { + async clone({ url, dir }: { url: string; dir: string }): Promise { this.config.logger?.info(`Cloning repo {dir=${dir},url=${url}}`); return git.clone({ fs, @@ -94,18 +104,30 @@ export class Git { } // https://isomorphic-git.org/docs/en/currentBranch - async currentBranch({ dir, fullName }: { dir: string; fullName?: boolean }) { + async currentBranch({ + dir, + fullName, + }: { + dir: string; + fullName?: boolean; + }): Promise { const fullname = fullName ?? false; return git.currentBranch({ fs, dir, fullname }); } // https://isomorphic-git.org/docs/en/fetch - async fetch({ dir, remote }: { dir: string; remote?: string }) { + async fetch({ + dir, + remote, + }: { + dir: string; + remote?: string; + }): Promise { const remoteValue = remote ?? 'origin'; this.config.logger?.info( `Fetching remote=${remoteValue} for repository {dir=${dir}}`, ); - return git.fetch({ + await git.fetch({ fs, http, dir, @@ -118,7 +140,7 @@ export class Git { }); } - async init({ dir }: { dir: string }) { + async init({ dir }: { dir: string }): Promise { this.config.logger?.info(`Init git repository {dir=${dir}}`); return git.init({ @@ -140,7 +162,7 @@ export class Git { ours?: string; author: { name: string; email: string }; committer: { name: string; email: string }; - }) { + }): Promise { this.config.logger?.info( `Merging branch '${theirs}' into '${ours}' for repository {dir=${dir}}`, ); @@ -174,12 +196,24 @@ export class Git { } // https://isomorphic-git.org/docs/en/readCommit - async readCommit({ dir, sha }: { dir: string; sha: string }) { + async readCommit({ + dir, + sha, + }: { + dir: string; + sha: string; + }): Promise { return git.readCommit({ fs, dir, oid: sha }); } // https://isomorphic-git.org/docs/en/resolveRef - async resolveRef({ dir, ref }: { dir: string; ref: string }) { + async resolveRef({ + dir, + ref, + }: { + dir: string; + ref: string; + }): Promise { return git.resolveRef({ fs, dir, ref }); } From 28bdc0328c084caaef95c5621f27046398339b7a Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 29 Dec 2020 14:26:28 +0100 Subject: [PATCH 33/33] chore: rework types slightly for string or undefined instead of void --- packages/backend-common/src/scm/git.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index 9007449d8b..b0a1df6541 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -110,9 +110,11 @@ export class Git { }: { dir: string; fullName?: boolean; - }): Promise { + }): Promise { const fullname = fullName ?? false; - return git.currentBranch({ fs, dir, fullname }); + return git.currentBranch({ fs, dir, fullname }) as Promise< + string | undefined + >; } // https://isomorphic-git.org/docs/en/fetch